增加蚊香社封面获取逻辑,及优化错误输出
This commit is contained in:
parent
5b102255d0
commit
28178fad38
11
config.yaml
11
config.yaml
@ -1,9 +1,18 @@
|
||||
crawler:
|
||||
code:
|
||||
prestige: [
|
||||
"abp",
|
||||
"abw",
|
||||
"abf",
|
||||
]
|
||||
video: [
|
||||
".mp4",
|
||||
".mkv",
|
||||
".avi",
|
||||
".wmv"
|
||||
]
|
||||
url: "https://awsimgsrc.dmm.co.jp/pics_dig/digital/video/*/*pl.jpg"
|
||||
url:
|
||||
# url规则(无须更改,除非规则已做调整)
|
||||
prestige: "https://www.prestige-av.com/api/media/goods/prestige/*/#/pb_*-#.jpg"
|
||||
other: "https://awsimgsrc.dmm.co.jp/pics_dig/digital/video/*/*pl.jpg"
|
||||
minHeight: 1000
|
33
crawler.go
33
crawler.go
@ -129,10 +129,15 @@ func (c *Crawler) getCoverCodeList(files []string) (coverList []coverCode) {
|
||||
return coverList
|
||||
}
|
||||
|
||||
// 获取封面图片
|
||||
func (c *Crawler) fetchCoverImg(code coverCode) error {
|
||||
if len(code.letters) < 2 || code.number < 1 {
|
||||
return nil
|
||||
// 组装封面图片地址
|
||||
func (c *Crawler) getCoverImgUrl(code coverCode) string {
|
||||
codePrestige := c.config.GetStringSlice("crawler.code.prestige")
|
||||
for _, pCode := range codePrestige {
|
||||
if code.letters == pCode {
|
||||
url := strings.ReplaceAll(c.config.GetString("crawler.url.prestige"), `*`, code.letters)
|
||||
url = strings.ReplaceAll(url, `#`, fmt.Sprintf("%03d", code.number))
|
||||
return url
|
||||
}
|
||||
}
|
||||
|
||||
format := "%s%05d"
|
||||
@ -141,14 +146,24 @@ func (c *Crawler) fetchCoverImg(code coverCode) error {
|
||||
}
|
||||
|
||||
codeStr := fmt.Sprintf(format, code.letters, code.number)
|
||||
imgUrl := strings.ReplaceAll(c.config.GetString("crawler.url"), `*`, codeStr)
|
||||
return strings.ReplaceAll(c.config.GetString("crawler.url.other"), `*`, codeStr)
|
||||
}
|
||||
|
||||
// 获取封面图片
|
||||
func (c *Crawler) fetchCoverImg(code coverCode) error {
|
||||
if len(code.letters) < 2 || code.number < 1 {
|
||||
return nil
|
||||
}
|
||||
|
||||
imgUrl := c.getCoverImgUrl(code)
|
||||
suffix := filepath.Ext(imgUrl)
|
||||
|
||||
fileName := filepath.Join(c.outputPath, fmt.Sprintf("%s-%03d%s",
|
||||
fileName := fmt.Sprintf("%s-%03d%s",
|
||||
strings.ToUpper(code.letters),
|
||||
code.number,
|
||||
suffix,
|
||||
))
|
||||
)
|
||||
filePath := filepath.Join(c.outputPath, fileName)
|
||||
|
||||
// 使用带超时的上下文
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
@ -181,7 +196,7 @@ func (c *Crawler) fetchCoverImg(code coverCode) error {
|
||||
// 先完整读取 HTTP Body
|
||||
imgData, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("读取图片数据失败: %w", err)
|
||||
return fmt.Errorf("读取封面数据失败(%s): %w", fileName, err)
|
||||
}
|
||||
|
||||
// 使用内存数据解码图片
|
||||
@ -196,7 +211,7 @@ func (c *Crawler) fetchCoverImg(code coverCode) error {
|
||||
}
|
||||
|
||||
// 将内存数据写入文件
|
||||
if err := os.WriteFile(fileName, imgData, 0644); err != nil {
|
||||
if err := os.WriteFile(filePath, imgData, 0644); err != nil {
|
||||
return fmt.Errorf("文件写入失败: %w", err)
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user