优化封面获取细节,并复用请求

This commit is contained in:
fantasticbin 2025-03-04 16:53:51 +08:00
parent 6e264ae8af
commit 185a1bad1c

View File

@ -2,6 +2,7 @@ package main
import (
"bytes"
"context"
"errors"
"fmt"
"github.com/spf13/viper"
@ -15,6 +16,7 @@ import (
"path/filepath"
"strconv"
"strings"
"time"
"unicode"
)
@ -22,6 +24,7 @@ type Crawler struct {
avPath string
outputPath string
config *viper.Viper
client *http.Client
}
func NewCrawler(avPath, outputPath string) *Crawler {
@ -38,6 +41,14 @@ func NewCrawler(avPath, outputPath string) *Crawler {
avPath: avPath,
outputPath: outputPath,
config: config,
client: &http.Client{
Timeout: 15 * time.Second,
Transport: &http.Transport{
MaxIdleConns: 10,
IdleConnTimeout: 30 * time.Second,
DisableCompression: true,
},
},
}
}
@ -119,31 +130,50 @@ func (c *Crawler) getCoverCodeList(files []string) (coverList []string) {
func (c *Crawler) fetchCoverImg(code string) error {
imgUrl := strings.ReplaceAll(c.config.GetString("crawler.url"), `*`, code)
suffix := filepath.Ext(imgUrl)
nameSlice := strings.Split(code, "00")
if len(nameSlice) < 2 {
offset := 0
// 如果第一个字符为 '1',则从下一个字符开始查找
if len(code) > 0 && code[0] == '1' {
offset = 1
}
// 获取号码所在的位置
index := strings.Index(code[offset:], "00")
if index == -1 {
return nil
}
if len(nameSlice[0]) > 4 {
nameSlice[0] = nameSlice[0][1:]
// 分隔字母部分及数字部分
letters := code[:index]
num := code[index+2:]
if offset > 0 {
letters = code[1:index]
}
fileName := filepath.Join(c.outputPath, fmt.Sprintf("%s-%s%s",
strings.ToUpper(nameSlice[0]),
nameSlice[1],
strings.ToUpper(letters),
num,
suffix,
))
req, err := http.NewRequest("GET", imgUrl, nil)
// 使用带超时的上下文
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
req, err := http.NewRequestWithContext(ctx, "GET", imgUrl, nil)
if err != nil {
return fmt.Errorf("创建请求失败: %w", err)
}
// 模拟浏览器请求
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36")
userAgents := []string{
"Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
"AppleWebKit/537.36 (KHTML, like Gecko)",
"Chrome/120.0.0.0 Safari/537.36",
}
req.Header.Set("User-Agent", userAgents[time.Now().UnixNano()%int64(len(userAgents))])
resp, err := http.DefaultClient.Do(req)
resp, err := c.client.Do(req)
if err != nil {
return fmt.Errorf("请求失败: %w", err)
}