fantasticbin/cover-crawler
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

并行执行抓取逻辑

Browse Source
This commit is contained in:
fantasticbin 2025-03-02 21:12:10 +08:00
parent 7debfa6f36
commit a007d4a430
3 changed files with 13 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
crawler.go
View File

@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"github.com/spf13/viper"
"golang.org/x/sync/errgroup"
"image"
_ "image/jpeg"
"io"
@ -20,6 +21,7 @@ type Crawler struct {
avPath string
outputPath string
config *viper.Viper
g errgroup.Group
}
func NewCrawler(avPath, outputPath string) *Crawler {
@ -219,10 +221,14 @@ func (c *Crawler) Handle() error {
}
for _, cover := range coverList {
if err := c.fetchCoverImg(cover); err != nil {
return err
}
c.g.Go(func() error {
if err := c.fetchCoverImg(cover); err != nil {
return err
}
return nil
})
}
return nil
return c.g.Wait()
}

1
go.mod
View File

@ -6,6 +6,7 @@ require (
fyne.io/fyne/v2 v2.5.4
github.com/spf13/viper v1.19.0
golang.org/x/image v0.18.0
golang.org/x/sync v0.7.0
)
require (

2
go.sum
View File

@ -455,6 +455,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=