|
|
@ -22,16 +22,16 @@ func NewCrawler() *Crawler {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func (c *Crawler) Start() {
|
|
|
|
func (c *Crawler) Start() {
|
|
|
|
c.path = output
|
|
|
|
c.path = config.output
|
|
|
|
if folderTitleUrl != "" {
|
|
|
|
if config.folderTitleUrl != "" {
|
|
|
|
chapterTitle := FindChapterTitle(folderTitleUrl)
|
|
|
|
chapterTitle := FindChapterTitle(config.folderTitleUrl)
|
|
|
|
title := strings.Join([]string{
|
|
|
|
title := strings.Join([]string{
|
|
|
|
"第",
|
|
|
|
"第",
|
|
|
|
strconv.Itoa(chapter),
|
|
|
|
strconv.Itoa(config.chapter),
|
|
|
|
"话-",
|
|
|
|
"话-",
|
|
|
|
chapterTitle,
|
|
|
|
chapterTitle,
|
|
|
|
}, "")
|
|
|
|
}, "")
|
|
|
|
c.path = filepath.Join(output, title) // 组装章节路径
|
|
|
|
c.path = filepath.Join(config.output, title) // 组装章节路径
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
err := IfPathNotExistDoMkdir(c.path)
|
|
|
|
err := IfPathNotExistDoMkdir(c.path)
|
|
|
@ -40,8 +40,8 @@ func (c *Crawler) Start() {
|
|
|
|
return
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
c.wg.Add(max)
|
|
|
|
c.wg.Add(config.max)
|
|
|
|
for i := 1; i <= max; i++ {
|
|
|
|
for i := 1; i <= config.max; i++ {
|
|
|
|
go c.do(i)
|
|
|
|
go c.do(i)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
@ -52,12 +52,12 @@ func (c *Crawler) do(num int) {
|
|
|
|
defer c.wg.Done()
|
|
|
|
defer c.wg.Done()
|
|
|
|
|
|
|
|
|
|
|
|
// 兼容未携带斜杆的地址
|
|
|
|
// 兼容未携带斜杆的地址
|
|
|
|
if url[0] != '/' {
|
|
|
|
if config.url[0] != '/' {
|
|
|
|
url = "/" + url
|
|
|
|
config.url = "/" + config.url
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
numString := strconv.Itoa(num)
|
|
|
|
numString := strconv.Itoa(num)
|
|
|
|
urlSlice := strings.Split(url, "_") // 取URL组装
|
|
|
|
urlSlice := strings.Split(config.url, "_") // 取URL组装
|
|
|
|
fileSlice := strings.Split(urlSlice[1], ".") // 取后缀名
|
|
|
|
fileSlice := strings.Split(urlSlice[1], ".") // 取后缀名
|
|
|
|
fileName := strings.Join([]string{ // 组装文件名
|
|
|
|
fileName := strings.Join([]string{ // 组装文件名
|
|
|
|
numString,
|
|
|
|
numString,
|
|
|
@ -70,7 +70,7 @@ func (c *Crawler) do(num int) {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
imgUrl := strings.Join([]string{ // 组装图片URL
|
|
|
|
imgUrl := strings.Join([]string{ // 组装图片URL
|
|
|
|
host,
|
|
|
|
config.host,
|
|
|
|
urlSlice[0],
|
|
|
|
urlSlice[0],
|
|
|
|
"_",
|
|
|
|
"_",
|
|
|
|
urlNum,
|
|
|
|
urlNum,
|
|
|
|