mutil-threading download

go/main/controller/download.go

@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 	"pikapi/main/database/comic_center"
 	"pikapi/main/utils"
+	"sync"
 	"time"
 )
 
@@ -17,12 +18,14 @@ import (
 // downloadRunning 如果为false则停止下载
 // downloadRestart 为true则取消从新启动下载功能
 
+var downloadThreadCount = 1
+var downloadThreadFetch = 100
+
 var downloadRunning = false
 var downloadRestart = false
 
 var downloadingComic *comic_center.ComicDownload
 var downloadingEp *comic_center.ComicDownloadEp
-var downloadingPicture *comic_center.ComicDownloadPicture
 
 var dlFlag = true
 
@@ -255,24 +258,46 @@ func downloadLoadPicture() {
 	if downloadHasStop() {
 		return
 	}
-	var err error
-	downloadingPicture, err = comic_center.LoadFirstNeedDownloadPicture(downloadingEp.ID)
+	// 获取到这个章节需要下载的图片
+	downloadingPictures, err := comic_center.LoadNeedDownloadPictures(downloadingEp.ID, downloadThreadFetch)
 	if err != nil {
 		panic(err)
 	}
-	go downloadInitPicture()
-}
-
-func downloadInitPicture() {
-	// 暂停检测
-	if downloadHasStop() {
-		return
-	}
-	if downloadingPicture == nil {
+	// 如果不需要下载
+	if len(*downloadingPictures) == 0 {
 		// 所有图片都下完了, 汇总EP下载情况
 		go downloadSummaryEp()
 		return
 	}
+	// 线程池
+	channel := make(chan int, downloadThreadCount)
+	defer close(channel)
+	wg := sync.WaitGroup{}
+	for i := 0; i < len(*downloadingPictures); i++ {
+		// 暂停检测
+		if downloadHasStop() {
+			wg.Wait()
+			return
+		}
+		channel <- 0
+		wg.Add(1)
+		// 不放入携程, 防止i已经变化
+		picPoint := &((*downloadingPictures)[i])
+		go func() {
+			downloadPicture(picPoint)
+			<-channel
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+	// 再次新一轮的下载, 直至 len(*downloadingPictures) == 0
+	go downloadLoadPicture()
+}
+
+var downloadEventChannelMutex = sync.Mutex{}
+
+// 这里不能使用暂停检测, 多次检测会导致问题
+func downloadPicture(downloadingPicture *comic_center.ComicDownloadPicture) {
 	// 下载图片, 最多重试5次
 	println("正在下载图片 " + fmt.Sprintf("%d", downloadingPicture.RankInEp))
 	for i := 0; i < 5; i++ {
@@ -280,22 +305,25 @@ func downloadInitPicture() {
 		if err != nil {
 			continue
 		}
-		// 对下载的漫画临时变量热更新并通知前端
-		downloadingPicture.DownloadFinished = true
-		downloadingEp.DownloadPictureCount = downloadingEp.DownloadPictureCount + 1
-		downloadingComic.DownloadPictureCount = downloadingComic.DownloadPictureCount + 1
-		downloadComicEventSend(downloadingComic)
+		func() {
+			downloadEventChannelMutex.Lock()
+			defer downloadEventChannelMutex.Unlock()
+			// 对下载的漫画临时变量热更新并通知前端
+			downloadingPicture.DownloadFinished = true
+			downloadingEp.DownloadPictureCount = downloadingEp.DownloadPictureCount + 1
+			downloadingComic.DownloadPictureCount = downloadingComic.DownloadPictureCount + 1
+			downloadComicEventSend(downloadingComic)
+		}()
 		break
 	}
 	// 没能下载成功, 图片置为下载失败
 	if !downloadingPicture.DownloadFinished {
 		err := comic_center.PictureFailed(downloadingPicture.ID)
 		if err != nil {
-			panic(err)
+			// ??? panic X channel ???
+			// panic(err)
 		}
 	}
-	// 加载下一张需要下载的图片
-	go downloadLoadPicture()
 }
 
 // 下载指定图片
@@ -371,8 +399,10 @@ func downloadSummaryEp() {
 	go downloadLoadEp()
 }
 
+// 边下载边导出(导出路径)
 var downloadAndExportPath = ""
 
+// 边下载边导出(导出图片)
 func downloadAndExport(
 	downloadingComic *comic_center.ComicDownload,
 	downloadingEp *comic_center.ComicDownloadEp,
@@ -412,12 +442,13 @@ func downloadAndExport(
 				return
 			}
 			// 写入文件
-			filePath := path.Join(epDir, fmt.Sprintf("%03d.%s", downloadingPicture.RankInEp, jFormat(format)))
+			filePath := path.Join(epDir, fmt.Sprintf("%03d.%s", downloadingPicture.RankInEp, aliasFormat(format)))
 			ioutil.WriteFile(filePath, buff, utils.CreateFileMode)
 		}
 	}
 }
 
+// 边下载边导出(导出logo)
 func downloadAndExportLogo(
 	downloadingComic *comic_center.ComicDownload,
 ) {
@@ -444,7 +475,7 @@ func downloadAndExportLogo(
 							return
 						}
 						// 写入文件
-						filePath := path.Join(comicDir, fmt.Sprintf("%s.%s", "logo", jFormat(f)))
+						filePath := path.Join(comicDir, fmt.Sprintf("%s.%s", "logo", aliasFormat(f)))
 						ioutil.WriteFile(filePath, buff, utils.CreateFileMode)
 					}
 				}
@@ -453,7 +484,8 @@ func downloadAndExportLogo(
 	}
 }
 
-func jFormat(format string) string {
+// jpeg的拓展名
+func aliasFormat(format string) string {
 	if format == "jpeg" {
 		return "jpg"
 	}

go/main/controller/pikapi.go

@@ -30,6 +30,7 @@ func InitPlugin(_remoteDir string, _downloadDir string, _tmpDir string) {
 	tmpDir = _tmpDir
 	comic_center.ResetAll()
 	downloadAndExportPath = loadDownloadAndExportPath()
+	downloadThreadCount = loadDownloadThreadCount()
 	go downloadBackground()
 	downloadRunning = true
 }
@@ -73,6 +74,25 @@ func loadDownloadAndExportPath() string {
 	return p
 }
 
+func saveDownloadThreadCount(value int) {
+	strValue := strconv.Itoa(value)
+	properties.SaveProperty("downloadThreadCount", strValue)
+	downloadThreadCount = value
+	downloadRestart = true
+}
+
+func loadDownloadThreadCount() int {
+	count, err := properties.LoadProperty("downloadThreadCount", "2")
+	if err != nil {
+		return 1
+	}
+	i, err := strconv.Atoi(count)
+	if err != nil {
+		return 1
+	}
+	return i
+}
+
 func setSwitchAddress(nSwitchAddress string) error {
 	err := properties.SaveSwitchAddress(nSwitchAddress)
 	if err != nil {
@@ -617,6 +637,15 @@ func FlatInvoke(method string, params string) (string, error) {
 		return loadDownloadAndExportPath(), nil
 	case "saveDownloadAndExportPath":
 		return "", saveDownloadAndExportPath(params)
+	case "saveDownloadThreadCount":
+		i, e := strconv.Atoi(params)
+		if e != nil {
+			return "", e
+		}
+		saveDownloadThreadCount(i)
+		return "", nil
+	case "loadDownloadThreadCount":
+		return strconv.Itoa(loadDownloadThreadCount()), nil
 	}
 	return "", errors.New("method not found : " + method)
 }

go/main/database/comic_center/center.go

@@ -350,22 +350,17 @@ func LoadFirstNeedDownloadEp(comicId string) (*ComicDownloadEp, error) {
 	return &ep, nil
 }
 
-func LoadFirstNeedDownloadPicture(epId string) (*ComicDownloadPicture, error) {
+// LoadNeedDownloadPictures 根据EP.ID获取需要下载的图片
+func LoadNeedDownloadPictures(epId string, limit int) (*[]ComicDownloadPicture, error) {
 	mutex.Lock()
 	defer mutex.Unlock()
-	var picture ComicDownloadPicture
-	err := db.First(
-		&picture,
-		"ep_id = ? AND download_failed = 0 AND download_finished = 0",
-		epId,
+	var pictures []ComicDownloadPicture
+	err := db.Find(
+		&pictures,
+		"ep_id = ? AND download_failed = 0 AND download_finished = 0 LIMIT ?",
+		epId, limit,
 	).Error
-	if err != nil {
-		if err == gorm.ErrRecordNotFound {
-			return nil, nil
-		}
-		return nil, err
-	}
-	return &picture, nil
+	return &pictures, err
 }
 
 func FetchPictures(comicId string, epId string, list *[]ComicDownloadPicture) error {

lib/basic/Method.dart

@@ -588,4 +588,16 @@ class Method {
       "flag": flag,
     });
   }
+
+  /// 获取下载线程数量
+  Future<int> loadDownloadThreadCount() async {
+    var strValue = await _flatInvoke("loadDownloadThreadCount", {});
+    return int.parse(strValue);
+  }
+
+  /// 设置下载线程数
+  Future saveDownloadThreadCount(int value) {
+    return _flatInvoke("saveDownloadThreadCount", "$value");
+  }
+
 }

lib/basic/config/DownloadThreadCount.dart

@@ -0,0 +1,30 @@
+import 'package:flutter/cupertino.dart';
+import 'package:flutter/material.dart';
+import 'package:pikapi/basic/Common.dart';
+import 'package:pikapi/basic/Method.dart';
+
+late int _downloadThreadCount;
+const _values = [1, 2, 3, 4, 5];
+
+Future initDownloadThreadCount() async {
+  _downloadThreadCount = await method.loadDownloadThreadCount();
+}
+
+Widget downloadThreadCountSetting() {
+  return StatefulBuilder(
+    builder: (BuildContext context, void Function(void Function()) setState) {
+      return ListTile(
+        title: Text("下载线程数"),
+        subtitle: Text("$_downloadThreadCount"),
+        onTap: () async {
+          int? value = await chooseListDialog(context, "选择下载线程数", _values);
+          if (value != null) {
+            await method.saveDownloadThreadCount(value);
+            _downloadThreadCount = value;
+            setState(() {});
+          }
+        },
+      );
+    },
+  );
+}

lib/screens/InitScreen.dart

@@ -7,6 +7,7 @@ import 'package:pikapi/basic/config/AutoFullScreen.dart';
 import 'package:pikapi/basic/config/ChooserRoot.dart';
 import 'package:pikapi/basic/config/ContentFailedReloadAction.dart';
 import 'package:pikapi/basic/config/DownloadAndExportPath.dart';
+import 'package:pikapi/basic/config/DownloadThreadCount.dart';
 import 'package:pikapi/basic/config/FullScreenAction.dart';
 import 'package:pikapi/basic/config/FullScreenUI.dart';
 import 'package:pikapi/basic/config/KeyboardController.dart';
@@ -65,6 +66,7 @@ class _InitScreenState extends State<InitScreen> {
     await initTimeZone();
     await initDownloadAndExportPath();
     await initAndroidSecureFlag();
+    await initDownloadThreadCount();
     // 登录, 如果token失效重新登录, 网络不好的时候可能需要1分钟
     if (await method.preLogin()) {
       // 如果token或username+password有效则直接进入登录好的界面

lib/screens/SettingsScreen.dart

@@ -10,6 +10,7 @@ import 'package:pikapi/basic/config/AutoFullScreen.dart';
 import 'package:pikapi/basic/config/ChooserRoot.dart';
 import 'package:pikapi/basic/config/ContentFailedReloadAction.dart';
 import 'package:pikapi/basic/config/DownloadAndExportPath.dart';
+import 'package:pikapi/basic/config/DownloadThreadCount.dart';
 import 'package:pikapi/basic/config/FullScreenAction.dart';
 import 'package:pikapi/basic/config/FullScreenUI.dart';
 import 'package:pikapi/basic/config/KeyboardController.dart';
@@ -132,6 +133,7 @@ class _SettingsScreenState extends State<SettingsScreen> {
                 setState(() {});
               },
             ),
+            downloadThreadCountSetting(),
             downloadAndExportPathSetting(),
             fontSetting(),
             Divider(),