♻️ Refactor the blocktree storage #11773

siyuan-note · Jun 20, 2024 · 7a3d4a0 · 7a3d4a0
1 parent 32413fa
commit 7a3d4a0
Show file tree

Hide file tree

Showing 22 changed files with 443 additions and 487 deletions.
diff --git a/app/appearance/langs/en_US.json b/app/appearance/langs/en_US.json
@@ -1377,7 +1377,7 @@
     "88": "Finished parsing [%d] data files, remaining to be processed [%d]",
     "89": "[%d/%d] Created [%d] of data indexes of block-level elements [%s]",
     "90": "[%d/%d] Created [%d] of search indexes of block-level elements [%s]",
-    "91": "Reading block tree data...",
+    "91": "TODO",
     "92": "Parsing document tree [%s]",
     "93": "[%d/%d] Cleaned up the index related to document [%s]",
     "94": "Upload failed: %s",

diff --git a/app/appearance/langs/es_ES.json b/app/appearance/langs/es_ES.json
@@ -1377,7 +1377,7 @@
     "88": "Se ha terminado de analizar [%d] archivos de datos, quedan por procesar [%d]",
     "89": "[%d/%d] Creado [%d] de índices de datos de elementos a nivel de bloque [%s]",
     "90": "[%d/%d] Creado [%d] de índices de búsqueda de elementos a nivel de bloque [%s]",
-    "91": "Leyendo datos del árbol de bloques...",
+    "91": "TODO",
     "92": "Analizando el árbol del documento [%s]",
     "93": "[%d/%d] ha limpiado el índice relacionado con el documento [%s]",
     "94": "Carga fallida: %s",

diff --git a/app/appearance/langs/fr_FR.json b/app/appearance/langs/fr_FR.json
@@ -1377,7 +1377,7 @@
     "88": "Fin de l'analyse des fichiers de données [%d], restant à traiter [%d]",
     "89": "[%d/%d] Créé [%d] d'index de données d'éléments de niveau bloc [%s]",
     "90": "[%d/%d] Création de [%d] index de recherche d'éléments de niveau bloc [%s]",
-    "91": "Lecture des données de l'arborescence des blocs...",
+    "91": "TODO",
     "92": "Analyse de l'arborescence du document [%s]",
     "93": "[%d/%d] a nettoyé l'index lié au document [%s]",
     "94": "Échec du téléchargement : %s",

diff --git a/app/appearance/langs/ja_JP.json b/app/appearance/langs/ja_JP.json
@@ -1377,7 +1377,7 @@
     "88": "[%d] 個のデータファイルの解析が完了し、処理待ちのデータファイルが [%d] 個残っています",
     "89": "[%d/%d] ブロックレベル要素 [%s] のデータインデックスを [%d] 個作成しました",
     "90": "[%d/%d] ブロックレベル要素 [%s] の検索インデックスを [%d] 個作成しました",
-    "91": "ブロックツリーデータを読み込んでいます...",
+    "91": "TODO",
     "92": "ドキュメントツリーを解析しています [%s]",
     "93": "[%d/%d] ドキュメント [%s] に関連するインデックスをクリーンアップしました",
     "94": "アップロードに失敗しました: %s",

diff --git a/app/appearance/langs/zh_CHT.json b/app/appearance/langs/zh_CHT.json
@@ -1377,7 +1377,7 @@
     "88": "已完成解析 [%d] 個資料文件，剩餘待處理 [%d]",
     "89": "[%d/%d] 已經創建 [%d] 個塊級元素的資料索引 [%s]",
     "90": "[%d/%d] 已經創建 [%d] 個塊級元素的搜索索引 [%s]",
-    "91": "正在讀取塊樹資料...",
+    "91": "TODO",
     "92": "正在解析文檔樹 [%s]",
     "93": "[%d/%d] 已經清理文檔 [%s] 相關的索引",
     "94": "上傳失敗：%s",

diff --git a/app/appearance/langs/zh_CN.json b/app/appearance/langs/zh_CN.json
@@ -1377,7 +1377,7 @@
     "88": "已完成解析 [%d] 个数据文件，剩余待处理 [%d]",
     "89": "[%d/%d] 已经创建 [%d] 个块级元素的数据索引 [%s]",
     "90": "[%d/%d] 已经创建 [%d] 个块级元素的搜索索引 [%s]",
-    "91": "正在读取块树数据...",
+    "91": "TODO",
     "92": "正在解析文档树 [%s]",
     "93": "[%d/%d] 已经清理文档 [%s] 相关的索引",
     "94": "上传失败：%s",

diff --git a/kernel/filesys/tree.go b/kernel/filesys/tree.go
@@ -88,7 +88,7 @@ func LoadTreeByData(data []byte, boxID, p string, luteEngine *lute.Lute) (ret *p
 					logging.LogErrorf("rebuild parent tree [%s] failed: %s", parentAbsPath, writeErr)
 				} else {
 					logging.LogInfof("rebuilt parent tree [%s]", parentAbsPath)
-					treenode.IndexBlockTree(parentTree)
+					treenode.UpsertBlockTree(parentTree)
 				}
 			} else {
 				logging.LogWarnf("read parent tree data [%s] failed: %s", parentAbsPath, readErr)
@@ -137,7 +137,7 @@ func prepareWriteTree(tree *parse.Tree) (data []byte, filePath string, err error
 		newP := treenode.NewParagraph()
 		tree.Root.AppendChild(newP)
 		tree.Root.SetIALAttr("updated", util.TimeFromID(newP.ID))
-		treenode.IndexBlockTree(tree)
+		treenode.UpsertBlockTree(tree)
 	}
 
 	filePath = filepath.Join(util.DataDir, tree.Box, tree.Path)

diff --git a/kernel/job/cron.go b/kernel/job/cron.go
@@ -23,14 +23,12 @@ import (
 	"github.com/siyuan-note/siyuan/kernel/model"
 	"github.com/siyuan-note/siyuan/kernel/sql"
 	"github.com/siyuan-note/siyuan/kernel/task"
-	"github.com/siyuan-note/siyuan/kernel/treenode"
 	"github.com/siyuan-note/siyuan/kernel/util"
 )
 
 func StartCron() {
 	go every(100*time.Millisecond, task.ExecTaskJob)
 	go every(5*time.Second, task.StatusJob)
-	go every(5*time.Second, treenode.SaveBlockTreeJob)
 	go every(5*time.Second, model.SyncDataJob)
 	go every(2*time.Hour, model.StatJob)
 	go every(2*time.Hour, model.RefreshCheckJob)

diff --git a/kernel/model/assets.go b/kernel/model/assets.go
@@ -825,7 +825,7 @@ func RenameAsset(oldPath, newName string) (err error) {
 					continue
 				}
 
-				treenode.IndexBlockTree(tree)
+				treenode.UpsertBlockTree(tree)
 				sql.UpsertTreeQueue(tree)
 
 				util.PushEndlessProgress(fmt.Sprintf(Conf.Language(111), util.EscapeHTML(tree.Root.IALAttr("title"))))

diff --git a/kernel/model/box.go b/kernel/model/box.go
@@ -531,7 +531,6 @@ func fullReindex() {
 	for _, openedBox := range openedBoxes {
 		index(openedBox.ID)
 	}
-	treenode.SaveBlockTree(true)
 	LoadFlashcards()
 	debug.FreeOSMemory()
 }

diff --git a/kernel/model/conf.go b/kernel/model/conf.go
@@ -627,7 +627,6 @@ func Close(force, setCurrentWorkspace bool, execInstallPkg int) (exitCode int) {
 
 	Conf.Close()
 	sql.CloseDatabase()
-	treenode.SaveBlockTree(false)
 	util.SaveAssetsTexts()
 	clearWorkspaceTemp()
 	clearCorruptedNotebooks()
@@ -818,24 +817,7 @@ func (conf *AppConf) language(num int) (ret string) {
 }
 
 func InitBoxes() {
-	initialized := false
-	if 1 > treenode.CountBlocks() {
-		if gulu.File.IsExist(util.BlockTreePath) {
-			util.IncBootProgress(20, Conf.Language(91))
-			go func() {
-				for i := 0; i < 40; i++ {
-					util.RandomSleep(50, 100)
-					util.IncBootProgress(1, Conf.Language(91))
-				}
-			}()
-
-			treenode.InitBlockTree(false)
-			initialized = true
-		}
-	} else { // 大于 1 的话说明在同步阶段已经加载过了
-		initialized = true
-	}
-
+	initialized := 0 < treenode.CountBlocks()
 	for _, box := range Conf.GetOpenedBoxes() {
 		box.UpdateHistoryGenerated() // 初始化历史生成时间为当前时间
 
@@ -844,10 +826,6 @@ func InitBoxes() {
 		}
 	}
 
-	if !initialized {
-		treenode.SaveBlockTree(true)
-	}
-
 	var dbSize string
 	if dbFile, err := os.Stat(util.DBPath); nil == err {
 		dbSize = humanize.BytesCustomCeil(uint64(dbFile.Size()), 2)
@@ -982,7 +960,8 @@ func clearWorkspaceTemp() {
 	os.RemoveAll(filepath.Join(util.TempDir, "import"))
 	os.RemoveAll(filepath.Join(util.TempDir, "repo"))
 	os.RemoveAll(filepath.Join(util.TempDir, "os"))
-	os.RemoveAll(filepath.Join(util.TempDir, "blocktree.msgpack")) // v2.7.2 前旧版的块数数据
+	os.RemoveAll(filepath.Join(util.TempDir, "blocktree.msgpack")) // v2.7.2 前旧版的块树数据
+	os.RemoveAll(filepath.Join(util.TempDir, "blocktree"))         // v3.1.0 前旧版的块树数据
 
 	// 退出时自动删除超过 7 天的安装包 https://github.com/siyuan-note/siyuan/issues/6128
 	install := filepath.Join(util.TempDir, "install")

diff --git a/kernel/model/file.go b/kernel/model/file.go
@@ -1086,7 +1086,7 @@ func indexWriteTreeIndexQueue(tree *parse.Tree) (err error) {
 }
 
 func indexWriteTreeUpsertQueue(tree *parse.Tree) (err error) {
-	treenode.IndexBlockTree(tree)
+	treenode.UpsertBlockTree(tree)
 	return writeTreeUpsertQueue(tree)
 }
 
@@ -1095,7 +1095,7 @@ func renameWriteJSONQueue(tree *parse.Tree) (err error) {
 		return
 	}
 	sql.RenameTreeQueue(tree)
-	treenode.IndexBlockTree(tree)
+	treenode.UpsertBlockTree(tree)
 	return
 }
 

diff --git a/kernel/model/index_fix.go b/kernel/model/index_fix.go
@@ -478,7 +478,7 @@ func reindexTree0(tree *parse.Tree, i, size int) {
 		tree.Root.SetIALAttr("updated", updated)
 		indexWriteTreeUpsertQueue(tree)
 	} else {
-		treenode.IndexBlockTree(tree)
+		treenode.UpsertBlockTree(tree)
 		sql.IndexTreeQueue(tree)
 	}
 

diff --git a/kernel/model/mount.go b/kernel/model/mount.go
@@ -30,7 +30,6 @@ import (
 	"github.com/88250/lute/ast"
 	"github.com/siyuan-note/filelock"
 	"github.com/siyuan-note/logging"
-	"github.com/siyuan-note/siyuan/kernel/treenode"
 	"github.com/siyuan-note/siyuan/kernel/util"
 )
 
@@ -247,7 +246,6 @@ func Mount(boxID string) (alreadyMount bool, err error) {
 	box.Index()
 	// 缓存根一级的文档树展开
 	ListDocTree(box.ID, "/", util.SortModeUnassigned, false, false, Conf.FileTree.MaxListCount)
-	treenode.SaveBlockTree(false)
 	util.ClearPushProgress(100)
 
 	if reMountGuide {

diff --git a/kernel/model/sync.go b/kernel/model/sync.go
@@ -352,7 +352,7 @@ func upsertIndexes(upsertFilePaths []string) (upsertRootIDs []string) {
 		if nil != err0 {
 			continue
 		}
-		treenode.IndexBlockTree(tree)
+		treenode.UpsertBlockTree(tree)
 		sql.UpsertTreeQueue(tree)
 
 		bts := treenode.GetBlockTreesByRootID(tree.ID)

diff --git a/kernel/model/transaction.go b/kernel/model/transaction.go
@@ -1378,7 +1378,7 @@ func (tx *Transaction) loadTree(id string) (ret *parse.Tree, err error) {
 
 func (tx *Transaction) writeTree(tree *parse.Tree) (err error) {
 	tx.trees[tree.ID] = tree
-	treenode.IndexBlockTree(tree)
+	treenode.UpsertBlockTree(tree)
 	return
 }
 

diff --git a/kernel/model/tree.go b/kernel/model/tree.go
@@ -274,7 +274,7 @@ func searchTreeInFilesystem(rootID string) {
 		return
 	}
 
-	treenode.IndexBlockTree(tree)
+	treenode.UpsertBlockTree(tree)
 	sql.IndexTreeQueue(tree)
 	logging.LogInfof("reindexed tree by filesystem [rootID=%s]", rootID)
 }
diff --git a/kernel/sql/database.go b/kernel/sql/database.go
@@ -82,6 +82,7 @@ func InitDatabase(forceRebuild bool) (err error) {
 	}
 
 	initDBConnection()
+	treenode.InitBlockTree(forceRebuild)
 
 	if !forceRebuild {
 		// 检查数据库结构版本，如果版本不一致的话说明改过表结构，需要重建
@@ -101,9 +102,6 @@ func InitDatabase(forceRebuild bool) (err error) {
 			err = nil
 		}
 	}
-	if gulu.File.IsExist(util.BlockTreePath) {
-		treenode.InitBlockTree(true)
-	}
 
 	initDBConnection()
 	initDBTables()
@@ -1278,6 +1276,11 @@ func CloseDatabase() {
 		logging.LogErrorf("close history database failed: %s", err)
 		return
 	}
+	if err := assetContentDB.Close(); nil != err {
+		logging.LogErrorf("close asset content database failed: %s", err)
+		return
+	}
+	treenode.CloseDatabase()
 	logging.LogInfof("closed database")
 }