feat: 增加基于sqlite的数据持久化能力

.gitignore

@@ -3,7 +3,7 @@ tmp
 build
 node_modules
 main
-data
+data/busuanzi.db
 .idea
 busuanzi
 *.rdb

core/sync.go

@@ -0,0 +1,131 @@
+package core
+
+import (
+	"context"
+	"log"
+	"time"
+
+	"github.com/redis/go-redis/v9"
+
+	"github.com/soxft/busuanzi/library/database"
+	"github.com/soxft/busuanzi/process/redisutil"
+)
+
+// LoadDataFromSQLite 从SQLite加载数据到Redis
+func LoadDataFromSQLite(ctx context.Context) error {
+	// 加载站点统计数据
+	var siteStats []struct {
+		SiteUnique string
+		SitePv     int64
+		SiteUv     int64
+	}
+	err := database.DB.WithContext(ctx).Table("site_statistics").Find(&siteStats).Error
+	if err != nil {
+		return err
+	}
+
+	// 加载页面统计数据
+	var pageStats []struct {
+		SiteUnique string
+		PathUnique string
+		PagePv     int64
+		PageUv     int64
+	}
+	err = database.DB.WithContext(ctx).Table("page_statistics").Find(&pageStats).Error
+	if err != nil {
+		return err
+	}
+
+	_redis := redisutil.RDB
+	// 更新站点统计数据
+	for _, stat := range siteStats {
+		rk := RKeys{
+			SitePvKey: "bsz:site_pv:" + stat.SiteUnique,
+			SiteUvKey: "bsz:site_uv:" + stat.SiteUnique,
+		}
+		_redis.Set(ctx, rk.SitePvKey, stat.SitePv, 0)
+		_redis.PFAdd(ctx, rk.SiteUvKey, "1")
+	}
+
+	// 更新页面统计数据
+	for _, stat := range pageStats {
+		rk := RKeys{
+			PagePvKey: "bsz:page_pv:" + stat.SiteUnique,
+			PageUvKey: "bsz:page_uv:" + stat.SiteUnique,
+		}
+		_redis.ZAdd(ctx, rk.PagePvKey, redis.Z{Score: float64(stat.PagePv), Member: stat.PathUnique})
+		_redis.PFAdd(ctx, rk.PageUvKey, "1")
+	}
+	return nil
+}
+
+// SyncToSQLite 将Redis数据同步到SQLite
+func SyncToSQLite(ctx context.Context) error {
+	_redis := redisutil.RDB
+	// 获取所有需要同步的站点
+	keys, err := _redis.Keys(ctx, "bsz:site_pv:*").Result()
+	if err != nil {
+		return err
+	}
+
+	for _, key := range keys {
+		siteUnique := key[len("bsz:site_pv:"):]
+
+		// 获取站点统计数据
+		sitePv, _ := _redis.Get(ctx, "bsz:site_pv:"+siteUnique).Int64()
+		siteUv, _ := _redis.PFCount(ctx, "bsz:site_uv:"+siteUnique).Result()
+
+		// 更新站点统计数据
+		err = database.DB.Exec(`
+			INSERT INTO site_statistics (site_unique, site_pv, site_uv)
+			VALUES (?, ?, ?)
+			ON CONFLICT(site_unique) DO UPDATE SET
+			site_pv = ?,
+			site_uv = ?,
+			updated_at = CURRENT_TIMESTAMP
+		`, siteUnique, sitePv, siteUv, sitePv, siteUv).Error
+		if err != nil {
+			log.Printf("同步站点数据失败: %v", err)
+			continue
+		}
+
+		// 获取该站点的所有页面PV
+		pageData, _ := _redis.ZRangeWithScores(ctx, "bsz:page_pv:"+siteUnique, 0, -1).Result()
+		for _, data := range pageData {
+			pathUnique := data.Member.(string)
+			pagePv := int64(data.Score)
+
+			// 更新页面统计数据
+			pageUv, _ := _redis.PFCount(ctx, "bsz:page_uv:"+siteUnique+":"+pathUnique).Result()
+			err = database.DB.Exec(`
+				INSERT INTO page_statistics (site_unique, path_unique, page_pv, page_uv)
+				VALUES (?, ?, ?, ?)
+				ON CONFLICT(site_unique, path_unique) DO UPDATE SET
+				page_pv = ?,
+				page_uv = ?,
+				updated_at = CURRENT_TIMESTAMP
+			`, siteUnique, pathUnique, pagePv, pageUv, pagePv, pageUv).Error
+			if err != nil {
+				log.Printf("同步页面数据失败: %v", err)
+			}
+		}
+	}
+	return nil
+}
+
+// StartSyncTask 启动定时同步任务
+func StartSyncTask(ctx context.Context, interval time.Duration) {
+	ticker := time.NewTicker(interval)
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				if err := SyncToSQLite(ctx); err != nil {
+					log.Printf("同步到SQLite失败: %v", err)
+				}
+			}
+		}
+	}()
+}

data/readme.md

@@ -0,0 +1 @@
+这个目录下存放一些持久化的数据，比如sqlite的数据文件。

go.mod

@@ -4,8 +4,11 @@ go 1.22.3
 
 require (
 	github.com/gin-gonic/gin v1.10.0
+	github.com/glebarez/sqlite v1.11.0
+	github.com/orcaman/concurrent-map/v2 v2.0.1
 	github.com/redis/go-redis/v9 v9.6.1
 	github.com/spf13/viper v1.19.0
+	gorm.io/gorm v1.25.12
 )
 
 require (
@@ -15,14 +18,19 @@ require (
 	github.com/cloudwego/base64x v0.1.4 // indirect
 	github.com/cloudwego/iasm v0.2.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.5 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/glebarez/go-sqlite v1.21.2 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-playground/validator/v10 v10.22.1 // indirect
 	github.com/goccy/go-json v0.10.3 // indirect
+	github.com/google/uuid v1.4.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
@@ -31,8 +39,8 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/orcaman/concurrent-map/v2 v2.0.1 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/sagikazarmark/locafero v0.6.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
@@ -52,4 +60,8 @@ require (
 	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	modernc.org/libc v1.22.5 // indirect
+	modernc.org/mathutil v1.5.0 // indirect
+	modernc.org/memory v1.5.0 // indirect
+	modernc.org/sqlite v1.23.1 // indirect
 )

go.sum

@@ -19,6 +19,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
@@ -29,6 +31,10 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU=
 github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y=
+github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo=
+github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k=
+github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw=
+github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ=
 github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
 github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
@@ -42,8 +48,16 @@ github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PU
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
+github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
+github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
+github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
+github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
@@ -76,6 +90,9 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4=
 github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA=
+github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk=
@@ -134,4 +151,14 @@ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8=
+gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=
+modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE=
+modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY=
+modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
+modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
+modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds=
+modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
+modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM=
+modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk=
 nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=

library/database/db.go

@@ -0,0 +1,64 @@
+package database
+
+import (
+	"log"
+	"sync"
+
+	"github.com/glebarez/sqlite"
+	"gorm.io/gorm"
+)
+
+var (
+	DB   *gorm.DB
+	once sync.Once
+)
+
+func InitDB(dbPath string) {
+	once.Do(func() {
+		var err error
+		DB, err = gorm.Open(sqlite.Open(dbPath), &gorm.Config{
+			// 禁用外键(指定外键时不会在mysql创建真实的外键约束)
+			DisableForeignKeyConstraintWhenMigrating: true,
+		})
+		if err != nil {
+			log.Panicf("failed to connect sqlite3: %v", err)
+		}
+		dbObj, err := DB.DB()
+		if err != nil {
+			log.Panicf("failed to get sqlite3 obj: %v", err)
+		}
+		// 参见： https://github.com/glebarez/sqlite/issues/52
+		dbObj.SetMaxOpenConns(1)
+
+		// 创建站点统计表
+		err = DB.Exec(`
+			CREATE TABLE IF NOT EXISTS site_statistics (
+				id INTEGER PRIMARY KEY AUTOINCREMENT,
+				site_unique TEXT NOT NULL UNIQUE,
+				site_pv INTEGER DEFAULT 0,
+				site_uv INTEGER DEFAULT 0,
+				updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+			)
+		`).Error
+		if err != nil {
+			log.Fatalf("创建站点统计表失败: %v", err)
+		}
+
+		// 创建页面统计表
+		err = DB.Exec(`
+			CREATE TABLE IF NOT EXISTS page_statistics (
+				id INTEGER PRIMARY KEY AUTOINCREMENT,
+				site_unique TEXT NOT NULL,
+				path_unique TEXT NOT NULL,
+				page_pv INTEGER DEFAULT 0,
+				page_uv INTEGER DEFAULT 0,
+				updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+					UNIQUE(site_unique, path_unique),
+					FOREIGN KEY(site_unique) REFERENCES site_statistics(site_unique)
+			)
+		`).Error
+		if err != nil {
+			log.Fatalf("创建页面统计表失败: %v", err)
+		}
+	})
+}

main.go

@@ -1,8 +1,16 @@
 package main
 
 import (
+	"context"
+	"log"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
 	"github.com/soxft/busuanzi/config"
 	"github.com/soxft/busuanzi/core"
+	"github.com/soxft/busuanzi/library/database"
 	"github.com/soxft/busuanzi/process/redisutil"
 	"github.com/soxft/busuanzi/process/webutil"
 )
@@ -13,5 +21,35 @@ func main() {
 
 	core.InitExpire()
 
+	// 初始化SQLite
+	database.InitDB("./data/busuanzi.db")
+
+	// 创建上下文用于控制后台任务
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// 从SQLite加载数据到Redis
+	if err := core.LoadDataFromSQLite(ctx); err != nil {
+		log.Printf("从SQLite加载数据失败: %v", err)
+	}
+
+	// 启动定时同步任务（每2分钟同步一次）
+	core.StartSyncTask(ctx, 5*time.Minute)
+
+	// 设置优雅退出
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
+
+	go func() {
+		<-c
+		log.Println("正在关闭服务...")
+		cancel()
+		// 最后同步一次数据
+		if err := core.SyncToSQLite(context.Background()); err != nil {
+			log.Printf("最终数据同步失败: %v", err)
+		}
+		os.Exit(0)
+	}()
+
 	webutil.Init()
 }