From 5cffab864939e08bdf351feb9c8fc9b64934cdfa Mon Sep 17 00:00:00 2001 From: LinkinStars Date: Fri, 10 Jan 2025 10:43:35 +0800 Subject: [PATCH] feat(service): implement file cleanup and deletion functionality --- cmd/wire_gen.go | 12 +- configs/config.yaml | 3 + go.mod | 2 +- internal/base/constant/upload.go | 1 + internal/base/cron/cron.go | 43 ++++- internal/controller/upload_controller.go | 9 +- internal/entity/file_record_entity.go | 45 +++++ internal/migrations/migrations.go | 1 + internal/migrations/v25.go | 31 ++++ internal/repo/answer/answer_repo.go | 19 +- internal/repo/file_record/file_record_repo.go | 84 +++++++++ internal/repo/provider.go | 2 + internal/repo/question/question_repo.go | 17 ++ internal/repo/revision/revision_repo.go | 12 +- internal/router/static_router.go | 12 +- internal/service/content/question_service.go | 2 + .../file_record/file_record_service.go | 172 ++++++++++++++++++ internal/service/object_info/object_info.go | 3 +- internal/service/provider.go | 2 + internal/service/revision/revision.go | 1 + .../service/service_config/service_config.go | 5 +- internal/service/tag_common/tag_common.go | 9 +- internal/service/uploader/upload.go | 52 ++++-- pkg/writer/writer.go | 5 + 24 files changed, 504 insertions(+), 40 deletions(-) create mode 100644 internal/entity/file_record_entity.go create mode 100644 internal/migrations/v25.go create mode 100644 internal/repo/file_record/file_record_repo.go create mode 100644 internal/service/file_record/file_record_service.go diff --git a/cmd/wire_gen.go b/cmd/wire_gen.go index b44fadfb2..702f60df3 100644 --- a/cmd/wire_gen.go +++ b/cmd/wire_gen.go @@ -48,6 +48,7 @@ import ( "github.com/apache/answer/internal/repo/comment" "github.com/apache/answer/internal/repo/config" "github.com/apache/answer/internal/repo/export" + "github.com/apache/answer/internal/repo/file_record" "github.com/apache/answer/internal/repo/limit" "github.com/apache/answer/internal/repo/meta" notification2 "github.com/apache/answer/internal/repo/notification" @@ -84,6 +85,7 @@ import ( "github.com/apache/answer/internal/service/dashboard" "github.com/apache/answer/internal/service/event_queue" export2 "github.com/apache/answer/internal/service/export" + file_record2 "github.com/apache/answer/internal/service/file_record" "github.com/apache/answer/internal/service/follow" "github.com/apache/answer/internal/service/importer" meta2 "github.com/apache/answer/internal/service/meta" @@ -227,10 +229,10 @@ func initApplication(debug bool, serverConf *conf.Server, dbConf *data.Database, contentRevisionService := content.NewRevisionService(revisionRepo, userCommon, questionCommon, answerService, objService, questionRepo, answerRepo, tagRepo, tagCommonService, notificationQueueService, activityQueueService, reportRepo, reviewService, reviewActivityRepo) revisionController := controller.NewRevisionController(contentRevisionService, rankService) rankController := controller.NewRankController(rankService) - badgeAwardRepo := badge_award.NewBadgeAwardRepo(dataData, uniqueIDRepo) + userAdminRepo := user.NewUserAdminRepo(dataData, authRepo) notificationRepo := notification2.NewNotificationRepo(dataData) pluginUserConfigRepo := plugin_config.NewPluginUserConfigRepo(dataData) - userAdminRepo := user.NewUserAdminRepo(dataData, authRepo) + badgeAwardRepo := badge_award.NewBadgeAwardRepo(dataData, uniqueIDRepo) userAdminService := user_admin.NewUserAdminService(userAdminRepo, userRoleRelService, authService, userCommon, userActiveActivityRepo, siteInfoCommonService, emailService, questionRepo, answerRepo, commentCommonRepo, userExternalLoginRepo, notificationRepo, pluginUserConfigRepo, badgeAwardRepo) userAdminController := controller_admin.NewUserAdminController(userAdminService) reasonRepo := reason.NewReasonRepo(configService) @@ -246,7 +248,9 @@ func initApplication(debug bool, serverConf *conf.Server, dbConf *data.Database, notificationController := controller.NewNotificationController(notificationService, rankService) dashboardService := dashboard.NewDashboardService(questionRepo, answerRepo, commentCommonRepo, voteRepo, userRepo, reportRepo, configService, siteInfoCommonService, serviceConf, reviewService, revisionRepo, dataData) dashboardController := controller.NewDashboardController(dashboardService) - uploaderService := uploader.NewUploaderService(serviceConf, siteInfoCommonService) + fileRecordRepo := file_record.NewFileRecordRepo(dataData) + fileRecordService := file_record2.NewFileRecordService(fileRecordRepo, revisionRepo, serviceConf, siteInfoCommonService) + uploaderService := uploader.NewUploaderService(serviceConf, siteInfoCommonService, fileRecordService) uploadController := controller.NewUploadController(uploaderService) activityActivityRepo := activity.NewActivityRepo(dataData, configService) activityCommon := activity_common2.NewActivityCommon(activityRepo, activityQueueService) @@ -287,7 +291,7 @@ func initApplication(debug bool, serverConf *conf.Server, dbConf *data.Database, renderController := controller.NewRenderController() pluginAPIRouter := router.NewPluginAPIRouter(connectorController, userCenterController, captchaController, embedController, renderController) ginEngine := server.NewHTTPServer(debug, staticRouter, answerAPIRouter, swaggerRouter, uiRouter, authUserMiddleware, avatarMiddleware, shortIDMiddleware, templateRouter, pluginAPIRouter, uiConf) - scheduledTaskManager := cron.NewScheduledTaskManager(siteInfoCommonService, questionService) + scheduledTaskManager := cron.NewScheduledTaskManager(siteInfoCommonService, questionService, fileRecordService, serviceConf) application := newApplication(serverConf, ginEngine, scheduledTaskManager) return application, func() { cleanup2() diff --git a/configs/config.yaml b/configs/config.yaml index 526710109..69c4e3034 100644 --- a/configs/config.yaml +++ b/configs/config.yaml @@ -33,6 +33,9 @@ swaggerui: address: ':80' service_config: upload_path: "/data/uploads" + clean_up_uploads: true + clean_orphan_uploads_period_hours: 48 + purge_deleted_files_period_days: 30 ui: public_url: '/' api_url: '/' diff --git a/go.mod b/go.mod index 7e207e6b6..89c6876b5 100644 --- a/go.mod +++ b/go.mod @@ -60,6 +60,7 @@ require ( golang.org/x/crypto v0.27.0 golang.org/x/image v0.20.0 golang.org/x/net v0.29.0 + golang.org/x/text v0.18.0 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.33.0 @@ -160,7 +161,6 @@ require ( golang.org/x/arch v0.10.0 // indirect golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect golang.org/x/sys v0.25.0 // indirect - golang.org/x/text v0.18.0 // indirect golang.org/x/tools v0.25.0 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect diff --git a/internal/base/constant/upload.go b/internal/base/constant/upload.go index c01c92790..d9001fded 100644 --- a/internal/base/constant/upload.go +++ b/internal/base/constant/upload.go @@ -25,4 +25,5 @@ const ( PostSubPath = "post" BrandingSubPath = "branding" FilesPostSubPath = "files/post" + DeletedSubPath = "deleted" ) diff --git a/internal/base/cron/cron.go b/internal/base/cron/cron.go index 2e7a92ab9..98e59decf 100644 --- a/internal/base/cron/cron.go +++ b/internal/base/cron/cron.go @@ -24,6 +24,8 @@ import ( "fmt" "github.com/apache/answer/internal/service/content" + "github.com/apache/answer/internal/service/file_record" + "github.com/apache/answer/internal/service/service_config" "github.com/apache/answer/internal/service/siteinfo_common" "github.com/robfig/cron/v3" "github.com/segmentfault/pacman/log" @@ -31,29 +33,36 @@ import ( // ScheduledTaskManager scheduled task manager type ScheduledTaskManager struct { - siteInfoService siteinfo_common.SiteInfoCommonService - questionService *content.QuestionService + siteInfoService siteinfo_common.SiteInfoCommonService + questionService *content.QuestionService + fileRecordService *file_record.FileRecordService + serviceConfig *service_config.ServiceConfig } // NewScheduledTaskManager new scheduled task manager func NewScheduledTaskManager( siteInfoService siteinfo_common.SiteInfoCommonService, questionService *content.QuestionService, + fileRecordService *file_record.FileRecordService, + serviceConfig *service_config.ServiceConfig, ) *ScheduledTaskManager { manager := &ScheduledTaskManager{ - siteInfoService: siteInfoService, - questionService: questionService, + siteInfoService: siteInfoService, + questionService: questionService, + fileRecordService: fileRecordService, + serviceConfig: serviceConfig, } return manager } func (s *ScheduledTaskManager) Run() { - fmt.Println("start cron") + log.Infof("cron job manager start") + s.questionService.SitemapCron(context.Background()) c := cron.New() _, err := c.AddFunc("0 */1 * * *", func() { ctx := context.Background() - fmt.Println("sitemap cron execution") + log.Infof("sitemap cron execution") s.questionService.SitemapCron(ctx) }) if err != nil { @@ -62,12 +71,32 @@ func (s *ScheduledTaskManager) Run() { _, err = c.AddFunc("0 */1 * * *", func() { ctx := context.Background() - fmt.Println("refresh hottest cron execution") + log.Infof("refresh hottest cron execution") s.questionService.RefreshHottestCron(ctx) }) if err != nil { log.Error(err) } + if s.serviceConfig.CleanUpUploads { + log.Infof("clean up uploads cron enabled") + + conf := s.serviceConfig + _, err = c.AddFunc(fmt.Sprintf("0 */%d * * *", conf.CleanOrphanUploadsPeriodHours), func() { + log.Infof("clean orphan upload files cron execution") + s.fileRecordService.CleanOrphanUploadFiles(context.Background()) + }) + if err != nil { + log.Error(err) + } + + _, err = c.AddFunc(fmt.Sprintf("0 0 */%d * *", conf.PurgeDeletedFilesPeriodDays), func() { + log.Infof("purge deleted files cron execution") + s.fileRecordService.PurgeDeletedFiles(context.Background()) + }) + if err != nil { + log.Error(err) + } + } c.Start() } diff --git a/internal/controller/upload_controller.go b/internal/controller/upload_controller.go index 0bd74bf33..337cf83d4 100644 --- a/internal/controller/upload_controller.go +++ b/internal/controller/upload_controller.go @@ -70,19 +70,20 @@ func (uc *UploadController) UploadFile(ctx *gin.Context) { ) source := ctx.PostForm("source") + userID := middleware.GetLoginUserIDFromContext(ctx) switch source { case fileFromAvatar: - url, err = uc.uploaderService.UploadAvatarFile(ctx) + url, err = uc.uploaderService.UploadAvatarFile(ctx, userID) case fileFromPost: - url, err = uc.uploaderService.UploadPostFile(ctx) + url, err = uc.uploaderService.UploadPostFile(ctx, userID) case fileFromBranding: if !middleware.GetIsAdminFromContext(ctx) { handler.HandleResponse(ctx, errors.Forbidden(reason.ForbiddenError), nil) return } - url, err = uc.uploaderService.UploadBrandingFile(ctx) + url, err = uc.uploaderService.UploadBrandingFile(ctx, userID) case fileFromPostAttachment: - url, err = uc.uploaderService.UploadPostAttachment(ctx) + url, err = uc.uploaderService.UploadPostAttachment(ctx, userID) default: handler.HandleResponse(ctx, errors.BadRequest(reason.UploadFileSourceUnsupported), nil) return diff --git a/internal/entity/file_record_entity.go b/internal/entity/file_record_entity.go new file mode 100644 index 000000000..83a917aac --- /dev/null +++ b/internal/entity/file_record_entity.go @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package entity + +import "time" + +const ( + FileRecordStatusAvailable = 1 + FileRecordStatusDeleted = 10 +) + +// FileRecord file record +type FileRecord struct { + ID int `xorm:"not null pk autoincr INT(10) id"` + CreatedAt time.Time `xorm:"not null default CURRENT_TIMESTAMP created TIMESTAMP created_at"` + UpdatedAt time.Time `xorm:"not null default CURRENT_TIMESTAMP updated TIMESTAMP updated_at"` + UserID string `xorm:"not null default 0 BIGINT(20) user_id"` + FilePath string `xorm:"not null VARCHAR(256) file_path"` + FileURL string `xorm:"not null VARCHAR(1024) file_url"` + ObjectID string `xorm:"not null default 0 INDEX BIGINT(20) object_id"` + Source string `xorm:"not null VARCHAR(128) source"` + Status int `xorm:"not null default 0 TINYINT(4) status"` +} + +// TableName file record table name +func (FileRecord) TableName() string { + return "file_record" +} diff --git a/internal/migrations/migrations.go b/internal/migrations/migrations.go index bc2ecda70..57f4778e5 100644 --- a/internal/migrations/migrations.go +++ b/internal/migrations/migrations.go @@ -100,6 +100,7 @@ var migrations = []Migration{ NewMigration("v1.4.0", "add badge/badge_group/badge_award table", addBadges, true), NewMigration("v1.4.1", "add question link", addQuestionLink, true), NewMigration("v1.4.2", "add the number of question links", addQuestionLinkedCount, true), + NewMigration("v1.4.5", "add file record", addFileRecord, true), } func GetMigrations() []Migration { diff --git a/internal/migrations/v25.go b/internal/migrations/v25.go new file mode 100644 index 000000000..a0e1f1c20 --- /dev/null +++ b/internal/migrations/v25.go @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package migrations + +import ( + "context" + + "github.com/apache/answer/internal/entity" + "xorm.io/xorm" +) + +func addFileRecord(ctx context.Context, x *xorm.Engine) error { + return x.Context(ctx).Sync(new(entity.FileRecord)) +} diff --git a/internal/repo/answer/answer_repo.go b/internal/repo/answer/answer_repo.go index 2c2787f65..cdfc58029 100644 --- a/internal/repo/answer/answer_repo.go +++ b/internal/repo/answer/answer_repo.go @@ -529,7 +529,24 @@ func (ar *answerRepo) updateSearch(ctx context.Context, answerID string) (err er } func (ar *answerRepo) DeletePermanentlyAnswers(ctx context.Context) error { - _, err := ar.data.DB.Context(ctx).Where("status = ?", entity.AnswerStatusDeleted).Delete(&entity.Answer{}) + // get all deleted answers ids + ids := make([]string, 0) + err := ar.data.DB.Context(ctx).Select("id").Table(new(entity.Answer).TableName()). + Where("status = ?", entity.AnswerStatusDeleted).Find(&ids) + if err != nil { + return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack() + } + if len(ids) == 0 { + return nil + } + + // delete all revisions permanently + _, err = ar.data.DB.Context(ctx).In("object_id", ids).Delete(&entity.Revision{}) + if err != nil { + return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack() + } + + _, err = ar.data.DB.Context(ctx).Where("status = ?", entity.AnswerStatusDeleted).Delete(&entity.Answer{}) if err != nil { return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack() } diff --git a/internal/repo/file_record/file_record_repo.go b/internal/repo/file_record/file_record_repo.go new file mode 100644 index 000000000..ed081be40 --- /dev/null +++ b/internal/repo/file_record/file_record_repo.go @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package file_record + +import ( + "context" + + "github.com/apache/answer/internal/base/pager" + "github.com/apache/answer/internal/service/file_record" + + "github.com/apache/answer/internal/base/data" + "github.com/apache/answer/internal/base/reason" + "github.com/apache/answer/internal/entity" + "github.com/segmentfault/pacman/errors" +) + +// fileRecordRepo fileRecord repository +type fileRecordRepo struct { + data *data.Data +} + +// NewFileRecordRepo new repository +func NewFileRecordRepo(data *data.Data) file_record.FileRecordRepo { + return &fileRecordRepo{ + data: data, + } +} + +// AddFileRecord add file record +func (fr *fileRecordRepo) AddFileRecord(ctx context.Context, fileRecord *entity.FileRecord) (err error) { + _, err = fr.data.DB.Context(ctx).Insert(fileRecord) + if err != nil { + err = errors.InternalServer(reason.DatabaseError).WithError(err).WithStack() + } + return +} + +// GetFileRecordPage get fileRecord page +func (fr *fileRecordRepo) GetFileRecordPage(ctx context.Context, page, pageSize int, cond *entity.FileRecord) ( + fileRecordList []*entity.FileRecord, total int64, err error) { + fileRecordList = make([]*entity.FileRecord, 0) + + session := fr.data.DB.Context(ctx) + total, err = pager.Help(page, pageSize, &fileRecordList, cond, session) + if err != nil { + err = errors.InternalServer(reason.DatabaseError).WithError(err).WithStack() + } + return +} + +// DeleteFileRecord delete file record +func (fr *fileRecordRepo) DeleteFileRecord(ctx context.Context, id int) (err error) { + _, err = fr.data.DB.Context(ctx).ID(id).Cols("status").Update(&entity.FileRecord{Status: entity.FileRecordStatusDeleted}) + if err != nil { + err = errors.InternalServer(reason.DatabaseError).WithError(err).WithStack() + } + return +} + +// UpdateFileRecord update file record +func (fr *fileRecordRepo) UpdateFileRecord(ctx context.Context, fileRecord *entity.FileRecord) (err error) { + _, err = fr.data.DB.Context(ctx).ID(fileRecord.ID).Update(fileRecord) + if err != nil { + err = errors.InternalServer(reason.DatabaseError).WithError(err).WithStack() + } + return +} diff --git a/internal/repo/provider.go b/internal/repo/provider.go index 1b29cc5e8..02f27f62f 100644 --- a/internal/repo/provider.go +++ b/internal/repo/provider.go @@ -33,6 +33,7 @@ import ( "github.com/apache/answer/internal/repo/comment" "github.com/apache/answer/internal/repo/config" "github.com/apache/answer/internal/repo/export" + "github.com/apache/answer/internal/repo/file_record" "github.com/apache/answer/internal/repo/limit" "github.com/apache/answer/internal/repo/meta" "github.com/apache/answer/internal/repo/notification" @@ -107,4 +108,5 @@ var ProviderSetRepo = wire.NewSet( badge.NewEventRuleRepo, badge_group.NewBadgeGroupRepo, badge_award.NewBadgeAwardRepo, + file_record.NewFileRecordRepo, ) diff --git a/internal/repo/question/question_repo.go b/internal/repo/question/question_repo.go index 117ee38f0..978c9ce32 100644 --- a/internal/repo/question/question_repo.go +++ b/internal/repo/question/question_repo.go @@ -168,6 +168,23 @@ func (qr *questionRepo) UpdateQuestionStatusWithOutUpdateTime(ctx context.Contex } func (qr *questionRepo) DeletePermanentlyQuestions(ctx context.Context) (err error) { + // get all deleted question ids + ids := make([]string, 0) + err = qr.data.DB.Context(ctx).Select("id").Table(new(entity.Question).TableName()). + Where("status = ?", entity.QuestionStatusDeleted).Find(&ids) + if err != nil { + return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack() + } + if len(ids) == 0 { + return nil + } + + // delete all revisions permanently + _, err = qr.data.DB.Context(ctx).In("object_id", ids).Delete(&entity.Revision{}) + if err != nil { + return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack() + } + _, err = qr.data.DB.Context(ctx).Where("status = ?", entity.QuestionStatusDeleted).Delete(&entity.Question{}) if err != nil { return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack() diff --git a/internal/repo/revision/revision_repo.go b/internal/repo/revision/revision_repo.go index ab00adf99..09ba1aacd 100644 --- a/internal/repo/revision/revision_repo.go +++ b/internal/repo/revision/revision_repo.go @@ -155,7 +155,17 @@ func (rr *revisionRepo) GetLastRevisionByObjectID(ctx context.Context, objectID revision *entity.Revision, exist bool, err error, ) { revision = &entity.Revision{} - exist, err = rr.data.DB.Context(ctx).Where("object_id = ?", objectID).OrderBy("created_at DESC").Get(revision) + exist, err = rr.data.DB.Context(ctx).Where("object_id = ?", objectID).Desc("created_at").Get(revision) + if err != nil { + return nil, false, errors.InternalServer(reason.DatabaseError).WithError(err).WithStack() + } + return +} + +// GetLastRevisionByFileURL get object's last revision by file url +func (rr *revisionRepo) GetLastRevisionByFileURL(ctx context.Context, fileURL string) (revision *entity.Revision, exist bool, err error) { + revision = &entity.Revision{} + exist, err = rr.data.DB.Context(ctx).Where("content LIKE ?", "%"+fileURL+"%").Desc("created_at").Get(revision) if err != nil { return nil, false, errors.InternalServer(reason.DatabaseError).WithError(err).WithStack() } diff --git a/internal/router/static_router.go b/internal/router/static_router.go index 807f4c735..a6c80fc04 100644 --- a/internal/router/static_router.go +++ b/internal/router/static_router.go @@ -20,11 +20,14 @@ package router import ( + "net/http" + "path/filepath" + "strings" + "github.com/apache/answer/internal/base/constant" "github.com/apache/answer/internal/service/service_config" + "github.com/apache/answer/pkg/dir" "github.com/gin-gonic/gin" - "path/filepath" - "strings" ) // StaticRouter static api router @@ -54,6 +57,11 @@ func (a *StaticRouter) RegisterStaticRouter(r *gin.RouterGroup) { realFilename := strings.TrimSuffix(filePath, "/"+originalFilename) + filepath.Ext(originalFilename) // The file local path is /uploads/files/post/hash.pdf fileLocalPath := filepath.Join(a.serviceConfig.UploadPath, constant.FilesPostSubPath, realFilename) + // If the file is not exist, return 404 + if !dir.CheckFileExist(fileLocalPath) { + c.Redirect(http.StatusFound, "/404") + return + } c.FileAttachment(fileLocalPath, originalFilename) }) } diff --git a/internal/service/content/question_service.go b/internal/service/content/question_service.go index 109b6b101..f3641dccd 100644 --- a/internal/service/content/question_service.go +++ b/internal/service/content/question_service.go @@ -1398,6 +1398,8 @@ func (qs *QuestionService) GetQuestionPage(ctx context.Context, req *schema.Ques return nil, 0, err } tagIDs = append(synTagIds, tagInfo.ID) + } else { + return questions, 0, nil } } diff --git a/internal/service/file_record/file_record_service.go b/internal/service/file_record/file_record_service.go new file mode 100644 index 000000000..abb983768 --- /dev/null +++ b/internal/service/file_record/file_record_service.go @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package file_record + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/apache/answer/internal/base/constant" + "github.com/apache/answer/internal/entity" + "github.com/apache/answer/internal/service/revision" + "github.com/apache/answer/internal/service/service_config" + "github.com/apache/answer/internal/service/siteinfo_common" + "github.com/apache/answer/pkg/checker" + "github.com/apache/answer/pkg/dir" + "github.com/apache/answer/pkg/writer" + "github.com/segmentfault/pacman/log" +) + +// FileRecordRepo file record repository +type FileRecordRepo interface { + AddFileRecord(ctx context.Context, fileRecord *entity.FileRecord) (err error) + UpdateFileRecord(ctx context.Context, fileRecord *entity.FileRecord) (err error) + GetFileRecordPage(ctx context.Context, page, pageSize int, cond *entity.FileRecord) ( + fileRecordList []*entity.FileRecord, total int64, err error) + DeleteFileRecord(ctx context.Context, id int) (err error) +} + +// FileRecordService file record service +type FileRecordService struct { + fileRecordRepo FileRecordRepo + revisionRepo revision.RevisionRepo + serviceConfig *service_config.ServiceConfig + siteInfoService siteinfo_common.SiteInfoCommonService +} + +// NewFileRecordService new file record service +func NewFileRecordService( + fileRecordRepo FileRecordRepo, + revisionRepo revision.RevisionRepo, + serviceConfig *service_config.ServiceConfig, + siteInfoService siteinfo_common.SiteInfoCommonService, +) *FileRecordService { + return &FileRecordService{ + fileRecordRepo: fileRecordRepo, + revisionRepo: revisionRepo, + serviceConfig: serviceConfig, + siteInfoService: siteInfoService, + } +} + +// AddFileRecord add file record +func (fs *FileRecordService) AddFileRecord(ctx context.Context, userID, filePath, fileURL, source string) { + record := &entity.FileRecord{ + UserID: userID, + FilePath: filePath, + FileURL: fileURL, + Source: source, + Status: entity.FileRecordStatusAvailable, + ObjectID: "0", + } + if err := fs.fileRecordRepo.AddFileRecord(ctx, record); err != nil { + log.Errorf("add file record error: %v", err) + } +} + +// CleanOrphanUploadFiles clean orphan upload files +func (fs *FileRecordService) CleanOrphanUploadFiles(ctx context.Context) { + page, pageSize := 1, 1000 + + for { + fileRecordList, total, err := fs.fileRecordRepo.GetFileRecordPage(ctx, page, pageSize, &entity.FileRecord{ + Status: entity.FileRecordStatusAvailable, + }) + if err != nil { + log.Errorf("get file record page error: %v", err) + return + } + if len(fileRecordList) == 0 || total == 0 { + break + } + for _, fileRecord := range fileRecordList { + // If this file record created in 48 hours, no need to check + if fileRecord.CreatedAt.AddDate(0, 0, 2).After(time.Now()) { + continue + } + if checker.IsNotZeroString(fileRecord.ObjectID) { + _, exist, err := fs.revisionRepo.GetLastRevisionByObjectID(ctx, fileRecord.ObjectID) + if err != nil { + log.Errorf("get last revision by object id error: %v", err) + continue + } + if exist { + continue + } + } else { + lastRevision, exist, err := fs.revisionRepo.GetLastRevisionByFileURL(ctx, fileRecord.FileURL) + if err != nil { + log.Errorf("get last revision by file url error: %v", err) + continue + } + if exist { + // update the file record object id + fileRecord.ObjectID = lastRevision.ObjectID + if err := fs.fileRecordRepo.UpdateFileRecord(ctx, fileRecord); err != nil { + log.Errorf("update file record object id error: %v", err) + } + continue + } + } + // Delete and move the file record + if err := fs.deleteAndMoveFileRecord(ctx, fileRecord); err != nil { + log.Error(err) + } + } + page++ + } +} + +func (fs *FileRecordService) PurgeDeletedFiles(ctx context.Context) { + deletedPath := filepath.Join(fs.serviceConfig.UploadPath, constant.DeletedSubPath) + log.Infof("purge deleted files: %s", deletedPath) + err := os.RemoveAll(deletedPath) + if err != nil { + log.Errorf("purge deleted files error: %v", err) + return + } + err = dir.CreateDirIfNotExist(deletedPath) + if err != nil { + log.Errorf("create deleted directory error: %v", err) + } + return +} + +func (fs *FileRecordService) deleteAndMoveFileRecord(ctx context.Context, fileRecord *entity.FileRecord) error { + // Delete the file record + if err := fs.fileRecordRepo.DeleteFileRecord(ctx, fileRecord.ID); err != nil { + return fmt.Errorf("delete file record error: %v", err) + } + + // Move the file to the deleted directory + oldFilename := filepath.Base(fileRecord.FilePath) + oldFilePath := filepath.Join(fs.serviceConfig.UploadPath, fileRecord.FilePath) + deletedPath := filepath.Join(fs.serviceConfig.UploadPath, constant.DeletedSubPath, oldFilename) + + if err := writer.MoveFile(oldFilePath, deletedPath); err != nil { + return fmt.Errorf("move file error: %v", err) + } + + log.Debugf("delete and move file: %s", fileRecord.FileURL) + return nil +} diff --git a/internal/service/object_info/object_info.go b/internal/service/object_info/object_info.go index 76b1d86d8..5ef438ff3 100644 --- a/internal/service/object_info/object_info.go +++ b/internal/service/object_info/object_info.go @@ -21,6 +21,7 @@ package object_info import ( "context" + "github.com/apache/answer/internal/base/constant" "github.com/apache/answer/internal/base/reason" "github.com/apache/answer/internal/schema" @@ -279,7 +280,7 @@ func (os *ObjService) GetInfo(ctx context.Context, objectID string) (objInfo *sc ObjectID: tagInfo.ID, TagID: tagInfo.ID, ObjectType: objectType, - Title: tagInfo.ParsedText, + Title: tagInfo.SlugName, Content: tagInfo.ParsedText, // todo trim } } diff --git a/internal/service/provider.go b/internal/service/provider.go index 840732359..4b1b64276 100644 --- a/internal/service/provider.go +++ b/internal/service/provider.go @@ -36,6 +36,7 @@ import ( "github.com/apache/answer/internal/service/dashboard" "github.com/apache/answer/internal/service/event_queue" "github.com/apache/answer/internal/service/export" + "github.com/apache/answer/internal/service/file_record" "github.com/apache/answer/internal/service/follow" "github.com/apache/answer/internal/service/importer" "github.com/apache/answer/internal/service/meta" @@ -126,4 +127,5 @@ var ProviderSetService = wire.NewSet( badge.NewBadgeAwardService, badge.NewBadgeGroupService, importer.NewImporterService, + file_record.NewFileRecordService, ) diff --git a/internal/service/revision/revision.go b/internal/service/revision/revision.go index 584272200..33ce285e0 100644 --- a/internal/service/revision/revision.go +++ b/internal/service/revision/revision.go @@ -31,6 +31,7 @@ type RevisionRepo interface { AddRevision(ctx context.Context, revision *entity.Revision, autoUpdateRevisionID bool) (err error) GetRevisionByID(ctx context.Context, revisionID string) (revision *entity.Revision, exist bool, err error) GetLastRevisionByObjectID(ctx context.Context, objectID string) (revision *entity.Revision, exist bool, err error) + GetLastRevisionByFileURL(ctx context.Context, fileURL string) (revision *entity.Revision, exist bool, err error) GetRevisionList(ctx context.Context, revision *entity.Revision) (revisionList []entity.Revision, err error) UpdateObjectRevisionId(ctx context.Context, revision *entity.Revision, session *xorm.Session) (err error) ExistUnreviewedByObjectID(ctx context.Context, objectID string) (revision *entity.Revision, exist bool, err error) diff --git a/internal/service/service_config/service_config.go b/internal/service/service_config/service_config.go index c5f27669a..90e399e17 100644 --- a/internal/service/service_config/service_config.go +++ b/internal/service/service_config/service_config.go @@ -20,5 +20,8 @@ package service_config type ServiceConfig struct { - UploadPath string `json:"upload_path" mapstructure:"upload_path" yaml:"upload_path"` + UploadPath string `json:"upload_path" mapstructure:"upload_path" yaml:"upload_path"` + CleanUpUploads bool `json:"clean_up_uploads" mapstructure:"clean_up_uploads" yaml:"clean_up_uploads"` + CleanOrphanUploadsPeriodHours int `json:"clean_orphan_uploads_period_hours" mapstructure:"clean_orphan_uploads_period_hours" yaml:"clean_orphan_uploads_period_hours"` + PurgeDeletedFilesPeriodDays int `json:"purge_deleted_files_period_days" mapstructure:"purge_deleted_files_period_days" yaml:"purge_deleted_files_period_days"` } diff --git a/internal/service/tag_common/tag_common.go b/internal/service/tag_common/tag_common.go index e5c136c34..4ce899752 100644 --- a/internal/service/tag_common/tag_common.go +++ b/internal/service/tag_common/tag_common.go @@ -349,10 +349,17 @@ func (ts *TagCommonService) AddTag(ctx context.Context, req *schema.AddTagReq) ( } tagInfoJson, _ := json.Marshal(tagInfo) revisionDTO.Content = string(tagInfoJson) - _, err = ts.revisionService.AddRevision(ctx, revisionDTO, true) + revisionID, err := ts.revisionService.AddRevision(ctx, revisionDTO, true) if err != nil { return nil, err } + ts.activityQueueService.Send(ctx, &schema.ActivityMsg{ + UserID: req.UserID, + ObjectID: tagInfo.ID, + OriginalObjectID: tagInfo.ID, + ActivityTypeKey: constant.ActTagCreated, + RevisionID: revisionID, + }) return &schema.AddTagResp{SlugName: tagInfo.SlugName}, nil } diff --git a/internal/service/uploader/upload.go b/internal/service/uploader/upload.go index 4b018cb4c..d5cc2dfbe 100644 --- a/internal/service/uploader/upload.go +++ b/internal/service/uploader/upload.go @@ -31,6 +31,8 @@ import ( "path/filepath" "strings" + "github.com/apache/answer/internal/service/file_record" + "github.com/apache/answer/internal/base/constant" "github.com/apache/answer/internal/base/reason" "github.com/apache/answer/internal/service/service_config" @@ -53,6 +55,7 @@ var ( constant.PostSubPath, constant.BrandingSubPath, constant.FilesPostSubPath, + constant.DeletedSubPath, } supportedThumbFileExtMapping = map[string]imaging.Format{ ".jpg": imaging.JPEG, @@ -63,22 +66,26 @@ var ( ) type UploaderService interface { - UploadAvatarFile(ctx *gin.Context) (url string, err error) - UploadPostFile(ctx *gin.Context) (url string, err error) - UploadPostAttachment(ctx *gin.Context) (url string, err error) - UploadBrandingFile(ctx *gin.Context) (url string, err error) + UploadAvatarFile(ctx *gin.Context, userID string) (url string, err error) + UploadPostFile(ctx *gin.Context, userID string) (url string, err error) + UploadPostAttachment(ctx *gin.Context, userID string) (url string, err error) + UploadBrandingFile(ctx *gin.Context, userID string) (url string, err error) AvatarThumbFile(ctx *gin.Context, fileName string, size int) (url string, err error) } // uploaderService uploader service type uploaderService struct { - serviceConfig *service_config.ServiceConfig - siteInfoService siteinfo_common.SiteInfoCommonService + serviceConfig *service_config.ServiceConfig + siteInfoService siteinfo_common.SiteInfoCommonService + fileRecordService *file_record.FileRecordService } // NewUploaderService new upload service -func NewUploaderService(serviceConfig *service_config.ServiceConfig, - siteInfoService siteinfo_common.SiteInfoCommonService) UploaderService { +func NewUploaderService( + serviceConfig *service_config.ServiceConfig, + siteInfoService siteinfo_common.SiteInfoCommonService, + fileRecordService *file_record.FileRecordService, +) UploaderService { for _, subPath := range subPathList { err := dir.CreateDirIfNotExist(filepath.Join(serviceConfig.UploadPath, subPath)) if err != nil { @@ -86,13 +93,14 @@ func NewUploaderService(serviceConfig *service_config.ServiceConfig, } } return &uploaderService{ - serviceConfig: serviceConfig, - siteInfoService: siteInfoService, + serviceConfig: serviceConfig, + siteInfoService: siteInfoService, + fileRecordService: fileRecordService, } } // UploadAvatarFile upload avatar file -func (us *uploaderService) UploadAvatarFile(ctx *gin.Context) (url string, err error) { +func (us *uploaderService) UploadAvatarFile(ctx *gin.Context, userID string) (url string, err error) { url, err = us.tryToUploadByPlugin(ctx, plugin.UserAvatar) if err != nil { return "", err @@ -174,7 +182,7 @@ func (us *uploaderService) AvatarThumbFile(ctx *gin.Context, fileName string, si return saveFilePath, nil } -func (us *uploaderService) UploadPostFile(ctx *gin.Context) ( +func (us *uploaderService) UploadPostFile(ctx *gin.Context, userID string) ( url string, err error) { url, err = us.tryToUploadByPlugin(ctx, plugin.UserPost) if err != nil { @@ -202,10 +210,15 @@ func (us *uploaderService) UploadPostFile(ctx *gin.Context) ( fileExt := strings.ToLower(path.Ext(fileHeader.Filename)) newFilename := fmt.Sprintf("%s%s", uid.IDStr12(), fileExt) avatarFilePath := path.Join(constant.PostSubPath, newFilename) - return us.uploadImageFile(ctx, fileHeader, avatarFilePath) + url, err = us.uploadImageFile(ctx, fileHeader, avatarFilePath) + if err != nil { + return "", err + } + us.fileRecordService.AddFileRecord(ctx, userID, avatarFilePath, url, string(plugin.UserPost)) + return url, nil } -func (us *uploaderService) UploadPostAttachment(ctx *gin.Context) ( +func (us *uploaderService) UploadPostAttachment(ctx *gin.Context, userID string) ( url string, err error) { url, err = us.tryToUploadByPlugin(ctx, plugin.UserPostAttachment) if err != nil { @@ -232,11 +245,16 @@ func (us *uploaderService) UploadPostAttachment(ctx *gin.Context) ( fileExt := strings.ToLower(path.Ext(fileHeader.Filename)) newFilename := fmt.Sprintf("%s%s", uid.IDStr12(), fileExt) - avatarFilePath := path.Join(constant.FilesPostSubPath, newFilename) - return us.uploadAttachmentFile(ctx, fileHeader, fileHeader.Filename, avatarFilePath) + attachmentFilePath := path.Join(constant.FilesPostSubPath, newFilename) + url, err = us.uploadAttachmentFile(ctx, fileHeader, fileHeader.Filename, attachmentFilePath) + if err != nil { + return "", err + } + us.fileRecordService.AddFileRecord(ctx, userID, attachmentFilePath, url, string(plugin.UserPostAttachment)) + return url, nil } -func (us *uploaderService) UploadBrandingFile(ctx *gin.Context) ( +func (us *uploaderService) UploadBrandingFile(ctx *gin.Context, userID string) ( url string, err error) { url, err = us.tryToUploadByPlugin(ctx, plugin.AdminBranding) if err != nil { diff --git a/pkg/writer/writer.go b/pkg/writer/writer.go index 49ded3dfb..ff08d2592 100644 --- a/pkg/writer/writer.go +++ b/pkg/writer/writer.go @@ -48,3 +48,8 @@ func WriteFile(filePath, content string) error { } return nil } + +// MoveFile move file to new path +func MoveFile(oldPath, newPath string) error { + return os.Rename(oldPath, newPath) +}