From 0c287cc917d032687a487f4b2764b941a602c89005f3ad042317df1215df8e70 Mon Sep 17 00:00:00 2001 From: DaanSelen Date: Wed, 22 Apr 2026 11:51:57 +0200 Subject: [PATCH] feat: add working watchdog cycle --- cmd/server/main.go | 6 +- internal/api/api.go | 4 +- internal/api/routes_file.go | 63 +++++++++++------ internal/crypto/crypto.go | 15 ++-- internal/database/database.go | 29 ++++---- internal/database/define.go | 7 +- internal/database/functions.go | 4 ++ internal/database/watchdog.go | 69 ++++++++++++++----- .../runtime.go => utility/environment.go} | 19 +---- internal/utility/filesystem.go | 32 +++++++++ 10 files changed, 166 insertions(+), 82 deletions(-) rename internal/{runtime/runtime.go => utility/environment.go} (79%) create mode 100644 internal/utility/filesystem.go diff --git a/cmd/server/main.go b/cmd/server/main.go index 8a8602c..284f194 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -3,7 +3,7 @@ package main import ( "eden-server/internal/api" "eden-server/internal/database" - "eden-server/internal/runtime" + "eden-server/internal/utility" "log/slog" "os" ) @@ -15,11 +15,11 @@ func main() { // grab the environment variables from the runtime environment slog.Info("grabbing environment variables") - env := runtime.GrabEnvironment() + env := utility.GrabEnvironment() // checking directories to ensure its expected environment is ready slog.Info("auditing operating environment") - if err := runtime.EnsureOperation(env.DataDirectory); err != nil { + if err := utility.EnsureOperation(env.DataDirectory); err != nil { slog.Error("failed to ensure the operating environment", "error", err) os.Exit(1) } diff --git a/internal/api/api.go b/internal/api/api.go index f51d7c8..cf0da38 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -1,7 +1,7 @@ package api import ( - "eden-server/internal/runtime" + "eden-server/internal/utility" "fmt" "log/slog" @@ -22,7 +22,7 @@ type RespObj struct { // All error messages from slog must have an error field with the golang error // See bottom the the kickoff function for details -func KickoffApi(logger *slog.Logger, env runtime.Environment, db *gorm.DB) { +func KickoffApi(logger *slog.Logger, env utility.Environment, db *gorm.DB) { gin.SetMode(gin.ReleaseMode) // For a nice looking logger: diff --git a/internal/api/routes_file.go b/internal/api/routes_file.go index 2cfa3f6..f1daa0d 100644 --- a/internal/api/routes_file.go +++ b/internal/api/routes_file.go @@ -3,7 +3,8 @@ package api import ( "eden-server/internal/crypto" "eden-server/internal/database" - "eden-server/internal/runtime" + "eden-server/internal/utility" + "errors" "log/slog" "net/http" "path/filepath" @@ -13,7 +14,7 @@ import ( "gorm.io/gorm" ) -func spawnFileRoutes(file *gin.RouterGroup, env runtime.Environment, db *gorm.DB) { +func spawnFileRoutes(file *gin.RouterGroup, env utility.Environment, db *gorm.DB) { // /file/ file.GET("/:filename", func(c *gin.Context) { f := c.Param("filename") @@ -27,26 +28,63 @@ func spawnFileRoutes(file *gin.RouterGroup, env runtime.Environment, db *gorm.DB file.POST("/upload", func(c *gin.Context) { f, err := c.FormFile("file") if err != nil { - slog.Error("failed to get file details from request", "error", err) + slog.Debug("no file or file headers provided on the request", "error", err) c.JSON(http.StatusBadRequest, RespObj{ Msg: "a file is required", }) return } + checksum, err := crypto.CalculateHashFromRequest(f) + if err != nil { + slog.Error("failed to calculate hash of file at given path", "error", err) + c.JSON(http.StatusInternalServerError, RespObj{ + Msg: ieMes, + }) + return + } + e := filepath.Ext(f.Filename) m := categorizeFilemode(e) if m == database.Unspecified { - slog.Warn("discarding file since its filetype is unsupported") + slog.Debug("discarding file since its filetype is unsupported") c.JSON(http.StatusUnsupportedMediaType, RespObj{ Msg: "unsupported filetype", }) return } - safeName := uuid.New().String()[:8] + "_" + string(m) + e + safeName := uuid.New().String() + "_" + string(m) + e destPath := filepath.Join(env.DataDirectory, "content", safeName) + fData := database.File{ + MediaType: m, + MetaName: f.Filename, + FileName: safeName, + FilePath: destPath, + Checksum: checksum, + } + + // first register the file to the database + // error out if something bad happens (duplicate, failing db, etc) + err = database.RegisterFile(db, fData) + if err != nil { + if errors.Is(err, gorm.ErrDuplicatedKey) { + slog.Debug("discarding file since its a duplicate", "error", err) + c.JSON(http.StatusConflict, RespObj{ + Msg: "file is a duplicate", + }) + return + } + + slog.Error("discarding file since an error occured", "error", err) + c.JSON(http.StatusInternalServerError, RespObj{ + Msg: ieMes, + }) + return + } + + // save to filesystem after everything has given a green light if err := c.SaveUploadedFile(f, destPath); err != nil { slog.Error("failed to receive the file over http:", "error", err) c.JSON(http.StatusInternalServerError, RespObj{ @@ -55,21 +93,6 @@ func spawnFileRoutes(file *gin.RouterGroup, env runtime.Environment, db *gorm.DB return } - cSum, err := crypto.CalculateHash(destPath) - if err != nil { - slog.Error("failed to calculate hash of file at given path", "error", err) - c.JSON(http.StatusInternalServerError, RespObj{ - Msg: ieMes, - }) - } - - fData := database.File{ - MediaType: m, - GivenName: f.Filename, - Filepath: destPath, - Checksum: cSum, - } - database.RegisterFile(db, fData) c.JSON(http.StatusCreated, RespObj{ Msg: "file has succesfully been uploaded", }) diff --git a/internal/crypto/crypto.go b/internal/crypto/crypto.go index 7734355..4ff021e 100644 --- a/internal/crypto/crypto.go +++ b/internal/crypto/crypto.go @@ -4,24 +4,23 @@ import ( "crypto/sha512" "encoding/hex" "io" - "os" + "mime/multipart" ) -func CalculateHash(p string) (string, error) { - f, err := os.Open(p) +func CalculateHashFromRequest(fileHeader *multipart.FileHeader) (string, error) { + src, err := fileHeader.Open() if err != nil { return "", err } - defer f.Close() + defer src.Close() h := sha512.New() - if _, err := io.Copy(h, f); err != nil { + if _, err := io.Copy(h, src); err != nil { return "", err } - sum := h.Sum(nil) // return the sha checksum in hex - return hex.EncodeToString(sum), nil + return hex.EncodeToString(h.Sum(nil)), nil // alternatively return in base64 - //return base64.StdEncoding.EncodeToString(sum), nil + //return base64.StdEncoding.EncodeToString(h.Sum(nil)), nil } diff --git a/internal/database/database.go b/internal/database/database.go index e28d056..2340e61 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -1,30 +1,35 @@ package database import ( - "eden-server/internal/runtime" + "eden-server/internal/utility" + "log/slog" "path/filepath" "time" "gorm.io/driver/sqlite" "gorm.io/gorm" + "gorm.io/gorm/logger" ) //var watchdogStop = make(chan struct{}) func KickoffDatabase(workDir string) (*gorm.DB, error) { dbLoc := filepath.Join(workDir, "garden.db") - db, err := gorm.Open(sqlite.Open(dbLoc), &gorm.Config{}) + db, err := gorm.Open(sqlite.Open(dbLoc), &gorm.Config{ + Logger: logger.Discard, // disable gorm logging since its not slog (yet) + TranslateError: true, + }) if err != nil { return nil, err } - if err := db.AutoMigrate(&State{}); err != nil { - return nil, err - } - if err := db.AutoMigrate(&Device{}); err != nil { - return nil, err - } - if err := db.AutoMigrate(&File{}); err != nil { + // try to use GORM automigrate if the schema changes + slog.Info("performing migration") + if err := db.AutoMigrate( + &State{}, + &Device{}, + &File{}, + ); err != nil { return nil, err } @@ -39,7 +44,7 @@ func KickoffDatabase(workDir string) (*gorm.DB, error) { return db, nil } -func KickoffDatabaseWatchdog(env runtime.Environment, db *gorm.DB) { +func KickoffDatabaseWatchdog(env utility.Environment, db *gorm.DB) { timeInterval := time.Second * time.Duration(env.WatchInterval) ticker := time.NewTicker(timeInterval) @@ -58,11 +63,11 @@ func KickoffDatabaseWatchdog(env runtime.Environment, db *gorm.DB) { */ // run the watchdog function once to see if all is well. - watchdog(env.DataDirectory, db) + watchdog(env, db) // then defer to a decoupled/disowned golang goroutine for range ticker.C { - watchdog(env.DataDirectory, db) + watchdog(env, db) } }() } diff --git a/internal/database/define.go b/internal/database/define.go index 91bf5fa..0707bf5 100644 --- a/internal/database/define.go +++ b/internal/database/define.go @@ -41,9 +41,10 @@ type Device struct { type File struct { ID int `gorm:"primaryKey"` MediaType MediaType `gorm:"type:varchar(20);not null;"` - GivenName string - Filepath string - Checksum string // base64 encoded sha512 checksum + MetaName string + FileName string + FilePath string + Checksum string `gorm:"uniqueIndex"` // hex encoded sha512 checksum CreatedAt time.Time UpdatedAt time.Time } diff --git a/internal/database/functions.go b/internal/database/functions.go index 6a11f33..2f6b1a9 100644 --- a/internal/database/functions.go +++ b/internal/database/functions.go @@ -19,3 +19,7 @@ func GetFiles(db *gorm.DB) ([]File, error) { func RegisterFile(db *gorm.DB, f File) error { return db.Create(&f).Error } + +func DeregisterFile(db *gorm.DB, f File) error { + return db.Delete(&f).Error +} diff --git a/internal/database/watchdog.go b/internal/database/watchdog.go index fbd0562..13facdb 100644 --- a/internal/database/watchdog.go +++ b/internal/database/watchdog.go @@ -1,36 +1,73 @@ package database import ( + "eden-server/internal/utility" "log/slog" "os" + "path/filepath" "gorm.io/gorm" ) -func watchdog(w string, db *gorm.DB) { +func watchdog(env utility.Environment, db *gorm.DB) { slog.Info("performing the watchdog cycle") - files, err := GetFiles(db) + fsFiles, err := os.ReadDir(env.ContentDirectory) + if err != nil { + slog.Error("failed to read the content directory on the filesystem", "error", err) + return + } + + dbFiles, err := GetFiles(db) if err != nil { slog.Error("failed to retrieve the files indexed from the database", "error", err) return } - var purgeList []string - for _, f := range files { - i, err := os.Stat(f.Filepath) - if err != nil { - if os.IsNotExist(err) { - purgeList = append(purgeList, f.Filepath) - continue - } - slog.Warn("stat failed", "file", f.Filepath, "error", err) - continue - } + // generate a set of filesystem contents + fsSet := make(map[string]struct{}) // cool name for the files that are (now) marked for annihilation + for _, f := range fsFiles { + // absolute path creation + fullPath := filepath.Join(env.ContentDirectory, f.Name()) + fsSet[fullPath] = struct{}{} + } - if i.IsDir() { - purgeList = append(purgeList, f.Filepath) // also mark it for purger if its a directory. We do not want that here + // generate a set of Database contents + dbSet := make(map[string]File) // cool name for the files that are going to be deregistered from the database + for _, f := range dbFiles { + dbSet[f.FilePath] = f + } + + // FS -> DB + // check for orphaned filesystem files + var fsPurgeScroll []string + for path := range fsSet { + if _, exists := dbSet[path]; !exists { + fsPurgeScroll = append(fsPurgeScroll, path) + } + } + + // DB -> FS + // check stale database records + var dbPurgeScroll []File + for path, f := range dbSet { + if _, exists := fsSet[path]; !exists { + dbPurgeScroll = append(dbPurgeScroll, f) + } + } + + if len(fsPurgeScroll) > 0 { + slog.Info("filesystem purge scroll is populated, engaging purge") + //filepath is stored in the slice, the filename or file object, see above + for _, fp := range fsPurgeScroll { + utility.RemoveFile(fp) + } + } + + if len(dbPurgeScroll) > 0 { + slog.Info("database purge scroll is populated, engaging purge") + for _, f := range dbPurgeScroll { + DeregisterFile(db, f) } } - slog.Info("purge list", "files", purgeList) } diff --git a/internal/runtime/runtime.go b/internal/utility/environment.go similarity index 79% rename from internal/runtime/runtime.go rename to internal/utility/environment.go index 772d1e4..3967c5d 100644 --- a/internal/runtime/runtime.go +++ b/internal/utility/environment.go @@ -1,4 +1,4 @@ -package runtime +package utility import ( "os" @@ -53,20 +53,3 @@ func GrabEnvironment() Environment { WatchInterval: safeIntGrab("WATCHDOG_INTERVAL", 60), } } - -// part of filesystem checking -func EnsureOperation(workDir string) error { - nDirs := []string{ - workDir, - filepath.Join(workDir), - filepath.Join(workDir, "content"), - } - - for _, p := range nDirs { - if err := os.MkdirAll(p, 0755); err != nil { - return err - } - } - - return nil -} diff --git a/internal/utility/filesystem.go b/internal/utility/filesystem.go new file mode 100644 index 0000000..127e9b9 --- /dev/null +++ b/internal/utility/filesystem.go @@ -0,0 +1,32 @@ +package utility + +import ( + "os" + "path/filepath" +) + +// part of filesystem checking +func EnsureOperation(workDir string) error { + nDirs := []string{ + workDir, + filepath.Join(workDir), + filepath.Join(workDir, "content"), + } + + for _, p := range nDirs { + if err := os.MkdirAll(p, 0755); err != nil { + return err + } + } + + return nil +} + +func RemoveFile(p string) error { + err := os.Remove(p) + if err != nil { + return err + } + + return nil +}