feat: add working watchdog cycle

cmd/server/main.go

@@ -3,7 +3,7 @@ package main
 import (
 	"eden-server/internal/api"
 	"eden-server/internal/database"
-	"eden-server/internal/runtime"
+	"eden-server/internal/utility"
 	"log/slog"
 	"os"
 )
@@ -15,11 +15,11 @@ func main() {
 
 	// grab the environment variables from the runtime environment
 	slog.Info("grabbing environment variables")
-	env := runtime.GrabEnvironment()
+	env := utility.GrabEnvironment()
 
 	// checking directories to ensure its expected environment is ready
 	slog.Info("auditing operating environment")
-	if err := runtime.EnsureOperation(env.DataDirectory); err != nil {
+	if err := utility.EnsureOperation(env.DataDirectory); err != nil {
 		slog.Error("failed to ensure the operating environment", "error", err)
 		os.Exit(1)
 	}

internal/api/api.go

@@ -1,7 +1,7 @@
 package api
 
 import (
-	"eden-server/internal/runtime"
+	"eden-server/internal/utility"
 	"fmt"
 	"log/slog"
 
@@ -22,7 +22,7 @@ type RespObj struct {
 // All error messages from slog must have an error field with the golang error
 // See bottom the the kickoff function for details
 
-func KickoffApi(logger *slog.Logger, env runtime.Environment, db *gorm.DB) {
+func KickoffApi(logger *slog.Logger, env utility.Environment, db *gorm.DB) {
 	gin.SetMode(gin.ReleaseMode)
 
 	// For a nice looking logger:

internal/api/routes_file.go

@@ -3,7 +3,8 @@ package api
 import (
 	"eden-server/internal/crypto"
 	"eden-server/internal/database"
-	"eden-server/internal/runtime"
+	"eden-server/internal/utility"
+	"errors"
 	"log/slog"
 	"net/http"
 	"path/filepath"
@@ -13,7 +14,7 @@ import (
 	"gorm.io/gorm"
 )
 
-func spawnFileRoutes(file *gin.RouterGroup, env runtime.Environment, db *gorm.DB) {
+func spawnFileRoutes(file *gin.RouterGroup, env utility.Environment, db *gorm.DB) {
 	// /file/<file-name>
 	file.GET("/:filename", func(c *gin.Context) {
 		f := c.Param("filename")
@@ -27,26 +28,63 @@ func spawnFileRoutes(file *gin.RouterGroup, env runtime.Environment, db *gorm.DB
 	file.POST("/upload", func(c *gin.Context) {
 		f, err := c.FormFile("file")
 		if err != nil {
-			slog.Error("failed to get file details from request", "error", err)
+			slog.Debug("no file or file headers provided on the request", "error", err)
 			c.JSON(http.StatusBadRequest, RespObj{
 				Msg: "a file is required",
 			})
 			return
 		}
 
+		checksum, err := crypto.CalculateHashFromRequest(f)
+		if err != nil {
+			slog.Error("failed to calculate hash of file at given path", "error", err)
+			c.JSON(http.StatusInternalServerError, RespObj{
+				Msg: ieMes,
+			})
+			return
+		}
+
 		e := filepath.Ext(f.Filename)
 		m := categorizeFilemode(e)
 		if m == database.Unspecified {
-			slog.Warn("discarding file since its filetype is unsupported")
+			slog.Debug("discarding file since its filetype is unsupported")
 			c.JSON(http.StatusUnsupportedMediaType, RespObj{
 				Msg: "unsupported filetype",
 			})
 			return
 		}
 
-		safeName := uuid.New().String()[:8] + "_" + string(m) + e
+		safeName := uuid.New().String() + "_" + string(m) + e
 		destPath := filepath.Join(env.DataDirectory, "content", safeName)
 
+		fData := database.File{
+			MediaType: m,
+			MetaName:  f.Filename,
+			FileName:  safeName,
+			FilePath:  destPath,
+			Checksum:  checksum,
+		}
+
+		// first register the file to the database
+		// error out if something bad happens (duplicate, failing db, etc)
+		err = database.RegisterFile(db, fData)
+		if err != nil {
+			if errors.Is(err, gorm.ErrDuplicatedKey) {
+				slog.Debug("discarding file since its a duplicate", "error", err)
+				c.JSON(http.StatusConflict, RespObj{
+					Msg: "file is a duplicate",
+				})
+				return
+			}
+
+			slog.Error("discarding file since an error occured", "error", err)
+			c.JSON(http.StatusInternalServerError, RespObj{
+				Msg: ieMes,
+			})
+			return
+		}
+
+		// save to filesystem after everything has given a green light
 		if err := c.SaveUploadedFile(f, destPath); err != nil {
 			slog.Error("failed to receive the file over http:", "error", err)
 			c.JSON(http.StatusInternalServerError, RespObj{
@@ -55,21 +93,6 @@ func spawnFileRoutes(file *gin.RouterGroup, env runtime.Environment, db *gorm.DB
 			return
 		}
 
-		cSum, err := crypto.CalculateHash(destPath)
-		if err != nil {
-			slog.Error("failed to calculate hash of file at given path", "error", err)
-			c.JSON(http.StatusInternalServerError, RespObj{
-				Msg: ieMes,
-			})
-		}
-
-		fData := database.File{
-			MediaType: m,
-			GivenName: f.Filename,
-			Filepath:  destPath,
-			Checksum:  cSum,
-		}
-		database.RegisterFile(db, fData)
 		c.JSON(http.StatusCreated, RespObj{
 			Msg: "file has succesfully been uploaded",
 		})

internal/crypto/crypto.go

@@ -4,24 +4,23 @@ import (
 	"crypto/sha512"
 	"encoding/hex"
 	"io"
-	"os"
+	"mime/multipart"
 )
 
-func CalculateHash(p string) (string, error) {
+func CalculateHashFromRequest(fileHeader *multipart.FileHeader) (string, error) {
-	f, err := os.Open(p)
+	src, err := fileHeader.Open()
 	if err != nil {
 		return "", err
 	}
-	defer f.Close()
+	defer src.Close()
 
 	h := sha512.New()
-	if _, err := io.Copy(h, f); err != nil {
+	if _, err := io.Copy(h, src); err != nil {
 		return "", err
 	}
 
-	sum := h.Sum(nil)
 	// return the sha checksum in hex
-	return hex.EncodeToString(sum), nil
+	return hex.EncodeToString(h.Sum(nil)), nil
 	// alternatively return in base64
-	//return base64.StdEncoding.EncodeToString(sum), nil
+	//return base64.StdEncoding.EncodeToString(h.Sum(nil)), nil
 }

internal/database/database.go

@@ -1,30 +1,35 @@
 package database
 
 import (
-	"eden-server/internal/runtime"
+	"eden-server/internal/utility"
+	"log/slog"
 	"path/filepath"
 	"time"
 
 	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
 )
 
 //var watchdogStop = make(chan struct{})
 
 func KickoffDatabase(workDir string) (*gorm.DB, error) {
 	dbLoc := filepath.Join(workDir, "garden.db")
-	db, err := gorm.Open(sqlite.Open(dbLoc), &gorm.Config{})
+	db, err := gorm.Open(sqlite.Open(dbLoc), &gorm.Config{
+		Logger:         logger.Discard, // disable gorm logging since its not slog (yet)
+		TranslateError: true,
+	})
 	if err != nil {
 		return nil, err
 	}
 
-	if err := db.AutoMigrate(&State{}); err != nil {
+	// try to use GORM automigrate if the schema changes
-		return nil, err
+	slog.Info("performing migration")
-	}
+	if err := db.AutoMigrate(
-	if err := db.AutoMigrate(&Device{}); err != nil {
+		&State{},
-		return nil, err
+		&Device{},
-	}
+		&File{},
-	if err := db.AutoMigrate(&File{}); err != nil {
+	); err != nil {
 		return nil, err
 	}
 
@@ -39,7 +44,7 @@ func KickoffDatabase(workDir string) (*gorm.DB, error) {
 	return db, nil
 }
 
-func KickoffDatabaseWatchdog(env runtime.Environment, db *gorm.DB) {
+func KickoffDatabaseWatchdog(env utility.Environment, db *gorm.DB) {
 	timeInterval := time.Second * time.Duration(env.WatchInterval)
 	ticker := time.NewTicker(timeInterval)
 
@@ -58,11 +63,11 @@ func KickoffDatabaseWatchdog(env runtime.Environment, db *gorm.DB) {
 		*/
 
 		// run the watchdog function once to see if all is well.
-		watchdog(env.DataDirectory, db)
+		watchdog(env, db)
 		// then defer to a decoupled/disowned golang goroutine
 
 		for range ticker.C {
-			watchdog(env.DataDirectory, db)
+			watchdog(env, db)
 		}
 	}()
 }

internal/database/define.go

@@ -41,9 +41,10 @@ type Device struct {
 type File struct {
 	ID        int       `gorm:"primaryKey"`
 	MediaType MediaType `gorm:"type:varchar(20);not null;"`
-	GivenName string
+	MetaName  string
-	Filepath  string
+	FileName  string
-	Checksum  string // base64 encoded sha512 checksum
+	FilePath  string
+	Checksum  string `gorm:"uniqueIndex"` // hex encoded sha512 checksum
 	CreatedAt time.Time
 	UpdatedAt time.Time
 }

internal/database/functions.go

@@ -19,3 +19,7 @@ func GetFiles(db *gorm.DB) ([]File, error) {
 func RegisterFile(db *gorm.DB, f File) error {
 	return db.Create(&f).Error
 }
+
+func DeregisterFile(db *gorm.DB, f File) error {
+	return db.Delete(&f).Error
+}

internal/database/watchdog.go

@@ -1,36 +1,73 @@
 package database
 
 import (
+	"eden-server/internal/utility"
 	"log/slog"
 	"os"
+	"path/filepath"
 
 	"gorm.io/gorm"
 )
 
-func watchdog(w string, db *gorm.DB) {
+func watchdog(env utility.Environment, db *gorm.DB) {
 	slog.Info("performing the watchdog cycle")
 
-	files, err := GetFiles(db)
+	fsFiles, err := os.ReadDir(env.ContentDirectory)
+	if err != nil {
+		slog.Error("failed to read the content directory on the filesystem", "error", err)
+		return
+	}
+
+	dbFiles, err := GetFiles(db)
 	if err != nil {
 		slog.Error("failed to retrieve the files indexed from the database", "error", err)
 		return
 	}
 
-	var purgeList []string
+	// generate a set of filesystem contents
-	for _, f := range files {
+	fsSet := make(map[string]struct{}) // cool name for the files that are (now) marked for annihilation
-		i, err := os.Stat(f.Filepath)
+	for _, f := range fsFiles {
-		if err != nil {
+		// absolute path creation
-			if os.IsNotExist(err) {
+		fullPath := filepath.Join(env.ContentDirectory, f.Name())
-				purgeList = append(purgeList, f.Filepath)
+		fsSet[fullPath] = struct{}{}
-				continue
-			}
-			slog.Warn("stat failed", "file", f.Filepath, "error", err)
-			continue
 	}
 
-		if i.IsDir() {
+	// generate a set of Database contents
-			purgeList = append(purgeList, f.Filepath) // also mark it for purger if its a directory. We do not want that here
+	dbSet := make(map[string]File) // cool name for the files that are going to be deregistered from the database
+	for _, f := range dbFiles {
+		dbSet[f.FilePath] = f
+	}
+
+	// FS -> DB
+	// check for orphaned filesystem files
+	var fsPurgeScroll []string
+	for path := range fsSet {
+		if _, exists := dbSet[path]; !exists {
+			fsPurgeScroll = append(fsPurgeScroll, path)
+		}
+	}
+
+	// DB -> FS
+	// check stale database records
+	var dbPurgeScroll []File
+	for path, f := range dbSet {
+		if _, exists := fsSet[path]; !exists {
+			dbPurgeScroll = append(dbPurgeScroll, f)
+		}
+	}
+
+	if len(fsPurgeScroll) > 0 {
+		slog.Info("filesystem purge scroll is populated, engaging purge")
+		//filepath is stored in the slice, the filename or file object, see above
+		for _, fp := range fsPurgeScroll {
+			utility.RemoveFile(fp)
+		}
+	}
+
+	if len(dbPurgeScroll) > 0 {
+		slog.Info("database purge scroll is populated, engaging purge")
+		for _, f := range dbPurgeScroll {
+			DeregisterFile(db, f)
 		}
 	}
-	slog.Info("purge list", "files", purgeList)
 }

internal/utility/environment.go

@@ -1,4 +1,4 @@
-package runtime
+package utility
 
 import (
 	"os"
@@ -53,20 +53,3 @@ func GrabEnvironment() Environment {
 		WatchInterval: safeIntGrab("WATCHDOG_INTERVAL", 60),
 	}
 }
-
-// part of filesystem checking
-func EnsureOperation(workDir string) error {
-	nDirs := []string{
-		workDir,
-		filepath.Join(workDir),
-		filepath.Join(workDir, "content"),
-	}
-
-	for _, p := range nDirs {
-		if err := os.MkdirAll(p, 0755); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

internal/utility/filesystem.go

@@ -0,0 +1,32 @@
+package utility
+
+import (
+	"os"
+	"path/filepath"
+)
+
+// part of filesystem checking
+func EnsureOperation(workDir string) error {
+	nDirs := []string{
+		workDir,
+		filepath.Join(workDir),
+		filepath.Join(workDir, "content"),
+	}
+
+	for _, p := range nDirs {
+		if err := os.MkdirAll(p, 0755); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func RemoveFile(p string) error {
+	err := os.Remove(p)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}