feat: add locally syncing and watchdog

cmd/server/main.go

@@ -17,6 +17,8 @@ func main() {
 	slog.Info("grabbing environment variables")
 	env := utility.GrabEnvironment()
 
+	// TO DO, allow cmd args parsing
+
 	// checking directories to ensure its expected environment is ready
 	slog.Info("auditing operating environment")
 	if err := utility.EnsureOperation(env.DataDirectory); err != nil {
@@ -29,10 +31,10 @@ func main() {
 	slog.Info("kicking off database connection")
 	db, err := database.KickoffDatabase(env.DataDirectory)
 	if err != nil {
-		slog.Error("failed to initiate a database connection")
+		slog.Error("failed to initiate a database connection", "error", err)
 		os.Exit(1)
 	}
-	slog.Info("kicking off database watchdog", "watch_interval", env.WatchInterval)
+	slog.Info("kicking off database watchdog", "watch_interval", env.WatchdogInterval)
 	database.KickoffDatabaseWatchdog(env, db)
 
 	// TO DO make gin log as json

go.mod

@@ -5,13 +5,11 @@ go 1.25.0
 require (
 	github.com/gin-gonic/gin v1.12.0
 	github.com/google/uuid v1.6.0
-	gorm.io/datatypes v1.2.7
 	gorm.io/driver/sqlite v1.6.0
 	gorm.io/gorm v1.31.1
 )
 
 require (
-	filippo.io/edwards25519 v1.1.0 // indirect
 	github.com/bytedance/gopkg v0.1.3 // indirect
 	github.com/bytedance/sonic v1.15.0 // indirect
 	github.com/bytedance/sonic/loader v0.5.0 // indirect
@@ -21,7 +19,6 @@ require (
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-playground/validator/v10 v10.30.1 // indirect
-	github.com/go-sql-driver/mysql v1.8.1 // indirect
 	github.com/goccy/go-json v0.10.5 // indirect
 	github.com/goccy/go-yaml v1.19.2 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
@@ -45,5 +42,4 @@ require (
 	golang.org/x/sys v0.41.0 // indirect
 	golang.org/x/text v0.34.0 // indirect
 	google.golang.org/protobuf v1.36.10 // indirect
-	gorm.io/driver/mysql v1.5.6 // indirect
 )

go.sum

@@ -1,5 +1,3 @@
-filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
-filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
 github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M=
 github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM=
 github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE=
@@ -25,30 +23,15 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
 github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
 github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w=
 github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM=
-github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
-github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
-github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
 github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
 github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM=
 github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
-github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA=
-github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
-github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A=
-github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
-github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
-github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 h1:L0QtFUgDarD7Fpv9jeVMgy/+Ec0mtnmYuImjTz6dtDA=
-github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
-github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw=
-github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
-github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
-github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
@@ -63,8 +46,6 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
 github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
-github.com/microsoft/go-mssqldb v1.7.2 h1:CHkFJiObW7ItKTJfHo1QX7QBBD1iV+mn1eOyRP3b/PA=
-github.com/microsoft/go-mssqldb v1.7.2/go.mod h1:kOvZKUdrhhFQmxLZqbwUV0rHkNkZpthMITIb2Ko1IoA=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -103,8 +84,6 @@ golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
 golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
 golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
 golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
-golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
-golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
 golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
@@ -116,16 +95,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gorm.io/datatypes v1.2.7 h1:ww9GAhF1aGXZY3EB3cJPJ7//JiuQo7DlQA7NNlVaTdk=
-gorm.io/datatypes v1.2.7/go.mod h1:M2iO+6S3hhi4nAyYe444Pcb0dcIiOMJ7QHaUXxyiNZY=
-gorm.io/driver/mysql v1.5.6 h1:Ld4mkIickM+EliaQZQx3uOJDJHtrd70MxAUqWqlx3Y8=
-gorm.io/driver/mysql v1.5.6/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM=
-gorm.io/driver/postgres v1.5.0 h1:u2FXTy14l45qc3UeCJ7QaAXZmZfDDv0YrthvmRq1l0U=
-gorm.io/driver/postgres v1.5.0/go.mod h1:FUZXzO+5Uqg5zzwzv4KK49R8lvGIyscBOqYrtI1Ce9A=
 gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ=
 gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8=
-gorm.io/driver/sqlserver v1.6.0 h1:VZOBQVsVhkHU/NzNhRJKoANt5pZGQAS1Bwc6m6dgfnc=
-gorm.io/driver/sqlserver v1.6.0/go.mod h1:WQzt4IJo/WHKnckU9jXBLMJIVNMVeTu25dnOzehntWw=
-gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
 gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg=
 gorm.io/gorm v1.31.1/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs=

internal/api/routes_api.go

@@ -9,21 +9,6 @@ import (
 	"gorm.io/gorm"
 )
 
-// 0: unspecified
-// 1: video
-// 2: presentation
-// 3: internet URL
-func categorizeFilemode(ext string) database.MediaType {
-	switch ext {
-	case ".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4a":
-		return database.Video
-	case ".pptx", ".ppt", ".key", ".odp":
-		return database.Presentation
-	default:
-		return database.Unspecified
-	}
-}
-
 func spawnApiRoutes(api *gin.RouterGroup /* env runtime.Environment,*/, db *gorm.DB) {
 	// prefix: api
 	// Display the information on what is going on at the moment

internal/api/routes_file.go

@@ -1,7 +1,6 @@
 package api
 
 import (
-	"eden-server/internal/crypto"
 	"eden-server/internal/database"
 	"eden-server/internal/utility"
 	"errors"
@@ -10,7 +9,6 @@ import (
 	"path/filepath"
 
 	"github.com/gin-gonic/gin"
-	"github.com/google/uuid"
 	"gorm.io/gorm"
 )
 
@@ -35,57 +33,38 @@ func spawnFileRoutes(file *gin.RouterGroup, env utility.Environment, db *gorm.DB
 			return
 		}
 
-		checksum, err := crypto.CalculateHashFromRequest(f)
+		readerStream, err := f.Open()
 		if err != nil {
-			slog.Error("failed to calculate hash of file at given path", "error", err)
+			slog.Error("failed to open uploaded file in memory")
+		}
+		defer readerStream.Close()
+
+		fileData, err := database.BuildFileRecord(readerStream, f.Filename, env.ContentDirectory)
+		if err != nil {
+			slog.Error("failed to enroll file to the database", "error", err)
 			c.JSON(http.StatusInternalServerError, RespObj{
 				Msg: ieMes,
 			})
 			return
 		}
 
-		e := filepath.Ext(f.Filename)
+		if err := database.RegisterFile(db, fileData); err != nil {
-		m := categorizeFilemode(e)
-		if m == database.Unspecified {
-			slog.Debug("discarding file since its filetype is unsupported")
-			c.JSON(http.StatusUnsupportedMediaType, RespObj{
-				Msg: "unsupported filetype",
-			})
-			return
-		}
-
-		safeName := uuid.New().String() + "_" + string(m) + e
-		destPath := filepath.Join(env.DataDirectory, "content", safeName)
-
-		fData := database.File{
-			MediaType: m,
-			MetaName:  f.Filename,
-			FileName:  safeName,
-			FilePath:  destPath,
-			Checksum:  checksum,
-		}
-
-		// first register the file to the database
-		// error out if something bad happens (duplicate, failing db, etc)
-		err = database.RegisterFile(db, fData)
-		if err != nil {
 			if errors.Is(err, gorm.ErrDuplicatedKey) {
 				slog.Debug("discarding file since its a duplicate", "error", err)
 				c.JSON(http.StatusConflict, RespObj{
 					Msg: "file is a duplicate",
 				})
-				return
+			} else {
+				slog.Error("failed to insert filedata to the database", "error", err)
+				c.JSON(http.StatusInternalServerError, RespObj{
+					Msg: ieMes,
+				})
 			}
-
-			slog.Error("discarding file since an error occured", "error", err)
-			c.JSON(http.StatusInternalServerError, RespObj{
-				Msg: ieMes,
-			})
 			return
 		}
 
 		// save to filesystem after everything has given a green light
-		if err := c.SaveUploadedFile(f, destPath); err != nil {
+		if err := c.SaveUploadedFile(f, fileData.FilePath); err != nil {
 			slog.Error("failed to receive the file over http:", "error", err)
 			c.JSON(http.StatusInternalServerError, RespObj{
 				Msg: ieMes,

internal/crypto/crypto.go

@@ -1,26 +0,0 @@
-package crypto
-
-import (
-	"crypto/sha512"
-	"encoding/hex"
-	"io"
-	"mime/multipart"
-)
-
-func CalculateHashFromRequest(fileHeader *multipart.FileHeader) (string, error) {
-	src, err := fileHeader.Open()
-	if err != nil {
-		return "", err
-	}
-	defer src.Close()
-
-	h := sha512.New()
-	if _, err := io.Copy(h, src); err != nil {
-		return "", err
-	}
-
-	// return the sha checksum in hex
-	return hex.EncodeToString(h.Sum(nil)), nil
-	// alternatively return in base64
-	//return base64.StdEncoding.EncodeToString(h.Sum(nil)), nil
-}

internal/database/database.go

@@ -2,7 +2,6 @@ package database
 
 import (
 	"eden-server/internal/utility"
-	"log/slog"
 	"path/filepath"
 	"time"
 
@@ -24,7 +23,6 @@ func KickoffDatabase(workDir string) (*gorm.DB, error) {
 	}
 
 	// try to use GORM automigrate if the schema changes
-	slog.Info("performing migration")
 	if err := db.AutoMigrate(
 		&State{},
 		&Device{},
@@ -36,7 +34,7 @@ func KickoffDatabase(workDir string) (*gorm.DB, error) {
 	// create the first row if it does not exist yet
 	if err := db.FirstOrCreate(&State{}, State{
 		ID:        0,
-		MediaType: "unspecified",
+		MediaType: Unspecified,
 	}).Error; err != nil {
 		return nil, err
 	}
@@ -45,23 +43,12 @@ func KickoffDatabase(workDir string) (*gorm.DB, error) {
 }
 
 func KickoffDatabaseWatchdog(env utility.Environment, db *gorm.DB) {
-	timeInterval := time.Second * time.Duration(env.WatchInterval)
+	timeInterval := time.Second * time.Duration(env.WatchdogInterval)
 	ticker := time.NewTicker(timeInterval)
 
 	go func() {
 		defer ticker.Stop()
 
-		/*
-			// Possible future mechanism to stop the watchdog
-			// must be inside a non-conditional for loop
-			select {
-			case <-ticker.C: // ticker event
-				watchdog(env.DataDirectory, db)
-			case <-watchdogStop:
-				return
-			}
-		*/
-
 		// run the watchdog function once to see if all is well.
 		watchdog(env, db)
 		// then defer to a decoupled/disowned golang goroutine

internal/database/define.go

@@ -2,49 +2,75 @@ package database
 
 import (
 	"time"
-
-	"gorm.io/datatypes"
 )
 
 type MediaType string
 
 const (
-	Unspecified  MediaType = "unspecified"
 	Video        MediaType = "video"
 	Presentation MediaType = "presentation"
 	Internet     MediaType = "internet"
+	Unspecified  MediaType = "unspecified"
 )
 
+type Timestamps struct {
+	CreatedAt time.Time `gorm:"not null;"`
+	UpdatedAt time.Time `gorm:"not null;"`
+	ExpiresAt time.Time
+}
+
 type State struct {
-	ID int `gorm:"primaryKey"`
+	ID int `gorm:"primaryKey;not null;"`
 	// unspecified
 	// video
 	// presentation
 	// internet URL
 	MediaType MediaType `gorm:"type:varchar(20);not null"` // Must specify what kind of file it is
-	Targets   datatypes.JSON
+	// Must be target list who are compelled to listen to the command
-	Location  string // Must be the location where the file is downloadable on the API
+	// can be none when there is no targets specified (init stage)
-	UpdatedAt time.Time
+	Targets []string `gorm:"type:json"`
+	// Must be the location where the file is downloadable on the API
+	// can be none when there is no media specified (init stage)
+	Location string
+	Timestamps
+}
+
+type Key struct {
+	ID       int `gorm:"primaryKey;not null;"`
+	MetaName string
+	KeyName  string `gorm:"not null;"`
+	// We don't store the key itself, we hash the key
+	KeyHash string `gorm:"not null;"`
+	// we're cooking without pepper
+	KeySalt   string    `gorm:"not null;"`
+	CreatedAt time.Time `gorm:"not null;"`
+	Timestamps
 }
 
 type Device struct {
-	ID            int `gorm:"primaryKey"`
+	ID int `gorm:"primaryKey;not null;"`
+	// Device type is meant as a field where can be specified what type of device this is
+	// eg Raspberry Pi, PC, things like that
 	DeviceType    string
-	Hostname      string
+	Hostname      string `gorm:"not null;"`
-	RemoteAddress string
+	RemoteAddress string `gorm:"not null;"`
-	Alive         bool
+	Alive         bool   `gorm:"not null;"`
-	Compliant     bool
+	Compliant     bool   `gorm:"not null;"`
-	CreatedAt     time.Time
+	Timestamps
-	UpdatedAt     time.Time
 }
 
 type File struct {
-	ID        int       `gorm:"primaryKey"`
+	ID int `gorm:"primaryKey;not null;"`
+	// unspecified
+	// video
+	// presentation
+	// internet URL
 	MediaType MediaType `gorm:"type:varchar(20);not null;"`
-	MetaName  string
+	// the name given by the user
-	FileName  string
+	MetaName string
-	FilePath  string
+	FileName string `gorm:"not null;"`
-	Checksum  string `gorm:"uniqueIndex"` // hex encoded sha512 checksum
+	FilePath string `gorm:"not null;"`
-	CreatedAt time.Time
+	// hex encoded sha512 checksum
-	UpdatedAt time.Time
+	Checksum string `gorm:"uniqueIndex;not null;"`
+	Timestamps
 }

internal/database/functions.go

@@ -1,13 +1,74 @@
 package database
 
 import (
+	"eden-server/internal/utility"
+	"fmt"
+	"io"
+	"log/slog"
+	"path/filepath"
+
+	"github.com/google/uuid"
 	"gorm.io/gorm"
 )
 
+// 0: unspecified
+// 1: video
+// 2: presentation
+// 3: internet URL
+func CategorizeMediaType(ext string) (MediaType, bool) {
+
+	switch ext {
+	case ".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4a":
+		return Video, true
+	case ".pptx", ".ppt", ".key", ".odp":
+		return Presentation, true
+	default:
+		slog.Debug("marking file as invalid undefined extension")
+		return "", false
+	}
+}
+
+func GenerateSafeName(category MediaType, ext string) string {
+	return uuid.New().String() + "_" + string(category) + ext
+}
+
+// it has been made more general for DRY purposes
+// this function should only be called after manually checking the filetype
+func BuildFileRecord(r io.Reader, origName string, contentDirectory string) (File, error) {
+	ext := filepath.Ext(origName)
+	category, ok := CategorizeMediaType(ext)
+	if !ok {
+		return File{}, fmt.Errorf("unsupported filetype")
+	}
+
+	checksum, err := utility.HashReader(r)
+	if err != nil {
+		slog.Error("failed to calculate hash of file at given path", "error", err)
+		return File{}, err
+	}
+
+	safeName := GenerateSafeName(category, ext)
+	destPath := filepath.Join(contentDirectory, safeName)
+
+	fData := File{
+		MediaType: category,
+		MetaName:  origName,
+		FileName:  safeName,
+		FilePath:  destPath,
+		Checksum:  checksum,
+	}
+
+	return fData, nil
+}
+
 func GetState(db *gorm.DB) (State, error) {
 	var state State
 
-	return state, db.First(&state).Error
+	if err := db.First(&state).Error; err != nil {
+		return State{}, err
+	}
+
+	return state, nil
 }
 
 func GetFiles(db *gorm.DB) ([]File, error) {

internal/database/watchdog.go

@@ -2,6 +2,8 @@ package database
 
 import (
 	"eden-server/internal/utility"
+	"errors"
+	"fmt"
 	"log/slog"
 	"os"
 	"path/filepath"
@@ -9,19 +11,11 @@ import (
 	"gorm.io/gorm"
 )
 
-func watchdog(env utility.Environment, db *gorm.DB) {
+func filesystemGather(env utility.Environment) (map[string]struct{}, error) {
-	slog.Info("performing the watchdog cycle")
-
 	fsFiles, err := os.ReadDir(env.ContentDirectory)
 	if err != nil {
 		slog.Error("failed to read the content directory on the filesystem", "error", err)
-		return
+		return nil, err
-	}
-
-	dbFiles, err := GetFiles(db)
-	if err != nil {
-		slog.Error("failed to retrieve the files indexed from the database", "error", err)
-		return
 	}
 
 	// generate a set of filesystem contents
@@ -32,41 +26,104 @@ func watchdog(env utility.Environment, db *gorm.DB) {
 		fsSet[fullPath] = struct{}{}
 	}
 
+	return fsSet, nil
+}
+
+func databaseGather(db *gorm.DB) (map[string]File, error) {
+	dbFiles, err := GetFiles(db)
+	if err != nil {
+		slog.Error("failed to retrieve the files indexed from the database", "error", err)
+		return nil, err
+	}
+
 	// generate a set of Database contents
 	dbSet := make(map[string]File) // cool name for the files that are going to be deregistered from the database
 	for _, f := range dbFiles {
 		dbSet[f.FilePath] = f
 	}
 
+	return dbSet, nil
+}
+
+func watchdog(env utility.Environment, db *gorm.DB) {
+	slog.Info("performing the watchdog cycle")
+
+	fsSet, err := filesystemGather(env)
+	if err != nil {
+		return
+	}
+	dbSet, err := databaseGather(db)
+	if err != nil {
+		return
+	}
+
 	// FS -> DB
 	// check for orphaned filesystem files
-	var fsPurgeScroll []string
+	var fsOrphans []string
 	for path := range fsSet {
 		if _, exists := dbSet[path]; !exists {
-			fsPurgeScroll = append(fsPurgeScroll, path)
+			fsOrphans = append(fsOrphans, path)
 		}
 	}
 
+	if len(fsOrphans) > 0 {
+		slog.Info("filesystem orphans detected, engaging flow")
+		//filepath is stored in the slice, the filename or file object, see above
+
+		for _, fp := range fsOrphans {
+			// this switch is guarded by the environment.go its check making sure its one of the three
+			// the following logic is used to actually perform the sync modes, removal, enrollment or ignore
+			switch env.WatchdogSyncMode {
+			case "sync":
+				readerStream, err := os.Open(fp)
+				if err != nil {
+
+				}
+				defer readerStream.Close()
+
+				fileData, err := BuildFileRecord(readerStream, filepath.Base(fp), env.ContentDirectory)
+				if err != nil {
+					slog.Error("failed to enroll local file into the database", "error", err)
+				}
+
+				fmt.Println(fileData)
+
+				if err := RegisterFile(db, fileData); err != nil {
+					if errors.Is(err, gorm.ErrDuplicatedKey) {
+						slog.Debug("discarding file since its a duplicate", "error", err)
+					} else {
+						slog.Error("failed to insert filedata to the database", "error", err)
+					}
+				}
+
+				if err := os.Rename(fp, fileData.FilePath); err != nil {
+					slog.Error("failed to move the locally inserted file", "error", err)
+				}
+
+			case "strict":
+				err := utility.RemoveFile(fp)
+				if err != nil {
+					slog.Warn("failed to remove local file from the filesystem", "error", err)
+				}
+			case "dry":
+				slog.Debug("dry mode enabled, not purging", "filepath", fp)
+			}
+		}
+
+	}
+
 	// DB -> FS
 	// check stale database records
-	var dbPurgeScroll []File
+	var dbdbOrphans []File
 	for path, f := range dbSet {
 		if _, exists := fsSet[path]; !exists {
-			dbPurgeScroll = append(dbPurgeScroll, f)
+			dbdbOrphans = append(dbdbOrphans, f)
 		}
 	}
 
-	if len(fsPurgeScroll) > 0 {
+	if len(dbdbOrphans) > 0 {
-		slog.Info("filesystem purge scroll is populated, engaging purge")
+		slog.Info("database orphans detected, engaging flow")
-		//filepath is stored in the slice, the filename or file object, see above
+		for _, f := range dbdbOrphans {
-		for _, fp := range fsPurgeScroll {
-			utility.RemoveFile(fp)
-		}
-	}
-
-	if len(dbPurgeScroll) > 0 {
-		slog.Info("database purge scroll is populated, engaging purge")
-		for _, f := range dbPurgeScroll {
 			DeregisterFile(db, f)
 		}
 	}

internal/utility/environment.go

@@ -1,8 +1,10 @@
 package utility
 
 import (
+	"log/slog"
 	"os"
 	"path/filepath"
+	"slices"
 	"strconv"
 )
 
@@ -13,14 +15,24 @@ type Environment struct {
 	ContentDirectory string
 	Hostname         string
 	Port             int
-	WatchInterval    int
+	WatchdogInterval int
+	WatchdogSyncMode string
 }
 
+var (
+	validSyncModes = []string{
+		"sync",
+		"strict",
+		"dry",
+	}
+)
+
 // part of environment checking
 func safeStringGrab(key, fallback string) string {
 	if v, ok := os.LookupEnv(key); ok {
 		return v
 	}
+	slog.Debug("using fallback", "key", key, "fallback", fallback)
 	return fallback
 }
 
@@ -30,6 +42,17 @@ func safeIntGrab(key string, fallback int) int {
 			return i
 		}
 	}
+	slog.Debug("using fallback", "key", key, "fallback", fallback)
+	return fallback
+}
+
+func safeSyncModeGrab(key, fallback string) string {
+	if v, ok := os.LookupEnv(key); ok {
+		if slices.Contains(validSyncModes, v) {
+			return v
+		}
+	}
+	slog.Debug("using fallback", "key", key, "fallback", fallback)
 	return fallback
 }
 
@@ -48,8 +71,11 @@ func GrabEnvironment() Environment {
 		DataDirectory:    safeStringGrab("DATA_DIR", fbBase),
 		ContentDirectory: safeStringGrab("CONTENT_DIR", fbContent),
 		Hostname:         safeStringGrab("HOSTNAME", "0.0.0.0"),
-
+		Port:             safeIntGrab("PORT", 8080),
-		Port:          safeIntGrab("PORT", 8080),
+		WatchdogInterval: safeIntGrab("WATCHDOG_INTERVAL", 60),
-		WatchInterval: safeIntGrab("WATCHDOG_INTERVAL", 60),
+		// sync: sync local files to the database, for example when you want to allow local inserting files which then get added to the database
+		// strict: make the database leading, this is when you only allow API uploads to be registered, and remove orphaned filesystem files
+		// dry: do nothing
+		WatchdogSyncMode: safeStringGrab("WATCHDOG_SYNC_MODE", "strict"),
 	}
 }

internal/utility/hash.go

@@ -0,0 +1,41 @@
+package utility
+
+import (
+	"crypto/sha512"
+	"encoding/hex"
+	"io"
+	"mime/multipart"
+	"os"
+)
+
+func HashReader(r io.Reader) (string, error) {
+	h := sha512.New()
+	if _, err := io.Copy(h, r); err != nil {
+		return "", err
+	}
+
+	// return the sha checksum in hex
+	return hex.EncodeToString(h.Sum(nil)), nil
+	// alternatively return in base64
+	//return base64.StdEncoding.EncodeToString(h.Sum(nil)), nil
+}
+
+func HashUpload(fileHeader *multipart.FileHeader) (string, error) {
+	stream, err := fileHeader.Open()
+	if err != nil {
+		return "", err
+	}
+	defer stream.Close()
+
+	return HashReader(stream)
+}
+
+func HashFile(path string) (string, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer file.Close()
+
+	return HashReader(file)
+}