diff --git a/cmd/server/main.go b/cmd/server/main.go index 284f194..6c6cce7 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -17,6 +17,8 @@ func main() { slog.Info("grabbing environment variables") env := utility.GrabEnvironment() + // TO DO, allow cmd args parsing + // checking directories to ensure its expected environment is ready slog.Info("auditing operating environment") if err := utility.EnsureOperation(env.DataDirectory); err != nil { @@ -29,10 +31,10 @@ func main() { slog.Info("kicking off database connection") db, err := database.KickoffDatabase(env.DataDirectory) if err != nil { - slog.Error("failed to initiate a database connection") + slog.Error("failed to initiate a database connection", "error", err) os.Exit(1) } - slog.Info("kicking off database watchdog", "watch_interval", env.WatchInterval) + slog.Info("kicking off database watchdog", "watch_interval", env.WatchdogInterval) database.KickoffDatabaseWatchdog(env, db) // TO DO make gin log as json diff --git a/go.mod b/go.mod index 9589014..dd22d5e 100644 --- a/go.mod +++ b/go.mod @@ -5,13 +5,11 @@ go 1.25.0 require ( github.com/gin-gonic/gin v1.12.0 github.com/google/uuid v1.6.0 - gorm.io/datatypes v1.2.7 gorm.io/driver/sqlite v1.6.0 gorm.io/gorm v1.31.1 ) require ( - filippo.io/edwards25519 v1.1.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect @@ -21,7 +19,6 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.30.1 // indirect - github.com/go-sql-driver/mysql v1.8.1 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/goccy/go-yaml v1.19.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect @@ -45,5 +42,4 @@ require ( golang.org/x/sys v0.41.0 // indirect golang.org/x/text v0.34.0 // indirect google.golang.org/protobuf v1.36.10 // indirect - gorm.io/driver/mysql v1.5.6 // indirect ) diff --git a/go.sum b/go.sum index 36078c4..4cdd4db 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,3 @@ -filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= -filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= @@ -25,30 +23,15 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w= github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= -github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= -github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= -github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= -github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= -github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= -github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= -github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= -github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 h1:L0QtFUgDarD7Fpv9jeVMgy/+Ec0mtnmYuImjTz6dtDA= -github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= -github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= -github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= -github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= @@ -63,8 +46,6 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/microsoft/go-mssqldb v1.7.2 h1:CHkFJiObW7ItKTJfHo1QX7QBBD1iV+mn1eOyRP3b/PA= -github.com/microsoft/go-mssqldb v1.7.2/go.mod h1:kOvZKUdrhhFQmxLZqbwUV0rHkNkZpthMITIb2Ko1IoA= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -103,8 +84,6 @@ golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= @@ -116,16 +95,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/datatypes v1.2.7 h1:ww9GAhF1aGXZY3EB3cJPJ7//JiuQo7DlQA7NNlVaTdk= -gorm.io/datatypes v1.2.7/go.mod h1:M2iO+6S3hhi4nAyYe444Pcb0dcIiOMJ7QHaUXxyiNZY= -gorm.io/driver/mysql v1.5.6 h1:Ld4mkIickM+EliaQZQx3uOJDJHtrd70MxAUqWqlx3Y8= -gorm.io/driver/mysql v1.5.6/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= -gorm.io/driver/postgres v1.5.0 h1:u2FXTy14l45qc3UeCJ7QaAXZmZfDDv0YrthvmRq1l0U= -gorm.io/driver/postgres v1.5.0/go.mod h1:FUZXzO+5Uqg5zzwzv4KK49R8lvGIyscBOqYrtI1Ce9A= gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8= -gorm.io/driver/sqlserver v1.6.0 h1:VZOBQVsVhkHU/NzNhRJKoANt5pZGQAS1Bwc6m6dgfnc= -gorm.io/driver/sqlserver v1.6.0/go.mod h1:WQzt4IJo/WHKnckU9jXBLMJIVNMVeTu25dnOzehntWw= -gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg= gorm.io/gorm v1.31.1/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs= diff --git a/internal/api/routes_api.go b/internal/api/routes_api.go index 9198bf1..978b4c0 100644 --- a/internal/api/routes_api.go +++ b/internal/api/routes_api.go @@ -9,21 +9,6 @@ import ( "gorm.io/gorm" ) -// 0: unspecified -// 1: video -// 2: presentation -// 3: internet URL -func categorizeFilemode(ext string) database.MediaType { - switch ext { - case ".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4a": - return database.Video - case ".pptx", ".ppt", ".key", ".odp": - return database.Presentation - default: - return database.Unspecified - } -} - func spawnApiRoutes(api *gin.RouterGroup /* env runtime.Environment,*/, db *gorm.DB) { // prefix: api // Display the information on what is going on at the moment diff --git a/internal/api/routes_file.go b/internal/api/routes_file.go index f1daa0d..e746fe0 100644 --- a/internal/api/routes_file.go +++ b/internal/api/routes_file.go @@ -1,7 +1,6 @@ package api import ( - "eden-server/internal/crypto" "eden-server/internal/database" "eden-server/internal/utility" "errors" @@ -10,7 +9,6 @@ import ( "path/filepath" "github.com/gin-gonic/gin" - "github.com/google/uuid" "gorm.io/gorm" ) @@ -35,57 +33,38 @@ func spawnFileRoutes(file *gin.RouterGroup, env utility.Environment, db *gorm.DB return } - checksum, err := crypto.CalculateHashFromRequest(f) + readerStream, err := f.Open() if err != nil { - slog.Error("failed to calculate hash of file at given path", "error", err) + slog.Error("failed to open uploaded file in memory") + } + defer readerStream.Close() + + fileData, err := database.BuildFileRecord(readerStream, f.Filename, env.ContentDirectory) + if err != nil { + slog.Error("failed to enroll file to the database", "error", err) c.JSON(http.StatusInternalServerError, RespObj{ Msg: ieMes, }) return } - e := filepath.Ext(f.Filename) - m := categorizeFilemode(e) - if m == database.Unspecified { - slog.Debug("discarding file since its filetype is unsupported") - c.JSON(http.StatusUnsupportedMediaType, RespObj{ - Msg: "unsupported filetype", - }) - return - } - - safeName := uuid.New().String() + "_" + string(m) + e - destPath := filepath.Join(env.DataDirectory, "content", safeName) - - fData := database.File{ - MediaType: m, - MetaName: f.Filename, - FileName: safeName, - FilePath: destPath, - Checksum: checksum, - } - - // first register the file to the database - // error out if something bad happens (duplicate, failing db, etc) - err = database.RegisterFile(db, fData) - if err != nil { + if err := database.RegisterFile(db, fileData); err != nil { if errors.Is(err, gorm.ErrDuplicatedKey) { slog.Debug("discarding file since its a duplicate", "error", err) c.JSON(http.StatusConflict, RespObj{ Msg: "file is a duplicate", }) - return + } else { + slog.Error("failed to insert filedata to the database", "error", err) + c.JSON(http.StatusInternalServerError, RespObj{ + Msg: ieMes, + }) } - - slog.Error("discarding file since an error occured", "error", err) - c.JSON(http.StatusInternalServerError, RespObj{ - Msg: ieMes, - }) return } // save to filesystem after everything has given a green light - if err := c.SaveUploadedFile(f, destPath); err != nil { + if err := c.SaveUploadedFile(f, fileData.FilePath); err != nil { slog.Error("failed to receive the file over http:", "error", err) c.JSON(http.StatusInternalServerError, RespObj{ Msg: ieMes, diff --git a/internal/crypto/crypto.go b/internal/crypto/crypto.go deleted file mode 100644 index 4ff021e..0000000 --- a/internal/crypto/crypto.go +++ /dev/null @@ -1,26 +0,0 @@ -package crypto - -import ( - "crypto/sha512" - "encoding/hex" - "io" - "mime/multipart" -) - -func CalculateHashFromRequest(fileHeader *multipart.FileHeader) (string, error) { - src, err := fileHeader.Open() - if err != nil { - return "", err - } - defer src.Close() - - h := sha512.New() - if _, err := io.Copy(h, src); err != nil { - return "", err - } - - // return the sha checksum in hex - return hex.EncodeToString(h.Sum(nil)), nil - // alternatively return in base64 - //return base64.StdEncoding.EncodeToString(h.Sum(nil)), nil -} diff --git a/internal/database/database.go b/internal/database/database.go index 2340e61..243c6a0 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -2,7 +2,6 @@ package database import ( "eden-server/internal/utility" - "log/slog" "path/filepath" "time" @@ -24,7 +23,6 @@ func KickoffDatabase(workDir string) (*gorm.DB, error) { } // try to use GORM automigrate if the schema changes - slog.Info("performing migration") if err := db.AutoMigrate( &State{}, &Device{}, @@ -36,7 +34,7 @@ func KickoffDatabase(workDir string) (*gorm.DB, error) { // create the first row if it does not exist yet if err := db.FirstOrCreate(&State{}, State{ ID: 0, - MediaType: "unspecified", + MediaType: Unspecified, }).Error; err != nil { return nil, err } @@ -45,23 +43,12 @@ func KickoffDatabase(workDir string) (*gorm.DB, error) { } func KickoffDatabaseWatchdog(env utility.Environment, db *gorm.DB) { - timeInterval := time.Second * time.Duration(env.WatchInterval) + timeInterval := time.Second * time.Duration(env.WatchdogInterval) ticker := time.NewTicker(timeInterval) go func() { defer ticker.Stop() - /* - // Possible future mechanism to stop the watchdog - // must be inside a non-conditional for loop - select { - case <-ticker.C: // ticker event - watchdog(env.DataDirectory, db) - case <-watchdogStop: - return - } - */ - // run the watchdog function once to see if all is well. watchdog(env, db) // then defer to a decoupled/disowned golang goroutine diff --git a/internal/database/define.go b/internal/database/define.go index 0707bf5..965f69d 100644 --- a/internal/database/define.go +++ b/internal/database/define.go @@ -2,49 +2,75 @@ package database import ( "time" - - "gorm.io/datatypes" ) type MediaType string const ( - Unspecified MediaType = "unspecified" Video MediaType = "video" Presentation MediaType = "presentation" Internet MediaType = "internet" + Unspecified MediaType = "unspecified" ) +type Timestamps struct { + CreatedAt time.Time `gorm:"not null;"` + UpdatedAt time.Time `gorm:"not null;"` + ExpiresAt time.Time +} + type State struct { - ID int `gorm:"primaryKey"` + ID int `gorm:"primaryKey;not null;"` // unspecified // video // presentation // internet URL MediaType MediaType `gorm:"type:varchar(20);not null"` // Must specify what kind of file it is - Targets datatypes.JSON - Location string // Must be the location where the file is downloadable on the API - UpdatedAt time.Time + // Must be target list who are compelled to listen to the command + // can be none when there is no targets specified (init stage) + Targets []string `gorm:"type:json"` + // Must be the location where the file is downloadable on the API + // can be none when there is no media specified (init stage) + Location string + Timestamps +} + +type Key struct { + ID int `gorm:"primaryKey;not null;"` + MetaName string + KeyName string `gorm:"not null;"` + // We don't store the key itself, we hash the key + KeyHash string `gorm:"not null;"` + // we're cooking without pepper + KeySalt string `gorm:"not null;"` + CreatedAt time.Time `gorm:"not null;"` + Timestamps } type Device struct { - ID int `gorm:"primaryKey"` + ID int `gorm:"primaryKey;not null;"` + // Device type is meant as a field where can be specified what type of device this is + // eg Raspberry Pi, PC, things like that DeviceType string - Hostname string - RemoteAddress string - Alive bool - Compliant bool - CreatedAt time.Time - UpdatedAt time.Time + Hostname string `gorm:"not null;"` + RemoteAddress string `gorm:"not null;"` + Alive bool `gorm:"not null;"` + Compliant bool `gorm:"not null;"` + Timestamps } type File struct { - ID int `gorm:"primaryKey"` + ID int `gorm:"primaryKey;not null;"` + // unspecified + // video + // presentation + // internet URL MediaType MediaType `gorm:"type:varchar(20);not null;"` - MetaName string - FileName string - FilePath string - Checksum string `gorm:"uniqueIndex"` // hex encoded sha512 checksum - CreatedAt time.Time - UpdatedAt time.Time + // the name given by the user + MetaName string + FileName string `gorm:"not null;"` + FilePath string `gorm:"not null;"` + // hex encoded sha512 checksum + Checksum string `gorm:"uniqueIndex;not null;"` + Timestamps } diff --git a/internal/database/functions.go b/internal/database/functions.go index 2f6b1a9..0e918ea 100644 --- a/internal/database/functions.go +++ b/internal/database/functions.go @@ -1,13 +1,74 @@ package database import ( + "eden-server/internal/utility" + "fmt" + "io" + "log/slog" + "path/filepath" + + "github.com/google/uuid" "gorm.io/gorm" ) +// 0: unspecified +// 1: video +// 2: presentation +// 3: internet URL +func CategorizeMediaType(ext string) (MediaType, bool) { + + switch ext { + case ".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4a": + return Video, true + case ".pptx", ".ppt", ".key", ".odp": + return Presentation, true + default: + slog.Debug("marking file as invalid undefined extension") + return "", false + } +} + +func GenerateSafeName(category MediaType, ext string) string { + return uuid.New().String() + "_" + string(category) + ext +} + +// it has been made more general for DRY purposes +// this function should only be called after manually checking the filetype +func BuildFileRecord(r io.Reader, origName string, contentDirectory string) (File, error) { + ext := filepath.Ext(origName) + category, ok := CategorizeMediaType(ext) + if !ok { + return File{}, fmt.Errorf("unsupported filetype") + } + + checksum, err := utility.HashReader(r) + if err != nil { + slog.Error("failed to calculate hash of file at given path", "error", err) + return File{}, err + } + + safeName := GenerateSafeName(category, ext) + destPath := filepath.Join(contentDirectory, safeName) + + fData := File{ + MediaType: category, + MetaName: origName, + FileName: safeName, + FilePath: destPath, + Checksum: checksum, + } + + return fData, nil +} + func GetState(db *gorm.DB) (State, error) { var state State - return state, db.First(&state).Error + if err := db.First(&state).Error; err != nil { + return State{}, err + } + + return state, nil } func GetFiles(db *gorm.DB) ([]File, error) { diff --git a/internal/database/watchdog.go b/internal/database/watchdog.go index 13facdb..12c2d73 100644 --- a/internal/database/watchdog.go +++ b/internal/database/watchdog.go @@ -2,6 +2,8 @@ package database import ( "eden-server/internal/utility" + "errors" + "fmt" "log/slog" "os" "path/filepath" @@ -9,19 +11,11 @@ import ( "gorm.io/gorm" ) -func watchdog(env utility.Environment, db *gorm.DB) { - slog.Info("performing the watchdog cycle") - +func filesystemGather(env utility.Environment) (map[string]struct{}, error) { fsFiles, err := os.ReadDir(env.ContentDirectory) if err != nil { slog.Error("failed to read the content directory on the filesystem", "error", err) - return - } - - dbFiles, err := GetFiles(db) - if err != nil { - slog.Error("failed to retrieve the files indexed from the database", "error", err) - return + return nil, err } // generate a set of filesystem contents @@ -32,41 +26,104 @@ func watchdog(env utility.Environment, db *gorm.DB) { fsSet[fullPath] = struct{}{} } + return fsSet, nil +} + +func databaseGather(db *gorm.DB) (map[string]File, error) { + dbFiles, err := GetFiles(db) + if err != nil { + slog.Error("failed to retrieve the files indexed from the database", "error", err) + return nil, err + } + // generate a set of Database contents dbSet := make(map[string]File) // cool name for the files that are going to be deregistered from the database for _, f := range dbFiles { dbSet[f.FilePath] = f } + return dbSet, nil +} + +func watchdog(env utility.Environment, db *gorm.DB) { + slog.Info("performing the watchdog cycle") + + fsSet, err := filesystemGather(env) + if err != nil { + return + } + dbSet, err := databaseGather(db) + if err != nil { + return + } + // FS -> DB // check for orphaned filesystem files - var fsPurgeScroll []string + var fsOrphans []string for path := range fsSet { if _, exists := dbSet[path]; !exists { - fsPurgeScroll = append(fsPurgeScroll, path) + fsOrphans = append(fsOrphans, path) } } + if len(fsOrphans) > 0 { + slog.Info("filesystem orphans detected, engaging flow") + //filepath is stored in the slice, the filename or file object, see above + + for _, fp := range fsOrphans { + // this switch is guarded by the environment.go its check making sure its one of the three + // the following logic is used to actually perform the sync modes, removal, enrollment or ignore + switch env.WatchdogSyncMode { + case "sync": + readerStream, err := os.Open(fp) + if err != nil { + + } + defer readerStream.Close() + + fileData, err := BuildFileRecord(readerStream, filepath.Base(fp), env.ContentDirectory) + if err != nil { + slog.Error("failed to enroll local file into the database", "error", err) + } + + fmt.Println(fileData) + + if err := RegisterFile(db, fileData); err != nil { + if errors.Is(err, gorm.ErrDuplicatedKey) { + slog.Debug("discarding file since its a duplicate", "error", err) + } else { + slog.Error("failed to insert filedata to the database", "error", err) + } + } + + if err := os.Rename(fp, fileData.FilePath); err != nil { + slog.Error("failed to move the locally inserted file", "error", err) + } + + case "strict": + err := utility.RemoveFile(fp) + if err != nil { + slog.Warn("failed to remove local file from the filesystem", "error", err) + } + case "dry": + slog.Debug("dry mode enabled, not purging", "filepath", fp) + } + } + + } + // DB -> FS // check stale database records - var dbPurgeScroll []File + var dbdbOrphans []File for path, f := range dbSet { if _, exists := fsSet[path]; !exists { - dbPurgeScroll = append(dbPurgeScroll, f) + dbdbOrphans = append(dbdbOrphans, f) } } - if len(fsPurgeScroll) > 0 { - slog.Info("filesystem purge scroll is populated, engaging purge") - //filepath is stored in the slice, the filename or file object, see above - for _, fp := range fsPurgeScroll { - utility.RemoveFile(fp) - } - } - - if len(dbPurgeScroll) > 0 { - slog.Info("database purge scroll is populated, engaging purge") - for _, f := range dbPurgeScroll { + if len(dbdbOrphans) > 0 { + slog.Info("database orphans detected, engaging flow") + for _, f := range dbdbOrphans { DeregisterFile(db, f) } } diff --git a/internal/utility/environment.go b/internal/utility/environment.go index 3967c5d..50ff059 100644 --- a/internal/utility/environment.go +++ b/internal/utility/environment.go @@ -1,8 +1,10 @@ package utility import ( + "log/slog" "os" "path/filepath" + "slices" "strconv" ) @@ -13,14 +15,24 @@ type Environment struct { ContentDirectory string Hostname string Port int - WatchInterval int + WatchdogInterval int + WatchdogSyncMode string } +var ( + validSyncModes = []string{ + "sync", + "strict", + "dry", + } +) + // part of environment checking func safeStringGrab(key, fallback string) string { if v, ok := os.LookupEnv(key); ok { return v } + slog.Debug("using fallback", "key", key, "fallback", fallback) return fallback } @@ -30,6 +42,17 @@ func safeIntGrab(key string, fallback int) int { return i } } + slog.Debug("using fallback", "key", key, "fallback", fallback) + return fallback +} + +func safeSyncModeGrab(key, fallback string) string { + if v, ok := os.LookupEnv(key); ok { + if slices.Contains(validSyncModes, v) { + return v + } + } + slog.Debug("using fallback", "key", key, "fallback", fallback) return fallback } @@ -48,8 +71,11 @@ func GrabEnvironment() Environment { DataDirectory: safeStringGrab("DATA_DIR", fbBase), ContentDirectory: safeStringGrab("CONTENT_DIR", fbContent), Hostname: safeStringGrab("HOSTNAME", "0.0.0.0"), - - Port: safeIntGrab("PORT", 8080), - WatchInterval: safeIntGrab("WATCHDOG_INTERVAL", 60), + Port: safeIntGrab("PORT", 8080), + WatchdogInterval: safeIntGrab("WATCHDOG_INTERVAL", 60), + // sync: sync local files to the database, for example when you want to allow local inserting files which then get added to the database + // strict: make the database leading, this is when you only allow API uploads to be registered, and remove orphaned filesystem files + // dry: do nothing + WatchdogSyncMode: safeStringGrab("WATCHDOG_SYNC_MODE", "strict"), } } diff --git a/internal/utility/hash.go b/internal/utility/hash.go new file mode 100644 index 0000000..8aebc61 --- /dev/null +++ b/internal/utility/hash.go @@ -0,0 +1,41 @@ +package utility + +import ( + "crypto/sha512" + "encoding/hex" + "io" + "mime/multipart" + "os" +) + +func HashReader(r io.Reader) (string, error) { + h := sha512.New() + if _, err := io.Copy(h, r); err != nil { + return "", err + } + + // return the sha checksum in hex + return hex.EncodeToString(h.Sum(nil)), nil + // alternatively return in base64 + //return base64.StdEncoding.EncodeToString(h.Sum(nil)), nil +} + +func HashUpload(fileHeader *multipart.FileHeader) (string, error) { + stream, err := fileHeader.Open() + if err != nil { + return "", err + } + defer stream.Close() + + return HashReader(stream) +} + +func HashFile(path string) (string, error) { + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + + return HashReader(file) +}