diff --git a/cmd/server/main.go b/cmd/server/main.go index 6f25924..78214d5 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -5,6 +5,7 @@ import ( "orbits-server/internal/server/api" "orbits-server/internal/server/bootstrap" "orbits-server/internal/server/database" + "orbits-server/internal/server/watchdog" "orbits-server/internal/shared/utility" "os" ) @@ -25,7 +26,7 @@ func main() { // initiating the database connection for which we safe things slog.Info("kicking off database connection") - db, err := database.KickoffDatabase(env.DataDirectory) + db, err := database.Kickoff(env.DataDirectory) if err != nil { slog.Error("failed to initiate a database connection", "error", err) os.Exit(1) @@ -33,8 +34,8 @@ func main() { // kick off the watchdog depending on the environment variables if env.Watchdog { - slog.Info("kicking off database watchdog", "watchdog_interval", env.WatchdogInterval) - database.KickoffDatabaseWatchdog(env, db) + slog.Info("kicking off watchdog", "watchdog_interval", env.WatchdogInterval) + watchdog.Kickoff(env, db) } else { slog.Info("skipping database watchdog", "watchdog_enabled", env.Watchdog) } @@ -42,5 +43,5 @@ func main() { // TO DO make gin log as json // get ready to kick off the http api with Vue frontend slog.Info("kicking off http api backend") - api.KickoffApi(logger, env, db) + api.Kickoff(logger, env, db) } diff --git a/internal/server/api/api.go b/internal/server/api/api.go index 4e91b1d..bd06f04 100644 --- a/internal/server/api/api.go +++ b/internal/server/api/api.go @@ -15,7 +15,7 @@ import ( // All error messages from slog must have an error field with the golang error // See bottom the the kickoff function for details -func KickoffApi(logger *slog.Logger, env bootstrap.Environment, db *gorm.DB) { +func Kickoff(logger *slog.Logger, env bootstrap.Environment, db *gorm.DB) { gin.SetMode(gin.ReleaseMode) // For a nice looking logger: diff --git a/internal/server/bootstrap/bootstrap.go b/internal/server/bootstrap/bootstrap.go index ff62fab..d54e053 100644 --- a/internal/server/bootstrap/bootstrap.go +++ b/internal/server/bootstrap/bootstrap.go @@ -8,22 +8,6 @@ import ( flag "github.com/spf13/pflag" ) -type Environment struct { - Version string `env:"VERSION" default:"0.0.1" flag:"version" usage:"option to specify a custom version"` - Codename string `env:"CODENAME" default:"Magical Anomaly" flag:"codename" usage:"option to change the release codename"` - LogLevel string `env:"LOG_LEVEL" default:"debug" flag:"log-level" usage:"option to change the loglevel"` - - DataDirectory string `env:"DATA_DIR" default:"./data" flag:"data-dir" usage:"option to specify where the state data gets stored"` - ContentDirectory string `env:"CONTENT_DIR" default:"./content" flag:"content-dir" usage:"option to specify where the content gets stored"` - Hostname string `env:"HOSTNAME" default:"0.0.0.0" flag:"hostname" usage:"option to specify the address/hostname to bind the api server to"` - Port int `env:"PORT" default:"8080" flag:"port" usage:"option to specify the port to bind the api server to"` - Authentication bool `env:"AUTHENTICATION" default:"true" flag:"authentication" usage:"option to disable authentication"` - - Watchdog bool `env:"WATCHDOG" default:"true" flag:"watchdog" usage:"option to disable watchdog"` - WatchdogInterval int `env:"WATCHDOG_INTERVAL" default:"60" flag:"watchdog-interval" usage:"option to specify the interval in second(s) on which watchdog runs"` - WatchdogSyncMode string `env:"WATCHDOG_SYNC_MODE" default:"strict" flag:"watchdog-mode" usage:"option to specify the mode watchdog will run with: strict|sync|dry"` -} - func loadFromEnv(env *Environment) { v := reflect.ValueOf(env).Elem() t := v.Type() diff --git a/internal/server/bootstrap/types.go b/internal/server/bootstrap/types.go new file mode 100644 index 0000000..91917f7 --- /dev/null +++ b/internal/server/bootstrap/types.go @@ -0,0 +1,18 @@ +package bootstrap + +type Environment struct { + Version string `env:"VERSION" default:"0.0.1" flag:"version" usage:"option to specify a custom version"` + Codename string `env:"CODENAME" default:"Magical Anomaly" flag:"codename" usage:"option to change the release codename"` + LogLevel string `env:"LOG_LEVEL" default:"debug" flag:"log-level" usage:"option to change the loglevel"` + + DataDirectory string `env:"DATA_DIR" default:"./data" flag:"data-dir" usage:"option to specify where the state data gets stored"` + ContentDirectory string `env:"CONTENT_DIR" default:"./content" flag:"content-dir" usage:"option to specify where the content gets stored"` + Hostname string `env:"HOSTNAME" default:"0.0.0.0" flag:"hostname" usage:"option to specify the address/hostname to bind the api server to"` + Port int `env:"PORT" default:"8080" flag:"port" usage:"option to specify the port to bind the api server to"` + Authentication bool `env:"AUTHENTICATION" default:"true" flag:"authentication" usage:"option to disable authentication"` + AdminKey string `env:"ADMIN_KEY" default:"" flag:"admin-key" usage:"option to specify a custom admin top-level authentication key"` + + Watchdog bool `env:"WATCHDOG" default:"true" flag:"watchdog" usage:"option to disable watchdog"` + WatchdogInterval int `env:"WATCHDOG_INTERVAL" default:"60" flag:"watchdog-interval" usage:"option to specify the interval in second(s) on which watchdog runs"` + WatchdogSyncMode string `env:"WATCHDOG_SYNC_MODE" default:"strict" flag:"watchdog-mode" usage:"option to specify the mode watchdog will run with: strict|sync|dry"` +} diff --git a/internal/server/database/database.go b/internal/server/database/database.go index cc02c1e..11ebc19 100644 --- a/internal/server/database/database.go +++ b/internal/server/database/database.go @@ -1,10 +1,8 @@ package database import ( - "orbits-server/internal/server/bootstrap" "orbits-server/internal/shared/utility" "path/filepath" - "time" "gorm.io/driver/sqlite" "gorm.io/gorm" @@ -13,7 +11,7 @@ import ( //var watchdogStop = make(chan struct{}) -func KickoffDatabase(workDir string) (*gorm.DB, error) { +func Kickoff(workDir string) (*gorm.DB, error) { dbLoc := filepath.Join(workDir, "station.db") db, err := gorm.Open(sqlite.Open(dbLoc), &gorm.Config{ Logger: logger.Discard, // disable gorm logging since its not slog (yet) @@ -45,20 +43,3 @@ func KickoffDatabase(workDir string) (*gorm.DB, error) { return db, nil } - -func KickoffDatabaseWatchdog(env bootstrap.Environment, db *gorm.DB) { - timeInterval := time.Second * time.Duration(env.WatchdogInterval) - ticker := time.NewTicker(timeInterval) - - go func() { - defer ticker.Stop() - - // run the watchdog function once to see if all is well. - watchdog(env, db) - // then defer to a decoupled/disowned golang goroutine - - for range ticker.C { - watchdog(env, db) - } - }() -} diff --git a/internal/server/database/functions.go b/internal/server/database/functions.go index 051cc3c..4989604 100644 --- a/internal/server/database/functions.go +++ b/internal/server/database/functions.go @@ -4,12 +4,60 @@ import ( "gorm.io/gorm" ) +/* +State functions +*/ + func LatestState(db *gorm.DB) (Command, error) { var state Command err := db.Last(&state).Error return state, err } +/* +Key functions +*/ + +func CountKeys(db *gorm.DB) (int64, error) { + var count int64 + err := db.Model(&AccessKey{}).Count(&count).Error + return count, err +} + +func ListKeys(db *gorm.DB) ([]AccessKey, error) { + var keys []AccessKey + err := db.Find(&keys).Error + return keys, err +} + +func CreateKey(db *gorm.DB, k AccessKey) error { + return db.Create(&k).Error +} + +func DeleteKeyByID(db *gorm.DB, id int) error { + res := db.Delete(&AccessKey{}, id) + + if res.Error != nil { + return res.Error + } + + if res.RowsAffected == 0 { + return gorm.ErrRecordNotFound + } + + return nil +} + +/* +File functions +*/ + +func CountFiles(db *gorm.DB) (int64, error) { + var count int64 + err := db.Model(&File{}).Count(&count).Error + return count, err +} + func ListFiles(db *gorm.DB) ([]File, error) { var files []File err := db.Find(&files).Error @@ -26,6 +74,16 @@ func CreateFile(db *gorm.DB, f File) error { return db.Create(&f).Error } -func DeleteFile(db *gorm.DB, f File) error { - return db.Delete(&f).Error +func DeleteFileByID(db *gorm.DB, id int) error { + res := db.Delete(&File{}, id) + + if res.Error != nil { + return res.Error + } + + if res.RowsAffected == 0 { + return gorm.ErrRecordNotFound + } + + return nil } diff --git a/internal/server/database/define.go b/internal/server/database/types.go similarity index 98% rename from internal/server/database/define.go rename to internal/server/database/types.go index 92040e7..c749a27 100644 --- a/internal/server/database/define.go +++ b/internal/server/database/types.go @@ -33,7 +33,7 @@ type AccessKey struct { MetaName string KeyName string `gorm:"not null;"` // We don't store the key itself, we hash the key - KeyHash string `gorm:"not null;"` + KeyHash string `gorm:"uniqueIndex;not null;"` // we're cooking without pepper Timestamps } diff --git a/internal/server/database/watchdog.go b/internal/server/database/watchdog.go deleted file mode 100644 index a8bfaa1..0000000 --- a/internal/server/database/watchdog.go +++ /dev/null @@ -1,134 +0,0 @@ -package database - -import ( - "errors" - "log/slog" - "orbits-server/internal/server/bootstrap" - "orbits-server/internal/shared/utility" - "os" - "path/filepath" - - "gorm.io/gorm" -) - -func filesystemGather(env bootstrap.Environment) (map[string]struct{}, error) { - fsFiles, err := os.ReadDir(env.ContentDirectory) - if err != nil { - slog.Error("failed to read the content directory on the filesystem", "error", err) - return nil, err - } - - // generate a set of filesystem contents - fsSet := make(map[string]struct{}) // cool name for the files that are (now) marked for annihilation - for _, f := range fsFiles { - // absolute path creation - fullPath := filepath.Join(env.ContentDirectory, f.Name()) - fsSet[fullPath] = struct{}{} - } - - return fsSet, nil -} - -func databaseGather(db *gorm.DB) (map[string]File, error) { - dbFiles, err := ListFiles(db) - if err != nil { - slog.Error("failed to retrieve the files indexed from the database", "error", err) - return nil, err - } - - // generate a set of Database contents - dbSet := make(map[string]File) // cool name for the files that are going to be deregistered from the database - for _, f := range dbFiles { - dbSet[f.FilePath] = f - } - - return dbSet, nil -} - -func watchdog(env bootstrap.Environment, db *gorm.DB) { - slog.Debug("performing the watchdog cycle") - - fsSet, err := filesystemGather(env) - if err != nil { - return - } - dbSet, err := databaseGather(db) - if err != nil { - return - } - - // FS -> DB - // check for orphaned filesystem files - var fsOrphans []string - for path := range fsSet { - if _, exists := dbSet[path]; !exists { - fsOrphans = append(fsOrphans, path) - } - } - - if len(fsOrphans) > 0 { - slog.Info("filesystem orphans detected, engaging flow") - //filepath is stored in the slice, the filename or file object, see above - - for _, fp := range fsOrphans { - // this switch is guarded by the environment.go its check making sure its one of the three - // the following logic is used to actually perform the sync modes, removal, enrollment or ignore - switch env.WatchdogSyncMode { - case "sync": - readerStream, err := os.Open(fp) - if err != nil { - slog.Error("failed to a reader stream") - continue - } - defer readerStream.Close() - - fileData, err := BuildFileRecord(readerStream, filepath.Base(fp), env.ContentDirectory) - if err != nil { - slog.Error("failed to enroll local file into the database", "error", err) - continue - } - - if err := CreateFile(db, fileData); err != nil { - if errors.Is(err, gorm.ErrDuplicatedKey) { - slog.Debug("discarding file since its a duplicate", "error", err) - } else { - slog.Error("failed to insert filedata to the database", "error", err) - } - continue - } - - // to fully finalize the enrollment process, we rename the locally inserted file to a unique filename - // this is to make all files comply, wether uploaded via the api or locally inserted with the filesystem - if err := os.Rename(fp, fileData.FilePath); err != nil { - slog.Error("failed to move the locally inserted file", "error", err) - } - case "strict": - err := utility.RemoveFile(fp) - if err != nil { - slog.Error("failed to remove local file from the filesystem", "error", err) - } - case "dry": - slog.Debug("dry mode enabled, not purging", "filepath", fp) - default: - slog.Warn("unknown watchdog mode", "mode", env.WatchdogSyncMode) - } - } - - } - - // DB -> FS - // check stale database records - var dbdbOrphans []File - for path, f := range dbSet { - if _, exists := fsSet[path]; !exists { - dbdbOrphans = append(dbdbOrphans, f) - } - } - - if len(dbdbOrphans) > 0 { - slog.Info("database orphans detected, engaging flow") - for _, f := range dbdbOrphans { - DeleteFile(db, f) - } - } -} diff --git a/internal/server/watchdog/executor.go b/internal/server/watchdog/executor.go new file mode 100644 index 0000000..910569d --- /dev/null +++ b/internal/server/watchdog/executor.go @@ -0,0 +1,57 @@ +package watchdog + +import ( + "log/slog" + "orbits-server/internal/server/bootstrap" + "orbits-server/internal/server/database" + "orbits-server/internal/shared/utility" + "os" + "path/filepath" + + "gorm.io/gorm" +) + +func applyFS(env bootstrap.Environment, db *gorm.DB, fsOrphans []string) { + for _, fp := range fsOrphans { + + switch env.WatchdogSyncMode { + + case "sync": + f, err := os.Open(fp) + if err != nil { + continue + } + + func() { + defer f.Close() + + fileData, err := database.BuildFileRecord( + f, + filepath.Base(fp), + env.ContentDirectory, + ) + if err != nil { + return + } + + database.CreateFile(db, fileData) + os.Rename(fp, fileData.FilePath) + }() + + case "strict": + utility.RemoveFile(fp) + + case "dry": + slog.Debug("dry mode", "file", fp) + } + } +} + +func applyDB(db *gorm.DB, dbOrphans []database.File) { + for _, f := range dbOrphans { + err := database.DeleteFileByID(db, f.ID) + if err != nil { + slog.Error("failed to apply database measures", "error", err) + } + } +} diff --git a/internal/server/watchdog/scanner.go b/internal/server/watchdog/scanner.go new file mode 100644 index 0000000..163e8c4 --- /dev/null +++ b/internal/server/watchdog/scanner.go @@ -0,0 +1,57 @@ +package watchdog + +import ( + "orbits-server/internal/server/bootstrap" + "orbits-server/internal/server/database" + "os" + "path/filepath" + + "gorm.io/gorm" +) + +func reconcile(fs map[string]struct{}, db map[string]database.File) Result { + r := Result{} + + for path := range fs { + if _, ok := db[path]; !ok { + r.FSOrphans = append(r.FSOrphans, path) + } + } + + for path, f := range db { + if _, ok := fs[path]; !ok { + r.DBOrphans = append(r.DBOrphans, f) + } + } + + return r +} + +func scanFS(env bootstrap.Environment) (map[string]struct{}, error) { + fsFiles, err := os.ReadDir(env.ContentDirectory) + if err != nil { + return nil, err + } + + res := make(map[string]struct{}) + for _, f := range fsFiles { + full := filepath.Join(env.ContentDirectory, f.Name()) + res[full] = struct{}{} + } + + return res, nil +} + +func scanDB(db *gorm.DB) (map[string]database.File, error) { + files, err := database.ListFiles(db) + if err != nil { + return nil, err + } + + res := make(map[string]database.File) + for _, f := range files { + res[f.FilePath] = f + } + + return res, nil +} diff --git a/internal/server/watchdog/watchdog.go b/internal/server/watchdog/watchdog.go new file mode 100644 index 0000000..4c5677f --- /dev/null +++ b/internal/server/watchdog/watchdog.go @@ -0,0 +1,61 @@ +package watchdog + +import ( + "log/slog" + "orbits-server/internal/server/bootstrap" + "orbits-server/internal/server/database" + "time" + + "gorm.io/gorm" +) + +type State struct { + FS map[string]struct{} + DB map[string]database.File +} + +type Result struct { + FSOrphans []string + DBOrphans []database.File +} + +func Kickoff(env bootstrap.Environment, db *gorm.DB) { + interval := time.Second * time.Duration(env.WatchdogInterval) + ticker := time.NewTicker(interval) + + go func() { + defer ticker.Stop() + + run(env, db) + + for range ticker.C { + run(env, db) + } + }() +} + +func run(env bootstrap.Environment, db *gorm.DB) { + slog.Debug("watchdog cycle start") + + fsState, err := scanFS(env) + if err != nil { + return + } + + dbState, err := scanDB(db) + if err != nil { + return + } + + result := reconcile(fsState, dbState) + + if len(result.FSOrphans) > 0 { + slog.Info("filesystem orphans detected") + applyFS(env, db, result.FSOrphans) + } + + if len(result.DBOrphans) > 0 { + slog.Info("database orphans detected") + applyDB(db, result.DBOrphans) + } +}