daanselen/orbits
SHA256
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: add working watchdog cycle

Browse Source
This commit is contained in:
DaanSelen
2026-04-22 11:51:57 +02:00
parent bf04e97850
commit 0c287cc917
10 changed files with 166 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/server/main.go
+3 -3
View File
@@ -3,7 +3,7 @@ package main
import (
"eden-server/internal/api"
"eden-server/internal/database"
"eden-server/internal/runtime"
"eden-server/internal/utility"
"log/slog"
"os"
)
@@ -15,11 +15,11 @@ func main() {
// grab the environment variables from the runtime environment
slog.Info("grabbing environment variables")
env := runtime.GrabEnvironment()
env := utility.GrabEnvironment()
// checking directories to ensure its expected environment is ready
slog.Info("auditing operating environment")
if err := runtime.EnsureOperation(env.DataDirectory); err != nil {
if err := utility.EnsureOperation(env.DataDirectory); err != nil {
slog.Error("failed to ensure the operating environment", "error", err)
os.Exit(1)
}
internal/api/api.go
+2 -2
View File
@@ -1,7 +1,7 @@
package api
import (
"eden-server/internal/runtime"
"eden-server/internal/utility"
"fmt"
"log/slog"
@@ -22,7 +22,7 @@ type RespObj struct {
// All error messages from slog must have an error field with the golang error
// See bottom the the kickoff function for details
func KickoffApi(logger *slog.Logger, env runtime.Environment, db *gorm.DB) {
func KickoffApi(logger *slog.Logger, env utility.Environment, db *gorm.DB) {
gin.SetMode(gin.ReleaseMode)
// For a nice looking logger:
internal/api/routes_file.go
+43 -20
View File
@@ -3,7 +3,8 @@ package api
import (
"eden-server/internal/crypto"
"eden-server/internal/database"
"eden-server/internal/runtime"
"eden-server/internal/utility"
"errors"
"log/slog"
"net/http"
"path/filepath"
@@ -13,7 +14,7 @@ import (
"gorm.io/gorm"
)
func spawnFileRoutes(file *gin.RouterGroup, env runtime.Environment, db *gorm.DB) {
func spawnFileRoutes(file *gin.RouterGroup, env utility.Environment, db *gorm.DB) {
// /file/<file-name>
file.GET("/:filename", func(c *gin.Context) {
f := c.Param("filename")
@@ -27,26 +28,63 @@ func spawnFileRoutes(file *gin.RouterGroup, env runtime.Environment, db *gorm.DB
file.POST("/upload", func(c *gin.Context) {
f, err := c.FormFile("file")
if err != nil {
slog.Error("failed to get file details from request", "error", err)
slog.Debug("no file or file headers provided on the request", "error", err)
c.JSON(http.StatusBadRequest, RespObj{
Msg: "a file is required",
})
return
}
checksum, err := crypto.CalculateHashFromRequest(f)
if err != nil {
slog.Error("failed to calculate hash of file at given path", "error", err)
c.JSON(http.StatusInternalServerError, RespObj{
Msg: ieMes,
})
return
}
e := filepath.Ext(f.Filename)
m := categorizeFilemode(e)
if m == database.Unspecified {
slog.Warn("discarding file since its filetype is unsupported")
slog.Debug("discarding file since its filetype is unsupported")
c.JSON(http.StatusUnsupportedMediaType, RespObj{
Msg: "unsupported filetype",
})
return
}
safeName := uuid.New().String()[:8] + "_" + string(m) + e
safeName := uuid.New().String() + "_" + string(m) + e
destPath := filepath.Join(env.DataDirectory, "content", safeName)
fData := database.File{
MediaType: m,
MetaName: f.Filename,
FileName: safeName,
FilePath: destPath,
Checksum: checksum,
}
// first register the file to the database
// error out if something bad happens (duplicate, failing db, etc)
err = database.RegisterFile(db, fData)
if err != nil {
if errors.Is(err, gorm.ErrDuplicatedKey) {
slog.Debug("discarding file since its a duplicate", "error", err)
c.JSON(http.StatusConflict, RespObj{
Msg: "file is a duplicate",
})
return
}
slog.Error("discarding file since an error occured", "error", err)
c.JSON(http.StatusInternalServerError, RespObj{
Msg: ieMes,
})
return
}
// save to filesystem after everything has given a green light
if err := c.SaveUploadedFile(f, destPath); err != nil {
slog.Error("failed to receive the file over http:", "error", err)
c.JSON(http.StatusInternalServerError, RespObj{
@@ -55,21 +93,6 @@ func spawnFileRoutes(file *gin.RouterGroup, env runtime.Environment, db *gorm.DB
return
}
cSum, err := crypto.CalculateHash(destPath)
if err != nil {
slog.Error("failed to calculate hash of file at given path", "error", err)
c.JSON(http.StatusInternalServerError, RespObj{
Msg: ieMes,
})
}
fData := database.File{
MediaType: m,
GivenName: f.Filename,
Filepath: destPath,
Checksum: cSum,
}
database.RegisterFile(db, fData)
c.JSON(http.StatusCreated, RespObj{
Msg: "file has succesfully been uploaded",
})
internal/crypto/crypto.go
+7 -8
View File
@@ -4,24 +4,23 @@ import (
"crypto/sha512"
"encoding/hex"
"io"
"os"
"mime/multipart"
)
func CalculateHash(p string) (string, error) {
f, err := os.Open(p)
func CalculateHashFromRequest(fileHeader *multipart.FileHeader) (string, error) {
src, err := fileHeader.Open()
if err != nil {
return "", err
}
defer f.Close()
defer src.Close()
h := sha512.New()
if _, err := io.Copy(h, f); err != nil {
if _, err := io.Copy(h, src); err != nil {
return "", err
}
sum := h.Sum(nil)
// return the sha checksum in hex
return hex.EncodeToString(sum), nil
return hex.EncodeToString(h.Sum(nil)), nil
// alternatively return in base64
//return base64.StdEncoding.EncodeToString(sum), nil
//return base64.StdEncoding.EncodeToString(h.Sum(nil)), nil
}
internal/database/database.go
+17 -12
View File
@@ -1,30 +1,35 @@
package database
import (
"eden-server/internal/runtime"
"eden-server/internal/utility"
"log/slog"
"path/filepath"
"time"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"
)
//var watchdogStop = make(chan struct{})
func KickoffDatabase(workDir string) (*gorm.DB, error) {
dbLoc := filepath.Join(workDir, "garden.db")
db, err := gorm.Open(sqlite.Open(dbLoc), &gorm.Config{})
db, err := gorm.Open(sqlite.Open(dbLoc), &gorm.Config{
Logger: logger.Discard, // disable gorm logging since its not slog (yet)
TranslateError: true,
})
if err != nil {
return nil, err
}
if err := db.AutoMigrate(&State{}); err != nil {
return nil, err
}
if err := db.AutoMigrate(&Device{}); err != nil {
return nil, err
}
if err := db.AutoMigrate(&File{}); err != nil {
// try to use GORM automigrate if the schema changes
slog.Info("performing migration")
if err := db.AutoMigrate(
&State{},
&Device{},
&File{},
); err != nil {
return nil, err
}
@@ -39,7 +44,7 @@ func KickoffDatabase(workDir string) (*gorm.DB, error) {
return db, nil
}
func KickoffDatabaseWatchdog(env runtime.Environment, db *gorm.DB) {
func KickoffDatabaseWatchdog(env utility.Environment, db *gorm.DB) {
timeInterval := time.Second * time.Duration(env.WatchInterval)
ticker := time.NewTicker(timeInterval)
@@ -58,11 +63,11 @@ func KickoffDatabaseWatchdog(env runtime.Environment, db *gorm.DB) {
*/
// run the watchdog function once to see if all is well.
watchdog(env.DataDirectory, db)
watchdog(env, db)
// then defer to a decoupled/disowned golang goroutine
for range ticker.C {
watchdog(env.DataDirectory, db)
watchdog(env, db)
}
}()
}
internal/database/define.go
+4 -3
View File
@@ -41,9 +41,10 @@ type Device struct {
type File struct {
ID int `gorm:"primaryKey"`
MediaType MediaType `gorm:"type:varchar(20);not null;"`
GivenName string
Filepath string
Checksum string // base64 encoded sha512 checksum
MetaName string
FileName string
FilePath string
Checksum string `gorm:"uniqueIndex"` // hex encoded sha512 checksum
CreatedAt time.Time
UpdatedAt time.Time
}
internal/database/functions.go
+4
View File
@@ -19,3 +19,7 @@ func GetFiles(db *gorm.DB) ([]File, error) {
func RegisterFile(db *gorm.DB, f File) error {
return db.Create(&f).Error
}
func DeregisterFile(db *gorm.DB, f File) error {
return db.Delete(&f).Error
}
internal/database/watchdog.go
+52 -15
View File
@@ -1,36 +1,73 @@
package database
import (
"eden-server/internal/utility"
"log/slog"
"os"
"path/filepath"
"gorm.io/gorm"
)
func watchdog(w string, db *gorm.DB) {
func watchdog(env utility.Environment, db *gorm.DB) {
slog.Info("performing the watchdog cycle")
files, err := GetFiles(db)
fsFiles, err := os.ReadDir(env.ContentDirectory)
if err != nil {
slog.Error("failed to read the content directory on the filesystem", "error", err)
return
}
dbFiles, err := GetFiles(db)
if err != nil {
slog.Error("failed to retrieve the files indexed from the database", "error", err)
return
}
var purgeList []string
for _, f := range files {
i, err := os.Stat(f.Filepath)
if err != nil {
if os.IsNotExist(err) {
purgeList = append(purgeList, f.Filepath)
continue
}
slog.Warn("stat failed", "file", f.Filepath, "error", err)
continue
// generate a set of filesystem contents
fsSet := make(map[string]struct{}) // cool name for the files that are (now) marked for annihilation
for _, f := range fsFiles {
// absolute path creation
fullPath := filepath.Join(env.ContentDirectory, f.Name())
fsSet[fullPath] = struct{}{}
}
if i.IsDir() {
purgeList = append(purgeList, f.Filepath) // also mark it for purger if its a directory. We do not want that here
// generate a set of Database contents
dbSet := make(map[string]File) // cool name for the files that are going to be deregistered from the database
for _, f := range dbFiles {
dbSet[f.FilePath] = f
}
// FS -> DB
// check for orphaned filesystem files
var fsPurgeScroll []string
for path := range fsSet {
if _, exists := dbSet[path]; !exists {
fsPurgeScroll = append(fsPurgeScroll, path)
}
}
// DB -> FS
// check stale database records
var dbPurgeScroll []File
for path, f := range dbSet {
if _, exists := fsSet[path]; !exists {
dbPurgeScroll = append(dbPurgeScroll, f)
}
}
if len(fsPurgeScroll) > 0 {
slog.Info("filesystem purge scroll is populated, engaging purge")
//filepath is stored in the slice, the filename or file object, see above
for _, fp := range fsPurgeScroll {
utility.RemoveFile(fp)
}
}
if len(dbPurgeScroll) > 0 {
slog.Info("database purge scroll is populated, engaging purge")
for _, f := range dbPurgeScroll {
DeregisterFile(db, f)
}
}
slog.Info("purge list", "files", purgeList)
}
internal/runtime/runtime.go → internal/utility/environment.go
+1 -18
View File
@@ -1,4 +1,4 @@
package runtime
package utility
import (
"os"
@@ -53,20 +53,3 @@ func GrabEnvironment() Environment {
WatchInterval: safeIntGrab("WATCHDOG_INTERVAL", 60),
}
}
// part of filesystem checking
func EnsureOperation(workDir string) error {
nDirs := []string{
workDir,
filepath.Join(workDir),
filepath.Join(workDir, "content"),
}
for _, p := range nDirs {
if err := os.MkdirAll(p, 0755); err != nil {
return err
}
}
return nil
}
internal/utility/filesystem.go
+32
View File
@@ -0,0 +1,32 @@
package utility
import (
"os"
"path/filepath"
)
// part of filesystem checking
func EnsureOperation(workDir string) error {
nDirs := []string{
workDir,
filepath.Join(workDir),
filepath.Join(workDir, "content"),
}
for _, p := range nDirs {
if err := os.MkdirAll(p, 0755); err != nil {
return err
}
}
return nil
}
func RemoveFile(p string) error {
err := os.Remove(p)
if err != nil {
return err
}
return nil
}