daanselen/orbits
SHA256
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: add locally syncing and watchdog

Browse Source
This commit is contained in:
DaanSelen
2026-04-22 15:26:59 +02:00
parent 0c287cc917
commit ec3a996d6a
12 changed files with 283 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/server/main.go
+4 -2
View File
@@ -17,6 +17,8 @@ func main() {
slog.Info("grabbing environment variables")
env := utility.GrabEnvironment()
// TO DO, allow cmd args parsing
// checking directories to ensure its expected environment is ready
slog.Info("auditing operating environment")
if err := utility.EnsureOperation(env.DataDirectory); err != nil {
@@ -29,10 +31,10 @@ func main() {
slog.Info("kicking off database connection")
db, err := database.KickoffDatabase(env.DataDirectory)
if err != nil {
slog.Error("failed to initiate a database connection")
slog.Error("failed to initiate a database connection", "error", err)
os.Exit(1)
}
slog.Info("kicking off database watchdog", "watch_interval", env.WatchInterval)
slog.Info("kicking off database watchdog", "watch_interval", env.WatchdogInterval)
database.KickoffDatabaseWatchdog(env, db)
// TO DO make gin log as json
go.mod
-4
View File
@@ -5,13 +5,11 @@ go 1.25.0
require (
github.com/gin-gonic/gin v1.12.0
github.com/google/uuid v1.6.0
gorm.io/datatypes v1.2.7
gorm.io/driver/sqlite v1.6.0
gorm.io/gorm v1.31.1
)
require (
filippo.io/edwards25519 v1.1.0 // indirect
github.com/bytedance/gopkg v0.1.3 // indirect
github.com/bytedance/sonic v1.15.0 // indirect
github.com/bytedance/sonic/loader v0.5.0 // indirect
@@ -21,7 +19,6 @@ require (
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.30.1 // indirect
github.com/go-sql-driver/mysql v1.8.1 // indirect
github.com/goccy/go-json v0.10.5 // indirect
github.com/goccy/go-yaml v1.19.2 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
@@ -45,5 +42,4 @@ require (
golang.org/x/sys v0.41.0 // indirect
golang.org/x/text v0.34.0 // indirect
google.golang.org/protobuf v1.36.10 // indirect
gorm.io/driver/mysql v1.5.6 // indirect
)
go.sum
-30
View File
@@ -1,5 +1,3 @@
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M=
github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM=
github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE=
@@ -25,30 +23,15 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w=
github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM=
github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM=
github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA=
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A=
github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 h1:L0QtFUgDarD7Fpv9jeVMgy/+Ec0mtnmYuImjTz6dtDA=
github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw=
github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
@@ -63,8 +46,6 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/microsoft/go-mssqldb v1.7.2 h1:CHkFJiObW7ItKTJfHo1QX7QBBD1iV+mn1eOyRP3b/PA=
github.com/microsoft/go-mssqldb v1.7.2/go.mod h1:kOvZKUdrhhFQmxLZqbwUV0rHkNkZpthMITIb2Ko1IoA=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -103,8 +84,6 @@ golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
@@ -116,16 +95,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gorm.io/datatypes v1.2.7 h1:ww9GAhF1aGXZY3EB3cJPJ7//JiuQo7DlQA7NNlVaTdk=
gorm.io/datatypes v1.2.7/go.mod h1:M2iO+6S3hhi4nAyYe444Pcb0dcIiOMJ7QHaUXxyiNZY=
gorm.io/driver/mysql v1.5.6 h1:Ld4mkIickM+EliaQZQx3uOJDJHtrd70MxAUqWqlx3Y8=
gorm.io/driver/mysql v1.5.6/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM=
gorm.io/driver/postgres v1.5.0 h1:u2FXTy14l45qc3UeCJ7QaAXZmZfDDv0YrthvmRq1l0U=
gorm.io/driver/postgres v1.5.0/go.mod h1:FUZXzO+5Uqg5zzwzv4KK49R8lvGIyscBOqYrtI1Ce9A=
gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ=
gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8=
gorm.io/driver/sqlserver v1.6.0 h1:VZOBQVsVhkHU/NzNhRJKoANt5pZGQAS1Bwc6m6dgfnc=
gorm.io/driver/sqlserver v1.6.0/go.mod h1:WQzt4IJo/WHKnckU9jXBLMJIVNMVeTu25dnOzehntWw=
gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg=
gorm.io/gorm v1.31.1/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs=
internal/api/routes_api.go
-15
View File
@@ -9,21 +9,6 @@ import (
"gorm.io/gorm"
)
// 0: unspecified
// 1: video
// 2: presentation
// 3: internet URL
func categorizeFilemode(ext string) database.MediaType {
switch ext {
case ".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4a":
return database.Video
case ".pptx", ".ppt", ".key", ".odp":
return database.Presentation
default:
return database.Unspecified
}
}
func spawnApiRoutes(api *gin.RouterGroup /* env runtime.Environment,*/, db *gorm.DB) {
// prefix: api
// Display the information on what is going on at the moment
internal/api/routes_file.go
+15 -36
View File
@@ -1,7 +1,6 @@
package api
import (
"eden-server/internal/crypto"
"eden-server/internal/database"
"eden-server/internal/utility"
"errors"
@@ -10,7 +9,6 @@ import (
"path/filepath"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"gorm.io/gorm"
)
@@ -35,57 +33,38 @@ func spawnFileRoutes(file *gin.RouterGroup, env utility.Environment, db *gorm.DB
return
}
checksum, err := crypto.CalculateHashFromRequest(f)
readerStream, err := f.Open()
if err != nil {
slog.Error("failed to calculate hash of file at given path", "error", err)
slog.Error("failed to open uploaded file in memory")
}
defer readerStream.Close()
fileData, err := database.BuildFileRecord(readerStream, f.Filename, env.ContentDirectory)
if err != nil {
slog.Error("failed to enroll file to the database", "error", err)
c.JSON(http.StatusInternalServerError, RespObj{
Msg: ieMes,
})
return
}
e := filepath.Ext(f.Filename)
m := categorizeFilemode(e)
if m == database.Unspecified {
slog.Debug("discarding file since its filetype is unsupported")
c.JSON(http.StatusUnsupportedMediaType, RespObj{
Msg: "unsupported filetype",
})
return
}
safeName := uuid.New().String() + "_" + string(m) + e
destPath := filepath.Join(env.DataDirectory, "content", safeName)
fData := database.File{
MediaType: m,
MetaName: f.Filename,
FileName: safeName,
FilePath: destPath,
Checksum: checksum,
}
// first register the file to the database
// error out if something bad happens (duplicate, failing db, etc)
err = database.RegisterFile(db, fData)
if err != nil {
if err := database.RegisterFile(db, fileData); err != nil {
if errors.Is(err, gorm.ErrDuplicatedKey) {
slog.Debug("discarding file since its a duplicate", "error", err)
c.JSON(http.StatusConflict, RespObj{
Msg: "file is a duplicate",
})
return
} else {
slog.Error("failed to insert filedata to the database", "error", err)
c.JSON(http.StatusInternalServerError, RespObj{
Msg: ieMes,
})
}
slog.Error("discarding file since an error occured", "error", err)
c.JSON(http.StatusInternalServerError, RespObj{
Msg: ieMes,
})
return
}
// save to filesystem after everything has given a green light
if err := c.SaveUploadedFile(f, destPath); err != nil {
if err := c.SaveUploadedFile(f, fileData.FilePath); err != nil {
slog.Error("failed to receive the file over http:", "error", err)
c.JSON(http.StatusInternalServerError, RespObj{
Msg: ieMes,
internal/crypto/crypto.go
-26
View File
@@ -1,26 +0,0 @@
package crypto
import (
"crypto/sha512"
"encoding/hex"
"io"
"mime/multipart"
)
func CalculateHashFromRequest(fileHeader *multipart.FileHeader) (string, error) {
src, err := fileHeader.Open()
if err != nil {
return "", err
}
defer src.Close()
h := sha512.New()
if _, err := io.Copy(h, src); err != nil {
return "", err
}
// return the sha checksum in hex
return hex.EncodeToString(h.Sum(nil)), nil
// alternatively return in base64
//return base64.StdEncoding.EncodeToString(h.Sum(nil)), nil
}
internal/database/database.go
+2 -15
View File
@@ -2,7 +2,6 @@ package database
import (
"eden-server/internal/utility"
"log/slog"
"path/filepath"
"time"
@@ -24,7 +23,6 @@ func KickoffDatabase(workDir string) (*gorm.DB, error) {
}
// try to use GORM automigrate if the schema changes
slog.Info("performing migration")
if err := db.AutoMigrate(
&State{},
&Device{},
@@ -36,7 +34,7 @@ func KickoffDatabase(workDir string) (*gorm.DB, error) {
// create the first row if it does not exist yet
if err := db.FirstOrCreate(&State{}, State{
ID: 0,
MediaType: "unspecified",
MediaType: Unspecified,
}).Error; err != nil {
return nil, err
}
@@ -45,23 +43,12 @@ func KickoffDatabase(workDir string) (*gorm.DB, error) {
}
func KickoffDatabaseWatchdog(env utility.Environment, db *gorm.DB) {
timeInterval := time.Second * time.Duration(env.WatchInterval)
timeInterval := time.Second * time.Duration(env.WatchdogInterval)
ticker := time.NewTicker(timeInterval)
go func() {
defer ticker.Stop()
/*
// Possible future mechanism to stop the watchdog
// must be inside a non-conditional for loop
select {
case <-ticker.C: // ticker event
watchdog(env.DataDirectory, db)
case <-watchdogStop:
return
}
*/
// run the watchdog function once to see if all is well.
watchdog(env, db)
// then defer to a decoupled/disowned golang goroutine
internal/database/define.go
+47 -21
View File
@@ -2,49 +2,75 @@ package database
import (
"time"
"gorm.io/datatypes"
)
type MediaType string
const (
Unspecified MediaType = "unspecified"
Video MediaType = "video"
Presentation MediaType = "presentation"
Internet MediaType = "internet"
Unspecified MediaType = "unspecified"
)
type Timestamps struct {
CreatedAt time.Time `gorm:"not null;"`
UpdatedAt time.Time `gorm:"not null;"`
ExpiresAt time.Time
}
type State struct {
ID int `gorm:"primaryKey"`
ID int `gorm:"primaryKey;not null;"`
// unspecified
// video
// presentation
// internet URL
MediaType MediaType `gorm:"type:varchar(20);not null"` // Must specify what kind of file it is
Targets datatypes.JSON
Location string // Must be the location where the file is downloadable on the API
UpdatedAt time.Time
// Must be target list who are compelled to listen to the command
// can be none when there is no targets specified (init stage)
Targets []string `gorm:"type:json"`
// Must be the location where the file is downloadable on the API
// can be none when there is no media specified (init stage)
Location string
Timestamps
}
type Key struct {
ID int `gorm:"primaryKey;not null;"`
MetaName string
KeyName string `gorm:"not null;"`
// We don't store the key itself, we hash the key
KeyHash string `gorm:"not null;"`
// we're cooking without pepper
KeySalt string `gorm:"not null;"`
CreatedAt time.Time `gorm:"not null;"`
Timestamps
}
type Device struct {
ID int `gorm:"primaryKey"`
ID int `gorm:"primaryKey;not null;"`
// Device type is meant as a field where can be specified what type of device this is
// eg Raspberry Pi, PC, things like that
DeviceType string
Hostname string
RemoteAddress string
Alive bool
Compliant bool
CreatedAt time.Time
UpdatedAt time.Time
Hostname string `gorm:"not null;"`
RemoteAddress string `gorm:"not null;"`
Alive bool `gorm:"not null;"`
Compliant bool `gorm:"not null;"`
Timestamps
}
type File struct {
ID int `gorm:"primaryKey"`
ID int `gorm:"primaryKey;not null;"`
// unspecified
// video
// presentation
// internet URL
MediaType MediaType `gorm:"type:varchar(20);not null;"`
MetaName string
FileName string
FilePath string
Checksum string `gorm:"uniqueIndex"` // hex encoded sha512 checksum
CreatedAt time.Time
UpdatedAt time.Time
// the name given by the user
MetaName string
FileName string `gorm:"not null;"`
FilePath string `gorm:"not null;"`
// hex encoded sha512 checksum
Checksum string `gorm:"uniqueIndex;not null;"`
Timestamps
}
internal/database/functions.go
+62 -1
View File
@@ -1,13 +1,74 @@
package database
import (
"eden-server/internal/utility"
"fmt"
"io"
"log/slog"
"path/filepath"
"github.com/google/uuid"
"gorm.io/gorm"
)
// 0: unspecified
// 1: video
// 2: presentation
// 3: internet URL
func CategorizeMediaType(ext string) (MediaType, bool) {
switch ext {
case ".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4a":
return Video, true
case ".pptx", ".ppt", ".key", ".odp":
return Presentation, true
default:
slog.Debug("marking file as invalid undefined extension")
return "", false
}
}
func GenerateSafeName(category MediaType, ext string) string {
return uuid.New().String() + "_" + string(category) + ext
}
// it has been made more general for DRY purposes
// this function should only be called after manually checking the filetype
func BuildFileRecord(r io.Reader, origName string, contentDirectory string) (File, error) {
ext := filepath.Ext(origName)
category, ok := CategorizeMediaType(ext)
if !ok {
return File{}, fmt.Errorf("unsupported filetype")
}
checksum, err := utility.HashReader(r)
if err != nil {
slog.Error("failed to calculate hash of file at given path", "error", err)
return File{}, err
}
safeName := GenerateSafeName(category, ext)
destPath := filepath.Join(contentDirectory, safeName)
fData := File{
MediaType: category,
MetaName: origName,
FileName: safeName,
FilePath: destPath,
Checksum: checksum,
}
return fData, nil
}
func GetState(db *gorm.DB) (State, error) {
var state State
return state, db.First(&state).Error
if err := db.First(&state).Error; err != nil {
return State{}, err
}
return state, nil
}
func GetFiles(db *gorm.DB) ([]File, error) {
internal/database/watchdog.go
+82 -25
View File
@@ -2,6 +2,8 @@ package database
import (
"eden-server/internal/utility"
"errors"
"fmt"
"log/slog"
"os"
"path/filepath"
@@ -9,19 +11,11 @@ import (
"gorm.io/gorm"
)
func watchdog(env utility.Environment, db *gorm.DB) {
slog.Info("performing the watchdog cycle")
func filesystemGather(env utility.Environment) (map[string]struct{}, error) {
fsFiles, err := os.ReadDir(env.ContentDirectory)
if err != nil {
slog.Error("failed to read the content directory on the filesystem", "error", err)
return
}
dbFiles, err := GetFiles(db)
if err != nil {
slog.Error("failed to retrieve the files indexed from the database", "error", err)
return
return nil, err
}
// generate a set of filesystem contents
@@ -32,41 +26,104 @@ func watchdog(env utility.Environment, db *gorm.DB) {
fsSet[fullPath] = struct{}{}
}
return fsSet, nil
}
func databaseGather(db *gorm.DB) (map[string]File, error) {
dbFiles, err := GetFiles(db)
if err != nil {
slog.Error("failed to retrieve the files indexed from the database", "error", err)
return nil, err
}
// generate a set of Database contents
dbSet := make(map[string]File) // cool name for the files that are going to be deregistered from the database
for _, f := range dbFiles {
dbSet[f.FilePath] = f
}
return dbSet, nil
}
func watchdog(env utility.Environment, db *gorm.DB) {
slog.Info("performing the watchdog cycle")
fsSet, err := filesystemGather(env)
if err != nil {
return
}
dbSet, err := databaseGather(db)
if err != nil {
return
}
// FS -> DB
// check for orphaned filesystem files
var fsPurgeScroll []string
var fsOrphans []string
for path := range fsSet {
if _, exists := dbSet[path]; !exists {
fsPurgeScroll = append(fsPurgeScroll, path)
fsOrphans = append(fsOrphans, path)
}
}
if len(fsOrphans) > 0 {
slog.Info("filesystem orphans detected, engaging flow")
//filepath is stored in the slice, the filename or file object, see above
for _, fp := range fsOrphans {
// this switch is guarded by the environment.go its check making sure its one of the three
// the following logic is used to actually perform the sync modes, removal, enrollment or ignore
switch env.WatchdogSyncMode {
case "sync":
readerStream, err := os.Open(fp)
if err != nil {
}
defer readerStream.Close()
fileData, err := BuildFileRecord(readerStream, filepath.Base(fp), env.ContentDirectory)
if err != nil {
slog.Error("failed to enroll local file into the database", "error", err)
}
fmt.Println(fileData)
if err := RegisterFile(db, fileData); err != nil {
if errors.Is(err, gorm.ErrDuplicatedKey) {
slog.Debug("discarding file since its a duplicate", "error", err)
} else {
slog.Error("failed to insert filedata to the database", "error", err)
}
}
if err := os.Rename(fp, fileData.FilePath); err != nil {
slog.Error("failed to move the locally inserted file", "error", err)
}
case "strict":
err := utility.RemoveFile(fp)
if err != nil {
slog.Warn("failed to remove local file from the filesystem", "error", err)
}
case "dry":
slog.Debug("dry mode enabled, not purging", "filepath", fp)
}
}
}
// DB -> FS
// check stale database records
var dbPurgeScroll []File
var dbdbOrphans []File
for path, f := range dbSet {
if _, exists := fsSet[path]; !exists {
dbPurgeScroll = append(dbPurgeScroll, f)
dbdbOrphans = append(dbdbOrphans, f)
}
}
if len(fsPurgeScroll) > 0 {
slog.Info("filesystem purge scroll is populated, engaging purge")
//filepath is stored in the slice, the filename or file object, see above
for _, fp := range fsPurgeScroll {
utility.RemoveFile(fp)
}
}
if len(dbPurgeScroll) > 0 {
slog.Info("database purge scroll is populated, engaging purge")
for _, f := range dbPurgeScroll {
if len(dbdbOrphans) > 0 {
slog.Info("database orphans detected, engaging flow")
for _, f := range dbdbOrphans {
DeregisterFile(db, f)
}
}
internal/utility/environment.go
+30 -4
View File
@@ -1,8 +1,10 @@
package utility
import (
"log/slog"
"os"
"path/filepath"
"slices"
"strconv"
)
@@ -13,14 +15,24 @@ type Environment struct {
ContentDirectory string
Hostname string
Port int
WatchInterval int
WatchdogInterval int
WatchdogSyncMode string
}
var (
validSyncModes = []string{
"sync",
"strict",
"dry",
}
)
// part of environment checking
func safeStringGrab(key, fallback string) string {
if v, ok := os.LookupEnv(key); ok {
return v
}
slog.Debug("using fallback", "key", key, "fallback", fallback)
return fallback
}
@@ -30,6 +42,17 @@ func safeIntGrab(key string, fallback int) int {
return i
}
}
slog.Debug("using fallback", "key", key, "fallback", fallback)
return fallback
}
func safeSyncModeGrab(key, fallback string) string {
if v, ok := os.LookupEnv(key); ok {
if slices.Contains(validSyncModes, v) {
return v
}
}
slog.Debug("using fallback", "key", key, "fallback", fallback)
return fallback
}
@@ -48,8 +71,11 @@ func GrabEnvironment() Environment {
DataDirectory: safeStringGrab("DATA_DIR", fbBase),
ContentDirectory: safeStringGrab("CONTENT_DIR", fbContent),
Hostname: safeStringGrab("HOSTNAME", "0.0.0.0"),
Port: safeIntGrab("PORT", 8080),
WatchInterval: safeIntGrab("WATCHDOG_INTERVAL", 60),
Port: safeIntGrab("PORT", 8080),
WatchdogInterval: safeIntGrab("WATCHDOG_INTERVAL", 60),
// sync: sync local files to the database, for example when you want to allow local inserting files which then get added to the database
// strict: make the database leading, this is when you only allow API uploads to be registered, and remove orphaned filesystem files
// dry: do nothing
WatchdogSyncMode: safeStringGrab("WATCHDOG_SYNC_MODE", "strict"),
}
}
internal/utility/hash.go
+41
View File
@@ -0,0 +1,41 @@
package utility
import (
"crypto/sha512"
"encoding/hex"
"io"
"mime/multipart"
"os"
)
func HashReader(r io.Reader) (string, error) {
h := sha512.New()
if _, err := io.Copy(h, r); err != nil {
return "", err
}
// return the sha checksum in hex
return hex.EncodeToString(h.Sum(nil)), nil
// alternatively return in base64
//return base64.StdEncoding.EncodeToString(h.Sum(nil)), nil
}
func HashUpload(fileHeader *multipart.FileHeader) (string, error) {
stream, err := fileHeader.Open()
if err != nil {
return "", err
}
defer stream.Close()
return HashReader(stream)
}
func HashFile(path string) (string, error) {
file, err := os.Open(path)
if err != nil {
return "", err
}
defer file.Close()
return HashReader(file)
}