daanselen/orbits
SHA256
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: add locally syncing and watchdog

Browse Source
This commit is contained in:
DaanSelen
2026-04-22 15:26:59 +02:00
parent 0c287cc917
commit ec3a996d6a
12 changed files with 283 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/database/database.go
+2 -15
View File
@@ -2,7 +2,6 @@ package database
import (
"eden-server/internal/utility"
"log/slog"
"path/filepath"
"time"
@@ -24,7 +23,6 @@ func KickoffDatabase(workDir string) (*gorm.DB, error) {
}
// try to use GORM automigrate if the schema changes
slog.Info("performing migration")
if err := db.AutoMigrate(
&State{},
&Device{},
@@ -36,7 +34,7 @@ func KickoffDatabase(workDir string) (*gorm.DB, error) {
// create the first row if it does not exist yet
if err := db.FirstOrCreate(&State{}, State{
ID: 0,
MediaType: "unspecified",
MediaType: Unspecified,
}).Error; err != nil {
return nil, err
}
@@ -45,23 +43,12 @@ func KickoffDatabase(workDir string) (*gorm.DB, error) {
}
func KickoffDatabaseWatchdog(env utility.Environment, db *gorm.DB) {
timeInterval := time.Second * time.Duration(env.WatchInterval)
timeInterval := time.Second * time.Duration(env.WatchdogInterval)
ticker := time.NewTicker(timeInterval)
go func() {
defer ticker.Stop()
/*
// Possible future mechanism to stop the watchdog
// must be inside a non-conditional for loop
select {
case <-ticker.C: // ticker event
watchdog(env.DataDirectory, db)
case <-watchdogStop:
return
}
*/
// run the watchdog function once to see if all is well.
watchdog(env, db)
// then defer to a decoupled/disowned golang goroutine
internal/database/define.go
+47 -21
View File
@@ -2,49 +2,75 @@ package database
import (
"time"
"gorm.io/datatypes"
)
type MediaType string
const (
Unspecified MediaType = "unspecified"
Video MediaType = "video"
Presentation MediaType = "presentation"
Internet MediaType = "internet"
Unspecified MediaType = "unspecified"
)
type Timestamps struct {
CreatedAt time.Time `gorm:"not null;"`
UpdatedAt time.Time `gorm:"not null;"`
ExpiresAt time.Time
}
type State struct {
ID int `gorm:"primaryKey"`
ID int `gorm:"primaryKey;not null;"`
// unspecified
// video
// presentation
// internet URL
MediaType MediaType `gorm:"type:varchar(20);not null"` // Must specify what kind of file it is
Targets datatypes.JSON
Location string // Must be the location where the file is downloadable on the API
UpdatedAt time.Time
// Must be target list who are compelled to listen to the command
// can be none when there is no targets specified (init stage)
Targets []string `gorm:"type:json"`
// Must be the location where the file is downloadable on the API
// can be none when there is no media specified (init stage)
Location string
Timestamps
}
type Key struct {
ID int `gorm:"primaryKey;not null;"`
MetaName string
KeyName string `gorm:"not null;"`
// We don't store the key itself, we hash the key
KeyHash string `gorm:"not null;"`
// we're cooking without pepper
KeySalt string `gorm:"not null;"`
CreatedAt time.Time `gorm:"not null;"`
Timestamps
}
type Device struct {
ID int `gorm:"primaryKey"`
ID int `gorm:"primaryKey;not null;"`
// Device type is meant as a field where can be specified what type of device this is
// eg Raspberry Pi, PC, things like that
DeviceType string
Hostname string
RemoteAddress string
Alive bool
Compliant bool
CreatedAt time.Time
UpdatedAt time.Time
Hostname string `gorm:"not null;"`
RemoteAddress string `gorm:"not null;"`
Alive bool `gorm:"not null;"`
Compliant bool `gorm:"not null;"`
Timestamps
}
type File struct {
ID int `gorm:"primaryKey"`
ID int `gorm:"primaryKey;not null;"`
// unspecified
// video
// presentation
// internet URL
MediaType MediaType `gorm:"type:varchar(20);not null;"`
MetaName string
FileName string
FilePath string
Checksum string `gorm:"uniqueIndex"` // hex encoded sha512 checksum
CreatedAt time.Time
UpdatedAt time.Time
// the name given by the user
MetaName string
FileName string `gorm:"not null;"`
FilePath string `gorm:"not null;"`
// hex encoded sha512 checksum
Checksum string `gorm:"uniqueIndex;not null;"`
Timestamps
}
internal/database/functions.go
+62 -1
View File
@@ -1,13 +1,74 @@
package database
import (
"eden-server/internal/utility"
"fmt"
"io"
"log/slog"
"path/filepath"
"github.com/google/uuid"
"gorm.io/gorm"
)
// 0: unspecified
// 1: video
// 2: presentation
// 3: internet URL
func CategorizeMediaType(ext string) (MediaType, bool) {
switch ext {
case ".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4a":
return Video, true
case ".pptx", ".ppt", ".key", ".odp":
return Presentation, true
default:
slog.Debug("marking file as invalid undefined extension")
return "", false
}
}
func GenerateSafeName(category MediaType, ext string) string {
return uuid.New().String() + "_" + string(category) + ext
}
// it has been made more general for DRY purposes
// this function should only be called after manually checking the filetype
func BuildFileRecord(r io.Reader, origName string, contentDirectory string) (File, error) {
ext := filepath.Ext(origName)
category, ok := CategorizeMediaType(ext)
if !ok {
return File{}, fmt.Errorf("unsupported filetype")
}
checksum, err := utility.HashReader(r)
if err != nil {
slog.Error("failed to calculate hash of file at given path", "error", err)
return File{}, err
}
safeName := GenerateSafeName(category, ext)
destPath := filepath.Join(contentDirectory, safeName)
fData := File{
MediaType: category,
MetaName: origName,
FileName: safeName,
FilePath: destPath,
Checksum: checksum,
}
return fData, nil
}
func GetState(db *gorm.DB) (State, error) {
var state State
return state, db.First(&state).Error
if err := db.First(&state).Error; err != nil {
return State{}, err
}
return state, nil
}
func GetFiles(db *gorm.DB) ([]File, error) {
internal/database/watchdog.go
+82 -25
View File
@@ -2,6 +2,8 @@ package database
import (
"eden-server/internal/utility"
"errors"
"fmt"
"log/slog"
"os"
"path/filepath"
@@ -9,19 +11,11 @@ import (
"gorm.io/gorm"
)
func watchdog(env utility.Environment, db *gorm.DB) {
slog.Info("performing the watchdog cycle")
func filesystemGather(env utility.Environment) (map[string]struct{}, error) {
fsFiles, err := os.ReadDir(env.ContentDirectory)
if err != nil {
slog.Error("failed to read the content directory on the filesystem", "error", err)
return
}
dbFiles, err := GetFiles(db)
if err != nil {
slog.Error("failed to retrieve the files indexed from the database", "error", err)
return
return nil, err
}
// generate a set of filesystem contents
@@ -32,41 +26,104 @@ func watchdog(env utility.Environment, db *gorm.DB) {
fsSet[fullPath] = struct{}{}
}
return fsSet, nil
}
func databaseGather(db *gorm.DB) (map[string]File, error) {
dbFiles, err := GetFiles(db)
if err != nil {
slog.Error("failed to retrieve the files indexed from the database", "error", err)
return nil, err
}
// generate a set of Database contents
dbSet := make(map[string]File) // cool name for the files that are going to be deregistered from the database
for _, f := range dbFiles {
dbSet[f.FilePath] = f
}
return dbSet, nil
}
func watchdog(env utility.Environment, db *gorm.DB) {
slog.Info("performing the watchdog cycle")
fsSet, err := filesystemGather(env)
if err != nil {
return
}
dbSet, err := databaseGather(db)
if err != nil {
return
}
// FS -> DB
// check for orphaned filesystem files
var fsPurgeScroll []string
var fsOrphans []string
for path := range fsSet {
if _, exists := dbSet[path]; !exists {
fsPurgeScroll = append(fsPurgeScroll, path)
fsOrphans = append(fsOrphans, path)
}
}
if len(fsOrphans) > 0 {
slog.Info("filesystem orphans detected, engaging flow")
//filepath is stored in the slice, the filename or file object, see above
for _, fp := range fsOrphans {
// this switch is guarded by the environment.go its check making sure its one of the three
// the following logic is used to actually perform the sync modes, removal, enrollment or ignore
switch env.WatchdogSyncMode {
case "sync":
readerStream, err := os.Open(fp)
if err != nil {
}
defer readerStream.Close()
fileData, err := BuildFileRecord(readerStream, filepath.Base(fp), env.ContentDirectory)
if err != nil {
slog.Error("failed to enroll local file into the database", "error", err)
}
fmt.Println(fileData)
if err := RegisterFile(db, fileData); err != nil {
if errors.Is(err, gorm.ErrDuplicatedKey) {
slog.Debug("discarding file since its a duplicate", "error", err)
} else {
slog.Error("failed to insert filedata to the database", "error", err)
}
}
if err := os.Rename(fp, fileData.FilePath); err != nil {
slog.Error("failed to move the locally inserted file", "error", err)
}
case "strict":
err := utility.RemoveFile(fp)
if err != nil {
slog.Warn("failed to remove local file from the filesystem", "error", err)
}
case "dry":
slog.Debug("dry mode enabled, not purging", "filepath", fp)
}
}
}
// DB -> FS
// check stale database records
var dbPurgeScroll []File
var dbdbOrphans []File
for path, f := range dbSet {
if _, exists := fsSet[path]; !exists {
dbPurgeScroll = append(dbPurgeScroll, f)
dbdbOrphans = append(dbdbOrphans, f)
}
}
if len(fsPurgeScroll) > 0 {
slog.Info("filesystem purge scroll is populated, engaging purge")
//filepath is stored in the slice, the filename or file object, see above
for _, fp := range fsPurgeScroll {
utility.RemoveFile(fp)
}
}
if len(dbPurgeScroll) > 0 {
slog.Info("database purge scroll is populated, engaging purge")
for _, f := range dbPurgeScroll {
if len(dbdbOrphans) > 0 {
slog.Info("database orphans detected, engaging flow")
for _, f := range dbdbOrphans {
DeregisterFile(db, f)
}
}