From aec7ef024a917a1948c46a674279cea224f0ecdc Mon Sep 17 00:00:00 2001 From: Yeuoly Date: Fri, 8 Nov 2024 19:31:32 +0800 Subject: [PATCH] fix: incorrect persistence storage path --- .env.example | 2 +- .gitignore | 1 + internal/core/persistence/init.go | 2 +- internal/core/persistence/local.go | 55 ------------------- internal/core/persistence/persistence_test.go | 12 ++-- .../core/persistence/{s3.go => wrapper.go} | 10 ++-- internal/types/app/config.go | 4 +- internal/types/app/default.go | 11 ++-- 8 files changed, 23 insertions(+), 74 deletions(-) delete mode 100644 internal/core/persistence/local.go rename internal/core/persistence/{s3.go => wrapper.go} (78%) diff --git a/.env.example b/.env.example index 0c89aea..b1a1ebf 100644 --- a/.env.example +++ b/.env.example @@ -28,7 +28,7 @@ PLUGIN_WORKING_PATH=./cwd # persistence storage PERSISTENCE_STORAGE_TYPE=local -PERSISTENCE_STORAGE_LOCAL_PATH=./persistence +PERSISTENCE_STORAGE_PATH=./persistence PERSISTENCE_STORAGE_MAX_SIZE=104857600 # plugin webhook diff --git a/.gitignore b/.gitignore index 13b3c0d..23ffb2e 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ __pycache__ media-cache subprocesses working +cwd/ \ No newline at end of file diff --git a/internal/core/persistence/init.go b/internal/core/persistence/init.go index f7f9723..560ad0d 100644 --- a/internal/core/persistence/init.go +++ b/internal/core/persistence/init.go @@ -12,7 +12,7 @@ var ( func InitPersistence(oss oss.OSS, config *app.Config) { persistence = &Persistence{ - storage: NewWrapper(oss), + storage: NewWrapper(oss, config.PersistenceStoragePath), max_storage_size: config.PersistenceStorageMaxSize, } diff --git a/internal/core/persistence/local.go b/internal/core/persistence/local.go deleted file mode 100644 index 0d32d99..0000000 --- a/internal/core/persistence/local.go +++ /dev/null @@ -1,55 +0,0 @@ -package persistence - -import ( - "os" - "path" -) - -type LocalWrapper struct { - path string -} - -func NewLocalWrapper(path string) *LocalWrapper { - // check if the path exists, create it if not - if _, err := os.Stat(path); os.IsNotExist(err) { - os.MkdirAll(path, 0755) - } - - return &LocalWrapper{ - path: path, - } -} - -func (l *LocalWrapper) getFilePath(tenant_id string, plugin_checksum string, key string) string { - return path.Join(l.path, tenant_id, plugin_checksum, key) -} - -func (l *LocalWrapper) Save(tenant_id string, plugin_checksum string, key string, data []byte) error { - // create the directory if it doesn't exist - dir := l.getFilePath(tenant_id, plugin_checksum, "") - if err := os.MkdirAll(dir, 0755); err != nil { - return err - } - - file_path := l.getFilePath(tenant_id, plugin_checksum, key) - return os.WriteFile(file_path, data, 0644) -} - -func (l *LocalWrapper) Load(tenant_id string, plugin_checksum string, key string) ([]byte, error) { - file_path := l.getFilePath(tenant_id, plugin_checksum, key) - return os.ReadFile(file_path) -} - -func (l *LocalWrapper) Delete(tenant_id string, plugin_checksum string, key string) error { - file_path := l.getFilePath(tenant_id, plugin_checksum, key) - return os.Remove(file_path) -} - -func (l *LocalWrapper) StateSize(tenant_id string, plugin_checksum string, key string) (int64, error) { - file_path := l.getFilePath(tenant_id, plugin_checksum, key) - info, err := os.Stat(file_path) - if err != nil { - return 0, err - } - return info.Size(), nil -} diff --git a/internal/core/persistence/persistence_test.go b/internal/core/persistence/persistence_test.go index 16aa9aa..bcf0a0c 100644 --- a/internal/core/persistence/persistence_test.go +++ b/internal/core/persistence/persistence_test.go @@ -31,8 +31,8 @@ func TestPersistenceStoreAndLoad(t *testing.T) { oss := local.NewLocalStorage("./storage") InitPersistence(oss, &app.Config{ - PersistenceStorageLocalPath: "./persistence_storage", - PersistenceStorageMaxSize: 1024 * 1024 * 1024, + PersistenceStoragePath: "./persistence_storage", + PersistenceStorageMaxSize: 1024 * 1024 * 1024, }) key := strings.RandomString(10) @@ -88,8 +88,8 @@ func TestPersistenceSaveAndLoadWithLongKey(t *testing.T) { defer db.Close() InitPersistence(local.NewLocalStorage("./storage"), &app.Config{ - PersistenceStorageLocalPath: "./persistence_storage", - PersistenceStorageMaxSize: 1024 * 1024 * 1024, + PersistenceStoragePath: "./persistence_storage", + PersistenceStorageMaxSize: 1024 * 1024 * 1024, }) key := strings.RandomString(65) @@ -118,8 +118,8 @@ func TestPersistenceDelete(t *testing.T) { oss := local.NewLocalStorage("./storage") InitPersistence(oss, &app.Config{ - PersistenceStorageLocalPath: "./persistence_storage", - PersistenceStorageMaxSize: 1024 * 1024 * 1024, + PersistenceStoragePath: "./persistence_storage", + PersistenceStorageMaxSize: 1024 * 1024 * 1024, }) key := strings.RandomString(10) diff --git a/internal/core/persistence/s3.go b/internal/core/persistence/wrapper.go similarity index 78% rename from internal/core/persistence/s3.go rename to internal/core/persistence/wrapper.go index 725d3f9..f25c50c 100644 --- a/internal/core/persistence/s3.go +++ b/internal/core/persistence/wrapper.go @@ -7,17 +7,19 @@ import ( ) type wrapper struct { - oss oss.OSS + oss oss.OSS + persistence_storage_path string } -func NewWrapper(oss oss.OSS) *wrapper { +func NewWrapper(oss oss.OSS, persistence_storage_path string) *wrapper { return &wrapper{ - oss: oss, + oss: oss, + persistence_storage_path: persistence_storage_path, } } func (s *wrapper) getFilePath(tenant_id string, plugin_checksum string, key string) string { - return path.Join(tenant_id, plugin_checksum, key) + return path.Join(s.persistence_storage_path, tenant_id, plugin_checksum, key) } func (s *wrapper) Save(tenant_id string, plugin_checksum string, key string, data []byte) error { diff --git a/internal/types/app/config.go b/internal/types/app/config.go index a037457..fdc463e 100644 --- a/internal/types/app/config.go +++ b/internal/types/app/config.go @@ -63,8 +63,8 @@ type Config struct { DBSslMode string `envconfig:"DB_SSL_MODE" validate:"required,oneof=disable require"` // persistence storage - PersistenceStorageLocalPath string `envconfig:"PERSISTENCE_STORAGE_LOCAL_PATH"` - PersistenceStorageMaxSize int64 `envconfig:"PERSISTENCE_STORAGE_MAX_SIZE"` + PersistenceStoragePath string `envconfig:"PERSISTENCE_STORAGE_PATH"` + PersistenceStorageMaxSize int64 `envconfig:"PERSISTENCE_STORAGE_MAX_SIZE"` // force verifying signature for all plugins, not allowing install plugin not signed ForceVerifyingSignature bool `envconfig:"FORCE_VERIFYING_SIGNATURE"` diff --git a/internal/types/app/default.go b/internal/types/app/default.go index 1629622..cb15552 100644 --- a/internal/types/app/default.go +++ b/internal/types/app/default.go @@ -20,12 +20,13 @@ func (config *Config) SetDefault() { setDefaultBool(&config.PluginRemoteInstallingEnabled, true) setDefaultBool(&config.PluginEndpointEnabled, true) setDefaultString(&config.DBSslMode, "disable") - setDefaultString(&config.PluginInstalledPath, "./storage/plugin") - setDefaultString(&config.PluginMediaCachePath, "./storage/assets") - setDefaultString(&config.PersistenceStorageLocalPath, "./storage/persistence") + setDefaultString(&config.PluginStorageLocalRoot, "./storage") + setDefaultString(&config.PluginInstalledPath, "./plugin") + setDefaultString(&config.PluginMediaCachePath, "./assets") + setDefaultString(&config.PersistenceStoragePath, "./persistence") setDefaultInt(&config.PersistenceStorageMaxSize, 100*1024*1024) - setDefaultString(&config.ProcessCachingPath, "./storage/subprocesses") - setDefaultString(&config.PluginPackageCachePath, "./storage/plugin_packages") + setDefaultString(&config.ProcessCachingPath, "./subprocesses") + setDefaultString(&config.PluginPackageCachePath, "./plugin_packages") setDefaultString(&config.PythonInterpreterPath, "/usr/bin/python3") }