Implementing checking the provider upon startup

edmondop · Oct 21, 2023 · 2b30909 · 2b30909
1 parent d0e6e63
commit 2b30909
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 0 deletions.
diff --git a/arroyo-compiler-service/src/main.rs b/arroyo-compiler-service/src/main.rs
@@ -42,6 +42,7 @@ pub async fn main() {
         .await
         .expect("unable to construct storage provider");
 
+    storage.ensure_valid()?;
     let last_used = Arc::new(AtomicU64::new(to_millis(SystemTime::now())));
 
     let service = CompileService {

diff --git a/arroyo-storage/src/lib.rs b/arroyo-storage/src/lib.rs
@@ -282,6 +282,20 @@ impl StorageProvider {
         Self::get_url_with_options(url, HashMap::new()).await
     }
 
+    pub async fn ensure_valid(storage: &Self) -> Result<(), StorageError> {
+        let url = match &storage.config {
+            BackendConfig::S3(_cfg) => "s3://non-existent-bucket/fake_dir",
+            BackendConfig::GCS(_cfg) => "gs://non-existent-bucket/fake_dir",
+            BackendConfig::Local(_cfg) => "/usr/bin/non-existing-file",
+        };
+        let result = storage.get(url).await;
+        match result {
+            Err(StorageError::ObjectStore(object_store::Error::NotFound {path:_, source:_ })) => Ok(()),
+            Err(err) => Err(err),
+            other => other.map(|_x| ())
+        }
+    }
+
     pub async fn get_url_with_options(
         url: &str,
         options: HashMap<String, String>,