GH-55 # Clean redis db prior to tranco initialisation

Cargo.toml

@@ -18,7 +18,7 @@ rocket-sentry = "0.17"
 redis = "0.25"
 log = "0.4"
 env_logger = "0.11"
-async-std = { version = "1.12", features = ["attributes"] }
+async-std = { version = "1.12", features = ["attributes", "tokio1"] }
 
 [dependencies.rocket_db_pools]
 version = "0.1.0"

src/tranco_top1m/mod.rs

@@ -2,7 +2,7 @@ use std::env;
 use std::error::Error;
 use csv::ReaderBuilder;
 use rocket_db_pools::Connection as RocketConnection;
-use rocket_db_pools::deadpool_redis::redis::AsyncCommands;
+use rocket_db_pools::deadpool_redis::redis::{AsyncCommands, RedisError};
 use serde::Serialize;
 
 use crate::Db;
@@ -73,11 +73,25 @@ impl TrancoTop1M {
             // TODO download
             "TODO".to_string()
         };
-        // TODO clear existing key
+        self.destroy_db().await.unwrap();
         self.add_domains_from_path(path).await.expect("tranco values can be added");
         let _: () = self.redis_client.set(TRANCO_TOP_1M_INITIALIZED_KEY, 1).await.unwrap();
     }
 
+    /// Remove all keys related to Tranco in the Redis DB
+    pub async fn destroy_db(&mut self) -> Result<(), RedisError> {
+        // First remove the init key so the other keys are not used in a partial state
+        let _ : () = self.redis_client.del(TRANCO_TOP_1M_INITIALIZED_KEY).await?;
+
+        // Then we remove all the jarm hash keys
+        let pattern = format!("{TRANCO_TOP_1M_JARM_PREFIX_KEY}*");
+        let keys: Vec<String> = self.redis_client.keys(pattern).await?;
+        for key in keys {
+            let _ : () = self.redis_client.del(key).await?;
+        }
+        Ok(())
+    }
+
     async fn add_domains_from_path(&mut self, path: String) -> Result<(), Box<dyn Error>> {
         let mut reader = ReaderBuilder::new().has_headers(false).from_path(path)?;
         for result in reader.records() {

tests/common.rs

@@ -11,6 +11,8 @@ use rocket::local::blocking::Client;
 #[allow(dead_code)]  // used in tests
 pub const DUMMY_SERVER_JARM_HASH: &str = "21d19d00021d21d00021d19d21d21d1a46380b04d662f0848f508dd171125d";
 
+pub const REDIS_URL: &str = "redis://127.0.0.1/";
+
 
 lazy_static! {
     static ref REDIS_MUTEX: Mutex<()> = Mutex::default();  // restrict redis parallel access
@@ -52,7 +54,7 @@ pub fn clean_redis<'a>() -> MutexGuard<'a, ()> {
         REDIS_MUTEX.clear_poison();
         e.into_inner()  // Prevent a failing test to fail the tests that follow
     });
-    let client = redis::Client::open("redis://127.0.0.1/").unwrap();
+    let client = redis::Client::open(REDIS_URL).unwrap();
     for i in 1..=10 {
         match clean_redis_commands(&client)   {
             Ok(_) => break,

tests/test_route_tranco_overlap.rs

@@ -10,9 +10,12 @@ mod test_route_tranco_overlap {
 
     use rocket::http;
     use rocket::local::blocking::Client;
+    use rocket_db_pools::deadpool_redis::{Config};
     use rstest::*;
+    use jarm_online::tranco_top1m::TrancoTop1M;
+    use jarm_online::tranco_top1m::RankedDomain;
 
-    use crate::common::{rocket_client, rocket_client_without_tranco_init};
+    use crate::common::{REDIS_URL, rocket_client, rocket_client_without_tranco_init};
     use crate::common::clean_redis;
 
     #[rstest]
@@ -81,4 +84,24 @@ mod test_route_tranco_overlap {
         assert_eq!(response.status(), http::Status::Ok);
         assert_eq!(response.into_string(), Some(expected_response.into()));
     }
+
+    #[rstest]
+    #[ignore = "Integration tests"]
+    async fn clear_tranco_values(_clean_redis: MutexGuard<'_, ()>, _rocket_client: Client) {
+        let cfg = Config::from_url(REDIS_URL);
+        let pool = cfg.create_pool(None).unwrap();
+        let connection = pool.get().await.unwrap();
+        let mut tranco = TrancoTop1M::new(connection);
+
+        while tranco.is_initialized().await != true { // Wait for redis to be initialized
+            task::sleep(Duration::from_millis(10)).await;
+        }
+        let jarm_hash = "3fd3fd20d3fd3fd21c3fd3fd3fd3fd2b66a312d81ed1efa0f55830f7490cb2";
+        assert_eq!(tranco.get(jarm_hash.to_string()).await, vec![RankedDomain { rank: 9, domain: "zhihu.com".to_string() }]);
+
+        tranco.destroy_db().await.unwrap();  // Clearing values in db means no hash will be found
+
+        assert_eq!(tranco.get(jarm_hash.to_string()).await, vec![]);
+        assert_eq!(tranco.is_initialized().await, false);
+    }
 }