Merge pull request #61 from Hugo-C/feature/GH-60

 GH-60 # Remove Alexa related code

Cargo.toml

@@ -27,4 +27,4 @@ version = "0.1.0"
 features = ["deadpool_redis"]
 
 [dev-dependencies]
-rstest = "0.18"
+rstest = "0.19"

Dockerfile

@@ -1,4 +1,4 @@
-FROM rust:1.76-bullseye as build-stage
+FROM rust:1.77-bullseye as build-stage
 
 ARG BUILD_TARGET="x86_64-unknown-linux-musl"
 ARG BUILD_OPTIONS="--release --target $BUILD_TARGET"

README.md

@@ -40,16 +40,6 @@ GET api/v1/jarm?host=<host>&port=<port>
 
 It takes a required `host` parameter and optionally `port` (default to 443).
 
-### Retrieve domains from alexa top 1 million that match a jarm hash
-
-**DEPRECATED** see `tranco-overlap` instead 
-
-````http request
-GET api/v1/alexa-overlap?jarm_hash=<jarm-hash>
-````
-
-The returned list is ordered by top alexa rank first
-
 ### Retrieve domains from tranco top 1 million that match a jarm hash
 
 ````http request

docker-compose.playwright.yml

@@ -15,10 +15,8 @@ services:
       service: jarm_online_api
     environment:
       - ROCKET_SENTRY_DSN=123  # this way sentry is disabled but do not crash
-      - ALEXA_TOP1M_RAW_DATA_PATH=/alexa_top1M.csv
       - FORCE_TRANCO_TOP1M_RAW_DATA_PATH=/tranco_top1M.csv
     volumes:
-      - ./tests/fixtures_data/alexa_top1M.csv:/alexa_top1M.csv
       - ./tests/fixtures_data/tranco_top1M.csv:/tranco_top1M.csv
     depends_on:
       - redis

docker-compose.yml.prod

@@ -23,9 +23,7 @@ services:
       - ROCKET_DATABASES={redis_db={url="redis://redis"}}
       - ROCKET_SENTRY_DSN=${ROCKET_SENTRY_DSN}
       - ROCKET_SENTRY_TRACES_SAMPLE_RATE=1.0
-      - ALEXA_TOP1M_RAW_DATA_PATH=/alexa_top1M.csv
-    volumes:
-      - ./data/result_aggregated.csv:/alexa_top1M.csv
+      - TRANCO_TOP1M_S3_URL=https://storage.googleapis.com/tranco-jarm/jarm-tranco-top-1m.csv
     restart: unless-stopped
     depends_on:
       - redis

src/lib.rs

@@ -2,17 +2,14 @@
 extern crate rocket;
 
 pub mod utils;
-pub mod alexa_top1m;
 pub mod tranco_top1m;
 
 use rocket_db_pools::{Connection, deadpool_redis};
-use crate::alexa_top1m::{AlexaTop1M, RankedDomain};
 use crate::tranco_top1m::{TrancoTop1M};
 use crate::tranco_top1m::RankedDomain as TrancoRankedDomain;
 
 use std::env;
-use std::path::Path;
-use rocket::{Build, fairing, Rocket, State};
+use rocket::{Build, fairing, Rocket};
 use rocket::serde::json::Json;
 use rust_jarm::Jarm;
 use serde::Serialize;
@@ -67,11 +64,6 @@ struct LastScanListResponse {
     last_scans: Vec<LastScanResponse>,
 }
 
-#[derive(Serialize)]
-struct AlexaOverlapResponse {
-    overlapping_domains: Vec<RankedDomain>,
-}
-
 #[derive(Serialize)]
 struct TrancoOverlapResponse {
     overlapping_domains: Vec<TrancoRankedDomain>,
@@ -84,12 +76,6 @@ pub fn scan_timeout_in_seconds() -> u64 {
         .expect("Valid timeout value")
 }
 
-pub fn alexa_top1m_raw_data_path() -> Box<Path> {
-    let raw_path = env::var("ALEXA_TOP1M_RAW_DATA_PATH")
-        .expect("ALEXA_TOP1M_RAW_DATA_PATH env var has to be set");
-    Path::new(&raw_path).into()
-}
-
 #[get("/?<host>&<port>")]
 async fn jarm(host: String, port: Option<String>, mut redis_client: Connection<Db>) -> Json<JarmResponse> {
     let _port = port.unwrap_or_else(|| "443".to_string());
@@ -136,15 +122,6 @@ async fn last_scans(mut redis_client: Connection<Db>) -> Json<LastScanListRespon
     Json(LastScanListResponse { last_scans })
 }
 
-#[get("/?<jarm_hash>")]
-fn alexa_overlap(alexa_top1m: &State<AlexaTop1M>, jarm_hash: String) -> Json<AlexaOverlapResponse> {  // TODO try str
-    let overlap = match alexa_top1m.get(jarm_hash.as_str()) {
-        None => vec![],
-        Some(overlap) => overlap.to_vec()
-    };
-    Json(AlexaOverlapResponse { overlapping_domains: overlap })
-}
-
 #[get("/?<jarm_hash>")]
 async fn tranco_overlap(redis_client: Connection<Db>, jarm_hash: String) -> Result<Json<TrancoOverlapResponse>, Custom<Json<ErrorResponse>>> {
     let mut tranco = TrancoTop1M::from(redis_client);
@@ -177,15 +154,11 @@ fn build_error_json(jarm_error: JarmError) -> Json<JarmResponse> {
 }
 
 pub fn build_rocket_without_tranco_initialisation() -> Rocket<Build> {
-    let alexa_top1m = AlexaTop1M::new(&alexa_top1m_raw_data_path())
-        .expect("AlexaTop1M built correctly");
     rocket::build()
         .mount("/jarm", routes![jarm])
         .mount("/last-scans", routes![last_scans])
-        .mount("/alexa-overlap", routes![alexa_overlap])
         .mount("/tranco-overlap", routes![tranco_overlap])
         .attach(Db::init())
-        .manage(alexa_top1m)
 }
 
 pub fn build_rocket() -> Rocket<Build> {

tests/common.rs

@@ -18,13 +18,6 @@ lazy_static! {
     static ref REDIS_MUTEX: Mutex<()> = Mutex::default();  // restrict redis parallel access
 }
 
-
-#[fixture]
-#[once]
-pub fn alexa_top1m_path() -> &'static Path {
-    Path::new("tests/fixtures_data/alexa_top1M.csv")
-}
-
 #[fixture]
 #[once]
 pub fn tranco_top1m_path() -> &'static Path {
@@ -33,22 +26,20 @@ pub fn tranco_top1m_path() -> &'static Path {
 
 #[fixture]
 #[once]
-pub fn set_env_var_top1m_path(alexa_top1m_path: &'static Path, tranco_top1m_path: &'static Path) -> &'static Path {
-    env::set_var("ALEXA_TOP1M_RAW_DATA_PATH", alexa_top1m_path.to_str().expect("valid path"));
+pub fn set_env_var_top1m_path(tranco_top1m_path: &'static Path) {
     env::set_var("FORCE_TRANCO_TOP1M_RAW_DATA_PATH", tranco_top1m_path.to_str().expect("valid path"));
-    alexa_top1m_path
 }
 
 #[fixture]
 #[allow(unused_variables)]
-pub fn rocket_client(set_env_var_top1m_path: &'static Path) -> Client {
+pub fn rocket_client(set_env_var_top1m_path: ()) -> Client {
     let test_rocket = build_rocket();
     Client::tracked(test_rocket).expect("valid rocket instance")
 }
 
 #[fixture]
 #[allow(unused_variables)]
-pub fn rocket_client_without_tranco_init(set_env_var_top1m_path: &'static Path) -> Client {
+pub fn rocket_client_without_tranco_init(set_env_var_top1m_path: ()) -> Client {
     let test_rocket = build_rocket_without_tranco_initialisation();
     Client::tracked(test_rocket).expect("valid rocket instance")
 }