pg classes fixed

sql/Classes/up.sql

@@ -1,6 +1,7 @@
 CREATE TYPE status_enum AS ENUM ('Open', 'Closed', 'Full', 'On Hold');
 CREATE TABLE Classes (
     "class_id"                        VARCHAR(255) PRIMARY KEY,
+    -- "career"                          VARCHAR(255),
     "course_id"                       VARCHAR(8)   NOT NULL,
     "section"                         VARCHAR(255) NOT NULL,
     "term"                            VARCHAR(50)  NOT NULL,

sql/Courses/up.sql

@@ -1,5 +1,5 @@
 CREATE TABLE Courses (
-    "course_code"                     VARCHAR(8) PRIMARY KEY, --id
+    "course_code"                     VARCHAR(8), --id
     "course_name"                     VARCHAR(255) NOT NULL,
     "uoc"                             INT NOT NULL,
     "faculty"                         VARCHAR(255),
@@ -8,4 +8,5 @@ CREATE TABLE Courses (
     "career"                          VARCHAR(255),
     "terms"                           TEXT,
     "modes"                           VARCHAR(255)[]
+    PRIMARY KEY ("course_code", "career")
 );

src/class_scraper.rs

@@ -24,6 +24,7 @@ pub struct Course {
 #[derive(Debug)]
 pub struct Class {
     pub course_id: String,
+    pub career: String,
     pub class_id: String,
     pub section: String,
     pub term: String,
@@ -42,6 +43,7 @@ pub struct Class {
 
 #[derive(Debug)]
 pub struct Time {
+    pub career: String,
     pub day: String,
     pub time: String,
     pub location: String,
@@ -61,7 +63,7 @@ pub struct ClassScraper {
 
 impl ClassScraper {
     pub async fn scrape(&mut self) -> Result<Course, Box<ScrapeError>> {
-        // println!("Currently working on {:?}", self.course_code);
+        println!("Currently working on {:?}", self.course_code);
         let html = fetch_url(&self.url)
             .await
             .expect(&format!("Something was wrong with the URL: {}", self.url));
@@ -155,20 +157,17 @@ impl ClassScraper {
 
         course_info.classes = class_activity_information
             .into_par_iter()
-            .map(|class_data| parse_class_info(class_data, self.course_code.clone()))
+            .map(|class_data| parse_class_info(class_data, self.course_code.clone(), self.career.clone()))
             .collect();
         let _ = course_info
             .classes
             .iter_mut()
             .map(|c| course_info.modes.insert(c.mode.to_string()))
             .collect::<Vec<_>>();
-        if course_info.course_code == "COMP6441" {
-            println!("TESTTEST {:?}", course_info.career);
-        }
         Ok(course_info)
     }
 }
-fn parse_class_info(class_data: Vec<String>, course_id: String) -> Class {
+fn parse_class_info(class_data: Vec<String>, course_id: String, career: String) -> Class {
     let mut map = HashMap::new();
     let mut i = 0;
     let mut times_parsed = Vec::<Time>::new();
@@ -180,7 +179,7 @@ fn parse_class_info(class_data: Vec<String>, course_id: String) -> Class {
             while j < class_data.len() && class_data[j] != "Class Notes" {
                 j += 1;
             }
-            times_parsed = parse_meeting_info(&class_data[i + 1..j]);
+            times_parsed = parse_meeting_info(&class_data[i + 1..j], career.clone());
             i = j + 1;
             continue;
         }
@@ -249,6 +248,7 @@ fn parse_class_info(class_data: Vec<String>, course_id: String) -> Class {
             .unwrap_or(&"".to_string())
             .to_string(),
         consent: map.get("Consent").unwrap_or(&"".to_string()).to_string(),
+        career,
         times: if times_parsed.is_empty() {
             None
         } else {
@@ -261,7 +261,7 @@ fn parse_class_info(class_data: Vec<String>, course_id: String) -> Class {
     }
 }
 
-fn parse_meeting_info(vec: &[String]) -> Vec<Time> {
+fn parse_meeting_info(vec: &[String], career: String) -> Vec<Time> {
     let days = vec!["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"];
     let mut meetings = Vec::new();
     let mut iter: Box<dyn Iterator<Item = &String>> = Box::new(vec.iter());
@@ -290,7 +290,7 @@ fn parse_meeting_info(vec: &[String]) -> Vec<Time> {
                     iter = Box::new(std::iter::once(instructor).chain(iter));
                 }
             }
-
+            timeslot.career = career.clone();
             meetings.push(timeslot);
         }
     }
@@ -300,6 +300,7 @@ fn parse_meeting_info(vec: &[String]) -> Vec<Time> {
 
 fn get_blank_time_struct() -> Time {
     Time {
+        career: "".to_string(),
         day: "".to_string(),
         time: "".to_string(),
         location: "".to_string(),

src/hasuragres_b_insert.rs

@@ -80,6 +80,7 @@ pub async fn send_batch_data(hdata: &impl HasuragresData) -> Result<(), Box<dyn
     let hasuragres_url = env::var("HASURAGRES_URL")?;
     let api_key = env::var("HASURAGRES_API_KEY")?;
     let client = Client::new();
+    println!("{:?} {:?}", hasuragres_url, api_key);
     println!("Starting to insert into Hasuragres!");
     let requests = vec![
         BatchInsertRequest {
@@ -110,6 +111,7 @@ pub async fn send_batch_data(hdata: &impl HasuragresData) -> Result<(), Box<dyn
                 table_name: "classes".to_string(),
                 columns: vec![
                     "class_id".to_string(),
+                    "career".to_string(),
                     "course_id".to_string(),
                     "section".to_string(),
                     "term".to_string(),
@@ -139,6 +141,7 @@ pub async fn send_batch_data(hdata: &impl HasuragresData) -> Result<(), Box<dyn
                 columns: vec![
                     "id".to_string(),
                     "class_id".to_string(),
+                    "career".to_string(),
                     "day".to_string(),
                     "instructor".to_string(),
                     "location".to_string(),
@@ -155,6 +158,7 @@ pub async fn send_batch_data(hdata: &impl HasuragresData) -> Result<(), Box<dyn
             payload: hdata.get_times(),
         },
     ];
+
     let response = client
         .post(format!("{}/batch_insert", hasuragres_url))
         .header("X-API-Key", api_key)

src/main.rs

@@ -141,6 +141,7 @@ fn convert_classes_times_to_json(course_vec: &mut Vec<Course>) -> Vec<serde_json
                         "id": generate_time_id(class, time),
                         "class_id": class.class_id,
                         "day": time.day,
+                        "career": time.career,
                         "instructor": time.instructor,
                         "location": time.location,
                         "time": time.time,
@@ -162,6 +163,7 @@ fn convert_classes_to_json(course_vec: &mut Vec<Course>) -> Vec<serde_json::Valu
                 "class_id": class.class_id,
                 "section": class.section,
                 "term": class.term,
+                "career": class.career,
                 "year": class.year,
                 "activity": class.activity,
                 "status": class.status,
@@ -188,10 +190,6 @@ async fn handle_scrape(course_vec: &mut Vec<Course>, start_year: i32) -> Result<
             let course = run_course_classes_page_scraper_job(all_school_offered_courses_scraper).await;
             course_vec.extend(course);
         }
-        // let mut rc = ClassScraper { course_code: "COMP6420".to_string(), course_name: "Hardware Security".to_string(), career: "Undergraduate".to_string(), uoc: 6, url: "https://timetable.unsw.edu.au/2025/COMP1511.html".to_string() };
-        // rc.scrape().await;
-        // println!("{:?}", rc);
-
     }
 
     Ok(())

src/subject_area_scraper.rs

@@ -1,4 +1,4 @@
-use std::sync::Arc;
+use std::{collections::HashSet, sync::Arc};
 
 use scraper::Selector;
 use tokio::sync::Mutex;
@@ -31,13 +31,18 @@ impl SubjectAreaScraper {
                 let link_selector = Selector::parse("td.data a").unwrap();
                 let uoc_selector = Selector::parse("td.data:nth-child(3)").unwrap();
                 let document = scraper::Html::parse_document(&html);
+                let mut course_code_career_set = HashSet::<String>::new();
                 for career_elem_ref in document.select(&career_selector) {
                     let career = extract_text(career_elem_ref);
                     if career.is_empty() {continue};
                     for row_node in document.select(&row_selector) {
                         // Extract data from each row
                         let course_code = extract_text(row_node.select(&code_selector).next().unwrap());
                         let course_name = extract_text(row_node.select(&name_selector).nth(1).unwrap());
+                        let name_hash = course_code.to_string() + &career;
+                        if course_code_career_set.contains(&name_hash) {
+                            continue;
+                        }
                         let year_to_scrape = extract_year(url).unwrap(); 
                         let url_to_scrape_further = get_html_link_to_page(
                             year_to_scrape as i32, 
@@ -50,12 +55,13 @@ impl SubjectAreaScraper {
                             .parse()
                             .expect("Could not parse UOC!");
                         self.class_scrapers.push(Arc::new(Mutex::new(ClassScraper {
-                            course_code,
+                            course_code: course_code.clone(),
                             course_name,
                             career: career.trim().to_string(),
                             uoc,
                             url: url_to_scrape_further,
                         })));
+                        course_code_career_set.insert(name_hash);
                     }
                 }