From 27d6ef67ee7cf9be539e5d71398ad5318613cc9a Mon Sep 17 00:00:00 2001 From: Luis Moreno Date: Fri, 13 Dec 2024 13:29:44 -0400 Subject: [PATCH 1/5] chore: update for beta5 --- .github/workflows/ci.yml | 2 +- dataflows/mask-user-pii/package-variant/Makefile | 2 +- dataflows/mask-user-pii/package-variant/README.MD | 3 +-- dataflows/split-sentence/package-variant/Makefile | 2 +- dataflows/split-sentence/package-variant/README.MD | 3 +-- 5 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ac14d78..40eac58 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: checks: uses: ./.github/workflows/check.yml with: - sdf_version: sdf-beta4 + sdf_version: sdf-beta5-dev branch: ${{ github.ref }} done: diff --git a/dataflows/mask-user-pii/package-variant/Makefile b/dataflows/mask-user-pii/package-variant/Makefile index e6b8679..fa0d33e 100644 --- a/dataflows/mask-user-pii/package-variant/Makefile +++ b/dataflows/mask-user-pii/package-variant/Makefile @@ -3,4 +3,4 @@ include $(BASE_DIR)/Makefile e2e: make -C $(BASE_DIR)/packages/mask-ssn build - $(SDF_BIN) run --skip-running --dev \ No newline at end of file + $(SDF_BIN) run --skip-running \ No newline at end of file diff --git a/dataflows/mask-user-pii/package-variant/README.MD b/dataflows/mask-user-pii/package-variant/README.MD index 4239877..959d117 100644 --- a/dataflows/mask-user-pii/package-variant/README.MD +++ b/dataflows/mask-user-pii/package-variant/README.MD @@ -26,11 +26,10 @@ Use the following command to compile the package: Use `sdf` command line tool to run the dataflow: ```bash -sdf run --dev --ui +sdf run --ui ``` Where: - * `--dev` - runs the dataflow in development mode (reads the package from local file system). * `--ui` - generates the graphical representation and run the Studio web server. diff --git a/dataflows/split-sentence/package-variant/Makefile b/dataflows/split-sentence/package-variant/Makefile index b99c71e..d118ecb 100644 --- a/dataflows/split-sentence/package-variant/Makefile +++ b/dataflows/split-sentence/package-variant/Makefile @@ -3,4 +3,4 @@ include $(BASE_DIR)/Makefile e2e: make -C $(BASE_DIR)/packages/parse-sentence build - $(SDF_BIN) run --skip-running --dev + $(SDF_BIN) run --skip-running diff --git a/dataflows/split-sentence/package-variant/README.MD b/dataflows/split-sentence/package-variant/README.MD index d79fa60..55cfc0e 100644 --- a/dataflows/split-sentence/package-variant/README.MD +++ b/dataflows/split-sentence/package-variant/README.MD @@ -26,11 +26,10 @@ Use the following command to compile the package: Use `sdf` command line tool to run the dataflow: ```bash -sdf run --dev --ui +sdf run --ui ``` Where: - * `--dev` - runs the dataflow in development mode (reads the package from local file system). * `--ui` - generates the graphical representation and run the Studio web server. From dc363932e5614e085d5f2f9889dfe6c71c46400f Mon Sep 17 00:00:00 2001 From: Luis Moreno Date: Fri, 13 Dec 2024 13:37:27 -0400 Subject: [PATCH 2/5] chore: use sql function --- dataflows/bank-processing/dataflow.yaml | 4 ---- dataflows/car-processing/dataflow.yaml | 3 +-- dataflows/hackernews-notify/dataflow.yaml | 7 +++---- dataflows/helsinki-transit/dataflow.yaml | 3 +-- dataflows/word-counter/dataflow.yaml | 3 +-- dataflows/word-probe/dataflow.yaml | 3 +-- 6 files changed, 7 insertions(+), 16 deletions(-) diff --git a/dataflows/bank-processing/dataflow.yaml b/dataflows/bank-processing/dataflow.yaml index 7cae455..3cfacfe 100644 --- a/dataflows/bank-processing/dataflow.yaml +++ b/dataflows/bank-processing/dataflow.yaml @@ -407,16 +407,12 @@ services: - operator: filter-map run: | fn generate_overdraft_event_due_transfer(ev: DataEvent) -> Result> { - let withdrawal = match ev.operation { DataOperation::CreditAccount(_) => return Ok(None), DataOperation::DebitAccount(debit) => debit, }; - let accounts = account_balance(); - let account = accounts.sql(&format!("select * from `account-balance` where name = '{}'", ev.name))?; - let rows = account.rows()?; if !rows.next() { diff --git a/dataflows/car-processing/dataflow.yaml b/dataflows/car-processing/dataflow.yaml index 7cbcdbe..cf18014 100644 --- a/dataflows/car-processing/dataflow.yaml +++ b/dataflows/car-processing/dataflow.yaml @@ -315,8 +315,7 @@ services: - operator: filter-map run: | fn check_license_plate(car: Car) -> Result> { - let plates = licence_plates(); - let lp = plates.sql(&format!("select * from `licence-plates` where _key = '{}'", car.license))?; + let lp = sql(&format!("select * from licence_plates where _key = '{}'", car.license))?; let rows = lp.rows()?; let maker_col = lp.col("maker")?; diff --git a/dataflows/hackernews-notify/dataflow.yaml b/dataflows/hackernews-notify/dataflow.yaml index a1ab889..c2e970a 100644 --- a/dataflows/hackernews-notify/dataflow.yaml +++ b/dataflows/hackernews-notify/dataflow.yaml @@ -141,12 +141,11 @@ services: run: | fn match_words(article_data: ArticleData) -> Result, ArticleData)>> { use format_sql_query::{Table, Column, QuotedData}; - let table = notify_table(); let query = format!("SELECT * FROM `{}` WHERE {} = {}", - Table("notify-table".into()), + Table("notify_table".into()), Column("_key".into()), QuotedData(&article_data.word.to_lowercase()) ); - let emails = match table.sql(&query) { + let emails = match sql(&query) { Ok(emails) => emails, Err(e) => { println!("Error: {}", e); @@ -184,4 +183,4 @@ services: sinks: - type: topic - id: notify-event \ No newline at end of file + id: notify-event diff --git a/dataflows/helsinki-transit/dataflow.yaml b/dataflows/helsinki-transit/dataflow.yaml index 617aa8a..df6df49 100644 --- a/dataflows/helsinki-transit/dataflow.yaml +++ b/dataflows/helsinki-transit/dataflow.yaml @@ -145,8 +145,7 @@ services: flush: run: | fn compute_top_vehicle() -> Result { - let mut stat = vehicle_stat(); - let top5 = stat.sql("select * from vehicle_stat order by speed desc limit 5")?; + let top5 = sql("select * from vehicle_stat order by speed desc limit 5")?; let rows = top5.rows()?; let mut top_vehicles = vec![]; let key = top5.key()?; diff --git a/dataflows/word-counter/dataflow.yaml b/dataflows/word-counter/dataflow.yaml index e8c0f02..495a0f1 100644 --- a/dataflows/word-counter/dataflow.yaml +++ b/dataflows/word-counter/dataflow.yaml @@ -111,9 +111,8 @@ services: # Read the full state and compute the top 3 words sorted by count. run: | fn compute_most_used_words() -> Result { - let word_counts = count_per_word(); - let top3 = word_counts.sql("select * from count_per_word order by count desc limit 3")?; + let top3 = sql("select * from count_per_word order by count desc limit 3")?; let rows = top3.rows()?; let columns = top3.schema(["_key","count"])?; diff --git a/dataflows/word-probe/dataflow.yaml b/dataflows/word-probe/dataflow.yaml index cee4bcc..ac5f061 100644 --- a/dataflows/word-probe/dataflow.yaml +++ b/dataflows/word-probe/dataflow.yaml @@ -84,8 +84,7 @@ services: - operator: map run: | fn query_word_count(word: String) -> Result { - let df = count_per_word(); - let count = df.sql(&format!("select * from `count-per-word` where _key = '{}'", word))?; + let count = sql(&format!("select * from count_per_word where _key = '{}'", word))?; let rows = count.rows()?; let columns = count.schema(["_key","count"])?; match &columns[..] { From 93c1cbe312d5d1e9f4e7354921ae3b8fde13eca6 Mon Sep 17 00:00:00 2001 From: Luis Moreno Date: Fri, 13 Dec 2024 17:42:34 -0400 Subject: [PATCH 3/5] chore: update example --- dataflows/bank-processing/dataflow.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/dataflows/bank-processing/dataflow.yaml b/dataflows/bank-processing/dataflow.yaml index 3cfacfe..9e3a95d 100644 --- a/dataflows/bank-processing/dataflow.yaml +++ b/dataflows/bank-processing/dataflow.yaml @@ -275,11 +275,10 @@ services: DataOperation::DebitAccount(debit) => debit, }; - let accounts = account_balance(); - let account = accounts.sql(&format!("select * from `account-balance` where name = '{}'", ev.name))?; + let account = sql(&format!("select * from `account-balance` where name = '{}'", ev.name))?; - let rows = accounts.rows()?; + let rows = account.rows()?; if !rows.next() { return Ok(None) @@ -411,8 +410,7 @@ services: DataOperation::CreditAccount(_) => return Ok(None), DataOperation::DebitAccount(debit) => debit, }; - let accounts = account_balance(); - let account = accounts.sql(&format!("select * from `account-balance` where name = '{}'", ev.name))?; + let account = sql(&format!("select * from `account-balance` where name = '{}'", ev.name))?; let rows = account.rows()?; if !rows.next() { From 138d8d936aa0dc866dd30a690cae5259e852a7fd Mon Sep 17 00:00:00 2001 From: morenol <22335041+morenol@users.noreply.github.com> Date: Mon, 16 Dec 2024 11:46:26 -0400 Subject: [PATCH 4/5] Update .github/workflows/ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 40eac58..573e909 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: checks: uses: ./.github/workflows/check.yml with: - sdf_version: sdf-beta5-dev + sdf_version: sdf-beta5 branch: ${{ github.ref }} done: From 88965cf6c24ca71b4ce6a5ca13b6669ea1b56d59 Mon Sep 17 00:00:00 2001 From: morenol <22335041+morenol@users.noreply.github.com> Date: Mon, 16 Dec 2024 11:46:53 -0400 Subject: [PATCH 5/5] Update nightly.yml --- .github/workflows/nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index a94ce9b..eec962b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -13,7 +13,7 @@ jobs: checks: uses: ./.github/workflows/check.yml with: - sdf_version: sdf-beta5-dev + sdf_version: sdf-beta6-dev branch: ${{ github.ref }} done: name: Done