diff --git a/.changes/1.4.0-b1.md b/.changes/1.4.0-b1.md new file mode 100644 index 00000000000..b2a0e96827c --- /dev/null +++ b/.changes/1.4.0-b1.md @@ -0,0 +1,89 @@ +## dbt-core 1.4.0-b1 - December 15, 2022 + +### Features + +- Added favor-state flag to optionally favor state nodes even if unselected node exists ([#2968](https://github.com/dbt-labs/dbt-core/issues/2968)) +- Update structured logging. Convert to using protobuf messages. Ensure events are enriched with node_info. ([#5610](https://github.com/dbt-labs/dbt-core/issues/5610)) +- Friendlier error messages when packages.yml is malformed ([#5486](https://github.com/dbt-labs/dbt-core/issues/5486)) +- Migrate dbt-utils current_timestamp macros into core + adapters ([#5521](https://github.com/dbt-labs/dbt-core/issues/5521)) +- Allow partitions in external tables to be supplied as a list ([#5929](https://github.com/dbt-labs/dbt-core/issues/5929)) +- extend -f flag shorthand for seed command ([#5990](https://github.com/dbt-labs/dbt-core/issues/5990)) +- This pulls the profile name from args when constructing a RuntimeConfig in lib.py, enabling the dbt-server to override the value that's in the dbt_project.yml ([#6201](https://github.com/dbt-labs/dbt-core/issues/6201)) +- Adding tarball install method for packages. Allowing package tarball to be specified via url in the packages.yaml. ([#4205](https://github.com/dbt-labs/dbt-core/issues/4205)) +- Added an md5 function to the base context ([#6246](https://github.com/dbt-labs/dbt-core/issues/6246)) +- Exposures support metrics in lineage ([#6057](https://github.com/dbt-labs/dbt-core/issues/6057)) +- Add support for Python 3.11 ([#6147](https://github.com/dbt-labs/dbt-core/issues/6147)) +- incremental predicates ([#5680](https://github.com/dbt-labs/dbt-core/issues/5680)) + +### Fixes + +- Account for disabled flags on models in schema files more completely ([#3992](https://github.com/dbt-labs/dbt-core/issues/3992)) +- Add validation of enabled config for metrics, exposures and sources ([#6030](https://github.com/dbt-labs/dbt-core/issues/6030)) +- check length of args of python model function before accessing it ([#6041](https://github.com/dbt-labs/dbt-core/issues/6041)) +- Add functors to ensure event types with str-type attributes are initialized to spec, even when provided non-str type params. ([#5436](https://github.com/dbt-labs/dbt-core/issues/5436)) +- Allow hooks to fail without halting execution flow ([#5625](https://github.com/dbt-labs/dbt-core/issues/5625)) +- Clarify Error Message for how many models are allowed in a Python file ([#6245](https://github.com/dbt-labs/dbt-core/issues/6245)) +- After this, will be possible to use default values for dbt.config.get ([#6309](https://github.com/dbt-labs/dbt-core/issues/6309)) +- Use full path for writing manifest ([#6055](https://github.com/dbt-labs/dbt-core/issues/6055)) +- [CT-1284] Change Python model default materialization to table ([#6345](https://github.com/dbt-labs/dbt-core/issues/6345)) +- Repair a regression which prevented basic logging before the logging subsystem is completely configured. ([#6434](https://github.com/dbt-labs/dbt-core/issues/6434)) + +### Docs + +- minor doc correction ([dbt-docs/#5791](https://github.com/dbt-labs/dbt-docs/issues/5791)) +- Generate API docs for new CLI interface ([dbt-docs/#5528](https://github.com/dbt-labs/dbt-docs/issues/5528)) +- ([dbt-docs/#5880](https://github.com/dbt-labs/dbt-docs/issues/5880)) +- Fix rendering of sample code for metrics ([dbt-docs/#323](https://github.com/dbt-labs/dbt-docs/issues/323)) +- Alphabetize `core/dbt/README.md` ([dbt-docs/#6368](https://github.com/dbt-labs/dbt-docs/issues/6368)) + +### Under the Hood + +- Put black config in explicit config ([#5946](https://github.com/dbt-labs/dbt-core/issues/5946)) +- Added flat_graph attribute the Manifest class's deepcopy() coverage ([#5809](https://github.com/dbt-labs/dbt-core/issues/5809)) +- Add mypy configs so `mypy` passes from CLI ([#5983](https://github.com/dbt-labs/dbt-core/issues/5983)) +- Exception message cleanup. ([#6023](https://github.com/dbt-labs/dbt-core/issues/6023)) +- Add dmypy cache to gitignore ([#6028](https://github.com/dbt-labs/dbt-core/issues/6028)) +- Provide useful errors when the value of 'materialized' is invalid ([#5229](https://github.com/dbt-labs/dbt-core/issues/5229)) +- Clean up string formatting ([#6068](https://github.com/dbt-labs/dbt-core/issues/6068)) +- Fixed extra whitespace in strings introduced by black. ([#1350](https://github.com/dbt-labs/dbt-core/issues/1350)) +- Remove the 'root_path' field from most nodes ([#6171](https://github.com/dbt-labs/dbt-core/issues/6171)) +- Combine certain logging events with different levels ([#6173](https://github.com/dbt-labs/dbt-core/issues/6173)) +- Convert threading tests to pytest ([#5942](https://github.com/dbt-labs/dbt-core/issues/5942)) +- Convert postgres index tests to pytest ([#5770](https://github.com/dbt-labs/dbt-core/issues/5770)) +- Convert use color tests to pytest ([#5771](https://github.com/dbt-labs/dbt-core/issues/5771)) +- Add github actions workflow to generate high level CLI API docs ([#5942](https://github.com/dbt-labs/dbt-core/issues/5942)) +- Functionality-neutral refactor of event logging system to improve encapsulation and modularity. ([#6139](https://github.com/dbt-labs/dbt-core/issues/6139)) +- Consolidate ParsedNode and CompiledNode classes ([#6383](https://github.com/dbt-labs/dbt-core/issues/6383)) +- Prevent doc gen workflow from running on forks ([#6386](https://github.com/dbt-labs/dbt-core/issues/6386)) +- Fix intermittent database connection failure in Windows CI test ([#6394](https://github.com/dbt-labs/dbt-core/issues/6394)) +- Refactor and clean up manifest nodes ([#6426](https://github.com/dbt-labs/dbt-core/issues/6426)) +- Restore important legacy logging behaviors, following refactor which removed them ([#6437](https://github.com/dbt-labs/dbt-core/issues/6437)) + +### Dependencies + +- Update pathspec requirement from ~=0.9.0 to >=0.9,<0.11 in /core ([#5917](https://github.com/dbt-labs/dbt-core/pull/5917)) +- Bump black from 22.8.0 to 22.10.0 ([#6019](https://github.com/dbt-labs/dbt-core/pull/6019)) +- Bump mashumaro[msgpack] from 3.0.4 to 3.1.1 in /core ([#6108](https://github.com/dbt-labs/dbt-core/pull/6108)) +- Update colorama requirement from <0.4.6,>=0.3.9 to >=0.3.9,<0.4.7 in /core ([#6144](https://github.com/dbt-labs/dbt-core/pull/6144)) +- Bump mashumaro[msgpack] from 3.1.1 to 3.2 in /core ([#4904](https://github.com/dbt-labs/dbt-core/issues/4904)) + +### Contributors +- [@andy-clapson](https://github.com/andy-clapson) ([dbt-docs/#5791](https://github.com/dbt-labs/dbt-docs/issues/5791)) +- [@chamini2](https://github.com/chamini2) ([#6041](https://github.com/dbt-labs/dbt-core/issues/6041)) +- [@daniel-murray](https://github.com/daniel-murray) ([#2968](https://github.com/dbt-labs/dbt-core/issues/2968)) +- [@dave-connors-3](https://github.com/dave-connors-3) ([#5990](https://github.com/dbt-labs/dbt-core/issues/5990)) +- [@dbeatty10](https://github.com/dbeatty10) ([dbt-docs/#6368](https://github.com/dbt-labs/dbt-docs/issues/6368), [#6394](https://github.com/dbt-labs/dbt-core/issues/6394)) +- [@devmessias](https://github.com/devmessias) ([#6309](https://github.com/dbt-labs/dbt-core/issues/6309)) +- [@eve-johns](https://github.com/eve-johns) ([#6068](https://github.com/dbt-labs/dbt-core/issues/6068)) +- [@haritamar](https://github.com/haritamar) ([#6246](https://github.com/dbt-labs/dbt-core/issues/6246)) +- [@jared-rimmer](https://github.com/jared-rimmer) ([#5486](https://github.com/dbt-labs/dbt-core/issues/5486)) +- [@josephberni](https://github.com/josephberni) ([#2968](https://github.com/dbt-labs/dbt-core/issues/2968)) +- [@joshuataylor](https://github.com/joshuataylor) ([#6147](https://github.com/dbt-labs/dbt-core/issues/6147)) +- [@justbldwn](https://github.com/justbldwn) ([#6245](https://github.com/dbt-labs/dbt-core/issues/6245)) +- [@luke-bassett](https://github.com/luke-bassett) ([#1350](https://github.com/dbt-labs/dbt-core/issues/1350)) +- [@max-sixty](https://github.com/max-sixty) ([#5946](https://github.com/dbt-labs/dbt-core/issues/5946), [#5983](https://github.com/dbt-labs/dbt-core/issues/5983), [#6028](https://github.com/dbt-labs/dbt-core/issues/6028)) +- [@paulbenschmidt](https://github.com/paulbenschmidt) ([dbt-docs/#5880](https://github.com/dbt-labs/dbt-docs/issues/5880)) +- [@pgoslatara](https://github.com/pgoslatara) ([#5929](https://github.com/dbt-labs/dbt-core/issues/5929)) +- [@racheldaniel](https://github.com/racheldaniel) ([#6201](https://github.com/dbt-labs/dbt-core/issues/6201)) +- [@timle2](https://github.com/timle2) ([#4205](https://github.com/dbt-labs/dbt-core/issues/4205)) +- [@dave-connors-3](https://github.com/dave-connors-3) ([#5680](https://github.com/dbt-labs/dbt-core/issues/5680)) diff --git a/.changes/1.4.0/Dependency-20220923-000646.yaml b/.changes/1.4.0/Dependency-20220923-000646.yaml new file mode 100644 index 00000000000..0375eeb125f --- /dev/null +++ b/.changes/1.4.0/Dependency-20220923-000646.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update pathspec requirement from ~=0.9.0 to >=0.9,<0.11 in /core" +time: 2022-09-23T00:06:46.00000Z +custom: + Author: dependabot[bot] + PR: "5917" diff --git a/.changes/1.4.0/Dependency-20221007-000848.yaml b/.changes/1.4.0/Dependency-20221007-000848.yaml new file mode 100644 index 00000000000..7e36733d14e --- /dev/null +++ b/.changes/1.4.0/Dependency-20221007-000848.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Bump black from 22.8.0 to 22.10.0" +time: 2022-10-07T00:08:48.00000Z +custom: + Author: dependabot[bot] + PR: "6019" diff --git a/.changes/1.4.0/Dependency-20221020-000753.yaml b/.changes/1.4.0/Dependency-20221020-000753.yaml new file mode 100644 index 00000000000..ce0f122826b --- /dev/null +++ b/.changes/1.4.0/Dependency-20221020-000753.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Bump mashumaro[msgpack] from 3.0.4 to 3.1.1 in /core" +time: 2022-10-20T00:07:53.00000Z +custom: + Author: dependabot[bot] + PR: "6108" diff --git a/.changes/1.4.0/Dependency-20221026-000910.yaml b/.changes/1.4.0/Dependency-20221026-000910.yaml new file mode 100644 index 00000000000..d68fa8a11ef --- /dev/null +++ b/.changes/1.4.0/Dependency-20221026-000910.yaml @@ -0,0 +1,6 @@ +kind: "Dependencies" +body: "Update colorama requirement from <0.4.6,>=0.3.9 to >=0.3.9,<0.4.7 in /core" +time: 2022-10-26T00:09:10.00000Z +custom: + Author: dependabot[bot] + PR: "6144" diff --git a/.changes/1.4.0/Dependency-20221205-002118.yaml b/.changes/1.4.0/Dependency-20221205-002118.yaml new file mode 100644 index 00000000000..f4203a5285c --- /dev/null +++ b/.changes/1.4.0/Dependency-20221205-002118.yaml @@ -0,0 +1,7 @@ +kind: "Dependencies" +body: "Bump mashumaro[msgpack] from 3.1.1 to 3.2 in /core" +time: 2022-12-05T00:21:18.00000Z +custom: + Author: dependabot[bot] + Issue: 4904 + PR: 6375 diff --git a/.changes/1.4.0/Docs-20220908-154157.yaml b/.changes/1.4.0/Docs-20220908-154157.yaml new file mode 100644 index 00000000000..e307f3bd5e0 --- /dev/null +++ b/.changes/1.4.0/Docs-20220908-154157.yaml @@ -0,0 +1,6 @@ +kind: Docs +body: minor doc correction +time: 2022-09-08T15:41:57.689162-04:00 +custom: + Author: andy-clapson + Issue: "5791" diff --git a/.changes/1.4.0/Docs-20221007-090656.yaml b/.changes/1.4.0/Docs-20221007-090656.yaml new file mode 100644 index 00000000000..070ecd48944 --- /dev/null +++ b/.changes/1.4.0/Docs-20221007-090656.yaml @@ -0,0 +1,6 @@ +kind: Docs +body: Generate API docs for new CLI interface +time: 2022-10-07T09:06:56.446078-05:00 +custom: + Author: stu-k + Issue: "5528" diff --git a/.changes/1.4.0/Docs-20221017-171411.yaml b/.changes/1.4.0/Docs-20221017-171411.yaml new file mode 100644 index 00000000000..487362c1d5c --- /dev/null +++ b/.changes/1.4.0/Docs-20221017-171411.yaml @@ -0,0 +1,5 @@ +kind: Docs +time: 2022-10-17T17:14:11.715348-05:00 +custom: + Author: paulbenschmidt + Issue: "5880" diff --git a/.changes/1.4.0/Docs-20221116-155743.yaml b/.changes/1.4.0/Docs-20221116-155743.yaml new file mode 100644 index 00000000000..84d90a67b99 --- /dev/null +++ b/.changes/1.4.0/Docs-20221116-155743.yaml @@ -0,0 +1,6 @@ +kind: Docs +body: Fix rendering of sample code for metrics +time: 2022-11-16T15:57:43.204201+01:00 +custom: + Author: jtcohen6 + Issue: "323" diff --git a/.changes/1.4.0/Docs-20221202-150523.yaml b/.changes/1.4.0/Docs-20221202-150523.yaml new file mode 100644 index 00000000000..b08a32cddf6 --- /dev/null +++ b/.changes/1.4.0/Docs-20221202-150523.yaml @@ -0,0 +1,6 @@ +kind: Docs +body: Alphabetize `core/dbt/README.md` +time: 2022-12-02T15:05:23.695333-07:00 +custom: + Author: dbeatty10 + Issue: "6368" diff --git a/.changes/1.4.0/Features-20220408-165459.yaml b/.changes/1.4.0/Features-20220408-165459.yaml new file mode 100644 index 00000000000..12cdf74c757 --- /dev/null +++ b/.changes/1.4.0/Features-20220408-165459.yaml @@ -0,0 +1,7 @@ +kind: Features +body: Added favor-state flag to optionally favor state nodes even if unselected node + exists +time: 2022-04-08T16:54:59.696564+01:00 +custom: + Author: daniel-murray josephberni + Issue: "2968" diff --git a/.changes/1.4.0/Features-20220817-154857.yaml b/.changes/1.4.0/Features-20220817-154857.yaml new file mode 100644 index 00000000000..ad53df05a3f --- /dev/null +++ b/.changes/1.4.0/Features-20220817-154857.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Update structured logging. Convert to using protobuf messages. Ensure events are enriched with node_info. +time: 2022-08-17T15:48:57.225267-04:00 +custom: + Author: gshank + Issue: "5610" diff --git a/.changes/1.4.0/Features-20220823-085727.yaml b/.changes/1.4.0/Features-20220823-085727.yaml new file mode 100644 index 00000000000..4d8daebbf5e --- /dev/null +++ b/.changes/1.4.0/Features-20220823-085727.yaml @@ -0,0 +1,7 @@ +kind: Features +body: incremental predicates +time: 2022-08-23T08:57:27.640804-05:00 +custom: + Author: dave-connors-3 + Issue: "5680" + PR: "5702" diff --git a/.changes/1.4.0/Features-20220912-125935.yaml b/.changes/1.4.0/Features-20220912-125935.yaml new file mode 100644 index 00000000000..d49f35fd0af --- /dev/null +++ b/.changes/1.4.0/Features-20220912-125935.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Friendlier error messages when packages.yml is malformed +time: 2022-09-12T12:59:35.121188+01:00 +custom: + Author: jared-rimmer + Issue: "5486" diff --git a/.changes/1.4.0/Features-20220914-095625.yaml b/.changes/1.4.0/Features-20220914-095625.yaml new file mode 100644 index 00000000000..d46b1bfa8d8 --- /dev/null +++ b/.changes/1.4.0/Features-20220914-095625.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Migrate dbt-utils current_timestamp macros into core + adapters +time: 2022-09-14T09:56:25.97818-07:00 +custom: + Author: colin-rogers-dbt + Issue: "5521" diff --git a/.changes/1.4.0/Features-20220925-211651.yaml b/.changes/1.4.0/Features-20220925-211651.yaml new file mode 100644 index 00000000000..d2c1911c720 --- /dev/null +++ b/.changes/1.4.0/Features-20220925-211651.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Allow partitions in external tables to be supplied as a list +time: 2022-09-25T21:16:51.051239654+02:00 +custom: + Author: pgoslatara + Issue: "5929" diff --git a/.changes/1.4.0/Features-20221003-110705.yaml b/.changes/1.4.0/Features-20221003-110705.yaml new file mode 100644 index 00000000000..637d8be58c6 --- /dev/null +++ b/.changes/1.4.0/Features-20221003-110705.yaml @@ -0,0 +1,6 @@ +kind: Features +body: extend -f flag shorthand for seed command +time: 2022-10-03T11:07:05.381632-05:00 +custom: + Author: dave-connors-3 + Issue: "5990" diff --git a/.changes/1.4.0/Features-20221102-150003.yaml b/.changes/1.4.0/Features-20221102-150003.yaml new file mode 100644 index 00000000000..9d8ba192687 --- /dev/null +++ b/.changes/1.4.0/Features-20221102-150003.yaml @@ -0,0 +1,7 @@ +kind: Features +body: This pulls the profile name from args when constructing a RuntimeConfig in lib.py, + enabling the dbt-server to override the value that's in the dbt_project.yml +time: 2022-11-02T15:00:03.000805-05:00 +custom: + Author: racheldaniel + Issue: "6201" diff --git a/.changes/1.4.0/Features-20221107-105018.yaml b/.changes/1.4.0/Features-20221107-105018.yaml new file mode 100644 index 00000000000..db6a0ab753a --- /dev/null +++ b/.changes/1.4.0/Features-20221107-105018.yaml @@ -0,0 +1,8 @@ +kind: Features +body: Adding tarball install method for packages. Allowing package tarball to be specified + via url in the packages.yaml. +time: 2022-11-07T10:50:18.464545-05:00 +custom: + Author: timle2 + Issue: "4205" + PR: "4689" diff --git a/.changes/1.4.0/Features-20221114-185207.yaml b/.changes/1.4.0/Features-20221114-185207.yaml new file mode 100644 index 00000000000..459bc8ce234 --- /dev/null +++ b/.changes/1.4.0/Features-20221114-185207.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Added an md5 function to the base context +time: 2022-11-14T18:52:07.788593+02:00 +custom: + Author: haritamar + Issue: "6246" diff --git a/.changes/1.4.0/Features-20221130-112913.yaml b/.changes/1.4.0/Features-20221130-112913.yaml new file mode 100644 index 00000000000..64832de2f68 --- /dev/null +++ b/.changes/1.4.0/Features-20221130-112913.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Exposures support metrics in lineage +time: 2022-11-30T11:29:13.256034-05:00 +custom: + Author: michelleark + Issue: "6057" diff --git a/.changes/1.4.0/Features-20221206-150704.yaml b/.changes/1.4.0/Features-20221206-150704.yaml new file mode 100644 index 00000000000..47939ea5a79 --- /dev/null +++ b/.changes/1.4.0/Features-20221206-150704.yaml @@ -0,0 +1,7 @@ +kind: Features +body: Add support for Python 3.11 +time: 2022-12-06T15:07:04.753127+01:00 +custom: + Author: joshuataylor MichelleArk jtcohen6 + Issue: "6147" + PR: "6326" diff --git a/.changes/1.4.0/Fixes-20220916-104854.yaml b/.changes/1.4.0/Fixes-20220916-104854.yaml new file mode 100644 index 00000000000..bd9af0469a7 --- /dev/null +++ b/.changes/1.4.0/Fixes-20220916-104854.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Account for disabled flags on models in schema files more completely +time: 2022-09-16T10:48:54.162273-05:00 +custom: + Author: emmyoop + Issue: "3992" diff --git a/.changes/1.4.0/Fixes-20221010-113218.yaml b/.changes/1.4.0/Fixes-20221010-113218.yaml new file mode 100644 index 00000000000..5b73b8d9ccd --- /dev/null +++ b/.changes/1.4.0/Fixes-20221010-113218.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Add validation of enabled config for metrics, exposures and sources +time: 2022-10-10T11:32:18.752322-05:00 +custom: + Author: emmyoop + Issue: "6030" diff --git a/.changes/1.4.0/Fixes-20221011-160715.yaml b/.changes/1.4.0/Fixes-20221011-160715.yaml new file mode 100644 index 00000000000..936546a5232 --- /dev/null +++ b/.changes/1.4.0/Fixes-20221011-160715.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: check length of args of python model function before accessing it +time: 2022-10-11T16:07:15.464093-04:00 +custom: + Author: chamini2 + Issue: "6041" diff --git a/.changes/1.4.0/Fixes-20221016-173742.yaml b/.changes/1.4.0/Fixes-20221016-173742.yaml new file mode 100644 index 00000000000..c7b00dddba8 --- /dev/null +++ b/.changes/1.4.0/Fixes-20221016-173742.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: Add functors to ensure event types with str-type attributes are initialized + to spec, even when provided non-str type params. +time: 2022-10-16T17:37:42.846683-07:00 +custom: + Author: versusfacit + Issue: "5436" diff --git a/.changes/1.4.0/Fixes-20221107-095314.yaml b/.changes/1.4.0/Fixes-20221107-095314.yaml new file mode 100644 index 00000000000..99da9c44522 --- /dev/null +++ b/.changes/1.4.0/Fixes-20221107-095314.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Allow hooks to fail without halting execution flow +time: 2022-11-07T09:53:14.340257-06:00 +custom: + Author: ChenyuLInx + Issue: "5625" diff --git a/.changes/1.4.0/Fixes-20221115-081021.yaml b/.changes/1.4.0/Fixes-20221115-081021.yaml new file mode 100644 index 00000000000..40c81fabacb --- /dev/null +++ b/.changes/1.4.0/Fixes-20221115-081021.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Clarify Error Message for how many models are allowed in a Python file +time: 2022-11-15T08:10:21.527884-05:00 +custom: + Author: justbldwn + Issue: "6245" diff --git a/.changes/1.4.0/Fixes-20221124-163419.yaml b/.changes/1.4.0/Fixes-20221124-163419.yaml new file mode 100644 index 00000000000..010a073269a --- /dev/null +++ b/.changes/1.4.0/Fixes-20221124-163419.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: After this, will be possible to use default values for dbt.config.get +time: 2022-11-24T16:34:19.039512764-03:00 +custom: + Author: devmessias + Issue: "6309" + PR: "6317" diff --git a/.changes/1.4.0/Fixes-20221202-164859.yaml b/.changes/1.4.0/Fixes-20221202-164859.yaml new file mode 100644 index 00000000000..6aad4ced192 --- /dev/null +++ b/.changes/1.4.0/Fixes-20221202-164859.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Use full path for writing manifest +time: 2022-12-02T16:48:59.029519-05:00 +custom: + Author: gshank + Issue: "6055" diff --git a/.changes/1.4.0/Fixes-20221213-112620.yaml b/.changes/1.4.0/Fixes-20221213-112620.yaml new file mode 100644 index 00000000000..a2220f9a920 --- /dev/null +++ b/.changes/1.4.0/Fixes-20221213-112620.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: '[CT-1284] Change Python model default materialization to table' +time: 2022-12-13T11:26:20.550017-08:00 +custom: + Author: aranke + Issue: "6345" diff --git a/.changes/1.4.0/Fixes-20221214-155307.yaml b/.changes/1.4.0/Fixes-20221214-155307.yaml new file mode 100644 index 00000000000..cb37e0a809c --- /dev/null +++ b/.changes/1.4.0/Fixes-20221214-155307.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: Repair a regression which prevented basic logging before the logging subsystem + is completely configured. +time: 2022-12-14T15:53:07.396512-05:00 +custom: + Author: peterallenwebb + Issue: "6434" diff --git a/.changes/1.4.0/Under the Hood-20220927-194259.yaml b/.changes/1.4.0/Under the Hood-20220927-194259.yaml new file mode 100644 index 00000000000..b6cb64b0155 --- /dev/null +++ b/.changes/1.4.0/Under the Hood-20220927-194259.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Put black config in explicit config +time: 2022-09-27T19:42:59.241433-07:00 +custom: + Author: max-sixty + Issue: "5946" diff --git a/.changes/1.4.0/Under the Hood-20220929-134406.yaml b/.changes/1.4.0/Under the Hood-20220929-134406.yaml new file mode 100644 index 00000000000..b0175190747 --- /dev/null +++ b/.changes/1.4.0/Under the Hood-20220929-134406.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Added flat_graph attribute the Manifest class's deepcopy() coverage +time: 2022-09-29T13:44:06.275941-04:00 +custom: + Author: peterallenwebb + Issue: "5809" diff --git a/.changes/1.4.0/Under the Hood-20221005-120310.yaml b/.changes/1.4.0/Under the Hood-20221005-120310.yaml new file mode 100644 index 00000000000..797be31c319 --- /dev/null +++ b/.changes/1.4.0/Under the Hood-20221005-120310.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Add mypy configs so `mypy` passes from CLI +time: 2022-10-05T12:03:10.061263-07:00 +custom: + Author: max-sixty + Issue: "5983" diff --git a/.changes/1.4.0/Under the Hood-20221007-094627.yaml b/.changes/1.4.0/Under the Hood-20221007-094627.yaml new file mode 100644 index 00000000000..d3a5da61566 --- /dev/null +++ b/.changes/1.4.0/Under the Hood-20221007-094627.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Exception message cleanup. +time: 2022-10-07T09:46:27.682872-05:00 +custom: + Author: emmyoop + Issue: "6023" diff --git a/.changes/1.4.0/Under the Hood-20221007-140044.yaml b/.changes/1.4.0/Under the Hood-20221007-140044.yaml new file mode 100644 index 00000000000..971d5a40ce8 --- /dev/null +++ b/.changes/1.4.0/Under the Hood-20221007-140044.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Add dmypy cache to gitignore +time: 2022-10-07T14:00:44.227644-07:00 +custom: + Author: max-sixty + Issue: "6028" diff --git a/.changes/1.4.0/Under the Hood-20221013-181912.yaml b/.changes/1.4.0/Under the Hood-20221013-181912.yaml new file mode 100644 index 00000000000..4f5218891b4 --- /dev/null +++ b/.changes/1.4.0/Under the Hood-20221013-181912.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Provide useful errors when the value of 'materialized' is invalid +time: 2022-10-13T18:19:12.167548-04:00 +custom: + Author: peterallenwebb + Issue: "5229" diff --git a/.changes/1.4.0/Under the Hood-20221017-151511.yaml b/.changes/1.4.0/Under the Hood-20221017-151511.yaml new file mode 100644 index 00000000000..94f4d27d6de --- /dev/null +++ b/.changes/1.4.0/Under the Hood-20221017-151511.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Fixed extra whitespace in strings introduced by black. +time: 2022-10-17T15:15:11.499246-05:00 +custom: + Author: luke-bassett + Issue: "1350" diff --git a/.changes/1.4.0/Under the Hood-20221017-155844.yaml b/.changes/1.4.0/Under the Hood-20221017-155844.yaml new file mode 100644 index 00000000000..c46ef040410 --- /dev/null +++ b/.changes/1.4.0/Under the Hood-20221017-155844.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Clean up string formatting +time: 2022-10-17T15:58:44.676549-04:00 +custom: + Author: eve-johns + Issue: "6068" diff --git a/.changes/1.4.0/Under the Hood-20221028-104837.yaml b/.changes/1.4.0/Under the Hood-20221028-104837.yaml new file mode 100644 index 00000000000..446d4898920 --- /dev/null +++ b/.changes/1.4.0/Under the Hood-20221028-104837.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Remove the 'root_path' field from most nodes +time: 2022-10-28T10:48:37.687886-04:00 +custom: + Author: gshank + Issue: "6171" diff --git a/.changes/1.4.0/Under the Hood-20221028-110344.yaml b/.changes/1.4.0/Under the Hood-20221028-110344.yaml new file mode 100644 index 00000000000..cbe8dacb3d5 --- /dev/null +++ b/.changes/1.4.0/Under the Hood-20221028-110344.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Combine certain logging events with different levels +time: 2022-10-28T11:03:44.887836-04:00 +custom: + Author: gshank + Issue: "6173" diff --git a/.changes/1.4.0/Under the Hood-20221108-074550.yaml b/.changes/1.4.0/Under the Hood-20221108-074550.yaml new file mode 100644 index 00000000000..a8fbc7e208b --- /dev/null +++ b/.changes/1.4.0/Under the Hood-20221108-074550.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Convert threading tests to pytest +time: 2022-11-08T07:45:50.589147-06:00 +custom: + Author: stu-k + Issue: "5942" diff --git a/.changes/1.4.0/Under the Hood-20221108-115633.yaml b/.changes/1.4.0/Under the Hood-20221108-115633.yaml new file mode 100644 index 00000000000..ea073719cda --- /dev/null +++ b/.changes/1.4.0/Under the Hood-20221108-115633.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Convert postgres index tests to pytest +time: 2022-11-08T11:56:33.743042-06:00 +custom: + Author: stu-k + Issue: "5770" diff --git a/.changes/1.4.0/Under the Hood-20221108-133104.yaml b/.changes/1.4.0/Under the Hood-20221108-133104.yaml new file mode 100644 index 00000000000..6829dc097eb --- /dev/null +++ b/.changes/1.4.0/Under the Hood-20221108-133104.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Convert use color tests to pytest +time: 2022-11-08T13:31:04.788547-06:00 +custom: + Author: stu-k + Issue: "5771" diff --git a/.changes/1.4.0/Under the Hood-20221116-130037.yaml b/.changes/1.4.0/Under the Hood-20221116-130037.yaml new file mode 100644 index 00000000000..ecdedd6bd2d --- /dev/null +++ b/.changes/1.4.0/Under the Hood-20221116-130037.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Add github actions workflow to generate high level CLI API docs +time: 2022-11-16T13:00:37.916202-06:00 +custom: + Author: stu-k + Issue: "5942" diff --git a/.changes/1.4.0/Under the Hood-20221118-145717.yaml b/.changes/1.4.0/Under the Hood-20221118-145717.yaml new file mode 100644 index 00000000000..934cd9dd5cb --- /dev/null +++ b/.changes/1.4.0/Under the Hood-20221118-145717.yaml @@ -0,0 +1,8 @@ +kind: Under the Hood +body: Functionality-neutral refactor of event logging system to improve encapsulation + and modularity. +time: 2022-11-18T14:57:17.792622-05:00 +custom: + Author: peterallenwebb + Issue: "6139" + PR: "6291" diff --git a/.changes/1.4.0/Under the Hood-20221205-164948.yaml b/.changes/1.4.0/Under the Hood-20221205-164948.yaml new file mode 100644 index 00000000000..579f973955b --- /dev/null +++ b/.changes/1.4.0/Under the Hood-20221205-164948.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: Consolidate ParsedNode and CompiledNode classes +time: 2022-12-05T16:49:48.563583-05:00 +custom: + Author: gshank + Issue: "6383" + PR: "6384" diff --git a/.changes/1.4.0/Under the Hood-20221206-094015.yaml b/.changes/1.4.0/Under the Hood-20221206-094015.yaml new file mode 100644 index 00000000000..ebcb9999430 --- /dev/null +++ b/.changes/1.4.0/Under the Hood-20221206-094015.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: Prevent doc gen workflow from running on forks +time: 2022-12-06T09:40:15.301984-06:00 +custom: + Author: stu-k + Issue: "6386" + PR: "6390" diff --git a/.changes/1.4.0/Under the Hood-20221206-113053.yaml b/.changes/1.4.0/Under the Hood-20221206-113053.yaml new file mode 100644 index 00000000000..a1f94f68f43 --- /dev/null +++ b/.changes/1.4.0/Under the Hood-20221206-113053.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: Fix intermittent database connection failure in Windows CI test +time: 2022-12-06T11:30:53.166009-07:00 +custom: + Author: MichelleArk dbeatty10 + Issue: "6394" + PR: "6395" diff --git a/.changes/1.4.0/Under the Hood-20221211-214240.yaml b/.changes/1.4.0/Under the Hood-20221211-214240.yaml new file mode 100644 index 00000000000..adeaefba257 --- /dev/null +++ b/.changes/1.4.0/Under the Hood-20221211-214240.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: Refactor and clean up manifest nodes +time: 2022-12-11T21:42:40.560074-05:00 +custom: + Author: gshank + Issue: "6426" + PR: "6427" diff --git a/.changes/1.4.0/Under the Hood-20221213-214106.yaml b/.changes/1.4.0/Under the Hood-20221213-214106.yaml new file mode 100644 index 00000000000..708c84661d6 --- /dev/null +++ b/.changes/1.4.0/Under the Hood-20221213-214106.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: Restore important legacy logging behaviors, following refactor which removed + them +time: 2022-12-13T21:41:06.815133-05:00 +custom: + Author: peterallenwebb + Issue: "6437" diff --git a/.changes/unreleased/Breaking Changes-20221205-141937.yaml b/.changes/unreleased/Breaking Changes-20221205-141937.yaml new file mode 100644 index 00000000000..be840b20a99 --- /dev/null +++ b/.changes/unreleased/Breaking Changes-20221205-141937.yaml @@ -0,0 +1,9 @@ +kind: Breaking Changes +body: Cleaned up exceptions to directly raise in code. Removed use of all exception + functions in the code base and marked them all as deprecated to be removed next + minor release. +time: 2022-12-05T14:19:37.863032-06:00 +custom: + Author: emmyoop + Issue: "6339" + PR: "6347" diff --git a/.changes/unreleased/Features-20221129-183239.yaml b/.changes/unreleased/Features-20221129-183239.yaml new file mode 100644 index 00000000000..22a92ea36a7 --- /dev/null +++ b/.changes/unreleased/Features-20221129-183239.yaml @@ -0,0 +1,7 @@ +kind: Features +body: Click CLI Flags work with UserConfig +time: 2022-11-29T18:32:39.068035-05:00 +custom: + Author: michelleark + Issue: "6327" + PR: "6266" diff --git a/.changes/unreleased/Fixes-20221016-173742.yaml b/.changes/unreleased/Fixes-20221016-173742.yaml new file mode 100644 index 00000000000..11d4a8c85f4 --- /dev/null +++ b/.changes/unreleased/Fixes-20221016-173742.yaml @@ -0,0 +1,8 @@ +kind: Fixes +body: Add functors to ensure event types with str-type attributes are initialized + to spec, even when provided non-str type params. +time: 2022-10-16T17:37:42.846683-07:00 +custom: + Author: versusfacit + Issue: "5436" + PR: "5874" diff --git a/.changes/unreleased/Fixes-20221117-220320.yaml b/.changes/unreleased/Fixes-20221117-220320.yaml new file mode 100644 index 00000000000..2f71fe213fc --- /dev/null +++ b/.changes/unreleased/Fixes-20221117-220320.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: Fix typo in util.py +time: 2022-11-17T22:03:20.4836855+09:00 +custom: + Author: eltociear + Issue: "4904" + PR: "6037" diff --git a/.changes/unreleased/Fixes-20221213-113915.yaml b/.changes/unreleased/Fixes-20221213-113915.yaml new file mode 100644 index 00000000000..b92a2d6cbc9 --- /dev/null +++ b/.changes/unreleased/Fixes-20221213-113915.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: '[CT-1591] Don''t parse empty Python files' +time: 2022-12-13T11:39:15.818464-08:00 +custom: + Author: aranke + Issue: "6345" diff --git a/.changes/unreleased/Under the Hood-20221017-151511.yaml b/.changes/unreleased/Under the Hood-20221017-151511.yaml new file mode 100644 index 00000000000..cbdcf04beb3 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20221017-151511.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: Fixed extra whitespace in strings introduced by black. +time: 2022-10-17T15:15:11.499246-05:00 +custom: + Author: luke-bassett + Issue: "1350" + PR: "6086" diff --git a/.changes/unreleased/Under the Hood-20221017-155844.yaml b/.changes/unreleased/Under the Hood-20221017-155844.yaml new file mode 100644 index 00000000000..84e6675351c --- /dev/null +++ b/.changes/unreleased/Under the Hood-20221017-155844.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: Clean up string formatting +time: 2022-10-17T15:58:44.676549-04:00 +custom: + Author: eve-johns + Issue: "6068" + PR: "6082" diff --git a/.changes/unreleased/Under the Hood-20221017-170500.yaml b/.changes/unreleased/Under the Hood-20221017-170500.yaml new file mode 100644 index 00000000000..8f5e20cd8ff --- /dev/null +++ b/.changes/unreleased/Under the Hood-20221017-170500.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: Click CLI supports logging +time: 2022-10-17T17:05:00.478948-05:00 +custom: + Author: iknox-fa + Issue: "5530" + PR: "6088" diff --git a/.changes/unreleased/Under the Hood-20221202-123046.yaml b/.changes/unreleased/Under the Hood-20221202-123046.yaml new file mode 100644 index 00000000000..9209fea0639 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20221202-123046.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: '`dbt clean` works with click cli' +time: 2022-12-02T12:30:46.711184-06:00 +custom: + Author: stu-k + Issue: "5542" + PR: "6369" diff --git a/.changes/unreleased/Under the Hood-20221204-161110.yaml b/.changes/unreleased/Under the Hood-20221204-161110.yaml new file mode 100644 index 00000000000..496ffdae7b2 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20221204-161110.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: Adds Profile support to the new Click CLI +time: 2022-12-04T16:11:10.417872-06:00 +custom: + Author: iknox-fa ChenyuLInx + Issue: "5536" + PR: "6336" diff --git a/.changes/unreleased/Under the Hood-20221205-092217.yaml b/.changes/unreleased/Under the Hood-20221205-092217.yaml new file mode 100644 index 00000000000..a9ebbb9699b --- /dev/null +++ b/.changes/unreleased/Under the Hood-20221205-092217.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: Pin ubuntu at 20.04 for some workflows +time: 2022-12-05T09:22:17.614875-06:00 +custom: + Author: stu-k + Issue: "9999" + PR: "6380" diff --git a/.changes/unreleased/Under the Hood-20221206-151759.yaml b/.changes/unreleased/Under the Hood-20221206-151759.yaml new file mode 100644 index 00000000000..cce61ce40bc --- /dev/null +++ b/.changes/unreleased/Under the Hood-20221206-151759.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: '`dbt run` works with Click cli' +time: 2022-12-06T15:17:59.765623-06:00 +custom: + Author: iknox-fa + Issue: "5551" + PR: "6396" diff --git a/.changes/unreleased/Under the Hood-20221212-110859.yaml b/.changes/unreleased/Under the Hood-20221212-110859.yaml new file mode 100644 index 00000000000..4c4bbaa294e --- /dev/null +++ b/.changes/unreleased/Under the Hood-20221212-110859.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: Cherry pick tox fix +time: 2022-12-12T11:08:59.440276-06:00 +custom: + Author: stu-k + Issue: "9999" + PR: "6430" diff --git a/.changes/unreleased/Under the Hood-20221214-112048.yaml b/.changes/unreleased/Under the Hood-20221214-112048.yaml new file mode 100644 index 00000000000..9ac833b6e60 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20221214-112048.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: functional tests run using click cli through dbtRunner +time: 2022-12-14T11:20:48.521869-05:00 +custom: + Author: MichelleArk + Issue: "6096" + PR: "6387" diff --git a/.changes/unreleased/Under the Hood-20221219-193435.yaml b/.changes/unreleased/Under the Hood-20221219-193435.yaml new file mode 100644 index 00000000000..82388dbb759 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20221219-193435.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Treat dense text blobs as binary for `git grep` +time: 2022-12-19T19:34:35.890275-07:00 +custom: + Author: dbeatty10 + Issue: "6294" diff --git a/.changes/unreleased/Under the Hood-20230105-104748.yaml b/.changes/unreleased/Under the Hood-20230105-104748.yaml new file mode 100644 index 00000000000..02f0e01988f --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230105-104748.yaml @@ -0,0 +1,7 @@ +kind: Under the Hood +body: Remove UnsetProfileConfig +time: 2023-01-05T10:47:48.707656-05:00 +custom: + Author: MichelleArk + Issue: "5539" + PR: "6504" diff --git a/.changes/unreleased/Under the Hood-20230109-151417.yaml b/.changes/unreleased/Under the Hood-20230109-151417.yaml new file mode 100644 index 00000000000..7a30067c4d9 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230109-151417.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: '[CT-932] Implement `dbt test` in Click' +time: 2023-01-09T15:14:17.524221-08:00 +custom: + Author: aranke + Issue: "5556" diff --git a/.changes/unreleased/Under the Hood-20230110-115725.yaml b/.changes/unreleased/Under the Hood-20230110-115725.yaml new file mode 100644 index 00000000000..81dfefdba91 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230110-115725.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Abstract manifest generation +time: 2023-01-10T11:57:25.193965-06:00 +custom: + Author: stu-k + Issue: "6357" diff --git a/.changes/unreleased/Under the Hood-20230111-145143.yaml b/.changes/unreleased/Under the Hood-20230111-145143.yaml new file mode 100644 index 00000000000..2f5c9c3fc1b --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230111-145143.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: '[CT-921] dbt compile works in click' +time: 2023-01-11T14:51:43.324107-08:00 +custom: + Author: aranke + Issue: "5545" diff --git a/.changes/unreleased/Under the Hood-20230117-162505.yaml b/.changes/unreleased/Under the Hood-20230117-162505.yaml new file mode 100644 index 00000000000..5980bd2632a --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230117-162505.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: dbt snapshot works in click +time: 2023-01-17T16:25:05.973769-08:00 +custom: + Author: ChenyuLInx + Issue: "5554" diff --git a/.changes/unreleased/Under the Hood-20230117-213729.yaml b/.changes/unreleased/Under the Hood-20230117-213729.yaml new file mode 100644 index 00000000000..8500a0a70b7 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230117-213729.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: dbt list working with click +time: 2023-01-17T21:37:29.91632-05:00 +custom: + Author: michelleark + Issue: "5549" diff --git a/.changes/unreleased/Under the Hood-20230119-105304.yaml b/.changes/unreleased/Under the Hood-20230119-105304.yaml new file mode 100644 index 00000000000..8d85fec68fc --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230119-105304.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Add dbt run-operation to click CLI +time: 2023-01-19T10:53:04.154871+01:00 +custom: + Author: jtcohen6 + Issue: "5552" diff --git a/.changes/unreleased/Under the Hood-20230119-205650.yaml b/.changes/unreleased/Under the Hood-20230119-205650.yaml new file mode 100644 index 00000000000..4121d4a3118 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230119-205650.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: dbt build working with new click framework +time: 2023-01-19T20:56:50.50549-05:00 +custom: + Author: michelleark + Issue: "5541" diff --git a/.changes/unreleased/Under the Hood-20230119-211040.yaml b/.changes/unreleased/Under the Hood-20230119-211040.yaml new file mode 100644 index 00000000000..2968fa6b6dc --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230119-211040.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: dbt docs generate works with new click framework +time: 2023-01-19T21:10:40.698851-05:00 +custom: + Author: michelleark + Issue: "5543" diff --git a/.changes/unreleased/Under the Hood-20230124-175110.yaml b/.changes/unreleased/Under the Hood-20230124-175110.yaml new file mode 100644 index 00000000000..9d8e5b35907 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230124-175110.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: dbt init works with click +time: 2023-01-24T17:51:10.74065-05:00 +custom: + Author: michelleark + Issue: "5548" diff --git a/.changes/unreleased/Under the Hood-20230125-041136.yaml b/.changes/unreleased/Under the Hood-20230125-041136.yaml new file mode 100644 index 00000000000..ff02d3ef752 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230125-041136.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: '[CT-920][CT-1900] Create Click CLI runner and use it to fix dbt docs commands' +time: 2023-01-25T04:11:36.57506-08:00 +custom: + Author: aranke + Issue: 5544 6722 diff --git a/.changes/unreleased/Under the Hood-20230125-102606.yaml b/.changes/unreleased/Under the Hood-20230125-102606.yaml new file mode 100644 index 00000000000..9e5cba8e28b --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230125-102606.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Migrate debug task to click +time: 2023-01-25T10:26:06.735994-06:00 +custom: + Author: stu-k + Issue: "5546" diff --git a/.changes/unreleased/Under the Hood-20230126-143102.yaml b/.changes/unreleased/Under the Hood-20230126-143102.yaml new file mode 100644 index 00000000000..505efabf53c --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230126-143102.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Implement --version for click cli +time: 2023-01-26T14:31:02.740282-06:00 +custom: + Author: stu-k + Issue: "6757" diff --git a/.changes/unreleased/Under the Hood-20230130-175752.yaml b/.changes/unreleased/Under the Hood-20230130-175752.yaml new file mode 100644 index 00000000000..95a4938915a --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230130-175752.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: "Enables the new Click Cli on the commandline! \U0001F680" +time: 2023-01-30T17:57:52.65626-06:00 +custom: + Author: iknox-fa + Issue: "6784" diff --git a/.changes/unreleased/Under the Hood-20230130-180917.yaml b/.changes/unreleased/Under the Hood-20230130-180917.yaml new file mode 100644 index 00000000000..64c35d67f12 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230130-180917.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: warn_error/warn_error_options mutual exclusivity in click +time: 2023-01-30T18:09:17.240662-05:00 +custom: + Author: michelleark + Issue: "6579" diff --git a/.changes/unreleased/Under the Hood-20230131-141806.yaml b/.changes/unreleased/Under the Hood-20230131-141806.yaml new file mode 100644 index 00000000000..7ebc38b8008 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230131-141806.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Lazily call --version +time: 2023-01-31T14:18:06.02312-06:00 +custom: + Author: stu-k + Issue: "6812" diff --git a/.changes/unreleased/Under the Hood-20230207-165111.yaml b/.changes/unreleased/Under the Hood-20230207-165111.yaml new file mode 100644 index 00000000000..2f9a609d6de --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230207-165111.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: flags.THREADS defaults to None +time: 2023-02-07T16:51:11.011984-05:00 +custom: + Author: michelleark + Issue: "6887" diff --git a/core/dbt/adapters/base/plugin.py b/core/dbt/adapters/base/plugin.py index f1a77f89b9d..58481f75439 100644 --- a/core/dbt/adapters/base/plugin.py +++ b/core/dbt/adapters/base/plugin.py @@ -7,9 +7,9 @@ def project_name_from_path(include_path: str) -> str: # avoid an import cycle - from dbt.config.project import Project + from dbt.config.project import PartialProject - partial = Project.partial_load(include_path) + partial = PartialProject.from_project_root(include_path) if partial.project_name is None: raise CompilationError(f"Invalid project at {include_path}: name not set!") return partial.project_name diff --git a/core/dbt/adapters/cache.py b/core/dbt/adapters/cache.py index 24a0e469df1..dc8ff14e67e 100644 --- a/core/dbt/adapters/cache.py +++ b/core/dbt/adapters/cache.py @@ -17,7 +17,7 @@ ) from dbt.events.functions import fire_event, fire_event_if from dbt.events.types import CacheAction, CacheDumpGraph -import dbt.flags as flags +from dbt.flags import get_flags from dbt.utils import lowercase @@ -319,6 +319,7 @@ def add(self, relation): :param BaseRelation relation: The underlying relation. """ + flags = get_flags() cached = _CachedRelation(relation) fire_event_if( flags.LOG_CACHE_EVENTS, @@ -456,7 +457,7 @@ def rename(self, old, new): ref_key_2=_make_msg_from_ref_key(new), ) ) - + flags = get_flags() fire_event_if( flags.LOG_CACHE_EVENTS, lambda: CacheDumpGraph(before_after="before", action="rename", dump=self.dump_graph()), diff --git a/core/dbt/cli/__init__.py b/core/dbt/cli/__init__.py index e69de29bb2d..8dc5c408aa2 100644 --- a/core/dbt/cli/__init__.py +++ b/core/dbt/cli/__init__.py @@ -0,0 +1 @@ +from .main import cli as dbt_cli # noqa diff --git a/core/dbt/cli/context.py b/core/dbt/cli/context.py new file mode 100644 index 00000000000..b8f541b9ad8 --- /dev/null +++ b/core/dbt/cli/context.py @@ -0,0 +1,16 @@ +import click +from typing import Optional + +from dbt.cli.main import cli as dbt + + +def make_context(args, command=dbt) -> Optional[click.Context]: + try: + ctx = command.make_context(command.name, args) + except click.exceptions.Exit: + return None + + ctx.invoked_subcommand = ctx.protected_args[0] if ctx.protected_args else None + ctx.obj = {} + + return ctx diff --git a/core/dbt/cli/example.py b/core/dbt/cli/example.py new file mode 100644 index 00000000000..afa6820efc8 --- /dev/null +++ b/core/dbt/cli/example.py @@ -0,0 +1,20 @@ +from dbt.cli.main import dbtRunner +from dbt.config.runtime import load_profile, load_project + +if __name__ == "__main__": + project_dir = "/Users/chenyuli/git/jaffle_shop" + cli_args = ["run", "--project-dir", project_dir] + + # initialize the dbt runner + dbt = dbtRunner() + # run the command + res, success = dbt.invoke(cli_args) + + # preload profile and project + profile = load_profile(project_dir, {}, "testing-postgres") + project = load_project(project_dir, False, profile, {}) + + # initialize the runner with pre-loaded profile and project, you can also pass in a preloaded manifest + dbt = dbtRunner(profile=profile, project=project) + # run the command, this will use the pre-loaded profile and project instead of loading + res, success = dbt.invoke(cli_args) diff --git a/core/dbt/cli/flags.py b/core/dbt/cli/flags.py index 3593a69de84..e7f8f06d036 100644 --- a/core/dbt/cli/flags.py +++ b/core/dbt/cli/flags.py @@ -1,44 +1,187 @@ # TODO Move this to /core/dbt/flags.py when we're ready to break things import os +import sys from dataclasses import dataclass +from importlib import import_module from multiprocessing import get_context from pprint import pformat as pf +from typing import Set, List -from click import get_current_context +from click import Context, get_current_context, BadOptionUsage +from click.core import ParameterSource + +from dbt.config.profile import read_user_config +from dbt.contracts.project import UserConfig +from dbt.helper_types import WarnErrorOptions +from dbt.config.project import PartialProject +from dbt.exceptions import DbtProjectError if os.name != "nt": # https://bugs.python.org/issue41567 import multiprocessing.popen_spawn_posix # type: ignore # noqa: F401 +# TODO anything that has a default in params should be removed here? +# Or maybe only the ones that's in the root click group +FLAGS_DEFAULTS = { + "INDIRECT_SELECTION": "eager", + "TARGET_PATH": None, + # cli args without user_config or env var option + "FULL_REFRESH": False, + "STRICT_MODE": False, + "STORE_FAILURES": False, +} + + +# For backwards compatability, some params are defined across multiple levels, +# Top-level value should take precedence. +# e.g. dbt --target-path test2 run --target-path test2 +EXPECTED_DUPLICATE_PARAMS = [ + "full_refresh", + "target_path", + "version_check", + "fail_fast", + "indirect_selection", + "store_failures", +] + + +def convert_config(config_name, config_value): + # This function should take care of converting the values from config and original + # set_from_args to the correct type + ret = config_value + if config_name.lower() == "warn_error_options": + ret = WarnErrorOptions( + include=config_value.get("include", []), exclude=config_value.get("exclude", []) + ) + return ret + @dataclass(frozen=True) class Flags: - def __init__(self, ctx=None) -> None: + def __init__(self, ctx: Context = None, user_config: UserConfig = None) -> None: + + # set the default flags + for key, value in FLAGS_DEFAULTS.items(): + object.__setattr__(self, key, value) if ctx is None: ctx = get_current_context() - def assign_params(ctx): + def assign_params(ctx, params_assigned_from_default): """Recursively adds all click params to flag object""" for param_name, param_value in ctx.params.items(): + # TODO: this is to avoid duplicate params being defined in two places (version_check in run and cli) + # However this is a bit of a hack and we should find a better way to do this + # N.B. You have to use the base MRO method (object.__setattr__) to set attributes # when using frozen dataclasses. # https://docs.python.org/3/library/dataclasses.html#frozen-instances - if hasattr(self, param_name): - raise Exception(f"Duplicate flag names found in click command: {param_name}") - object.__setattr__(self, param_name.upper(), param_value) + if hasattr(self, param_name.upper()): + if param_name not in EXPECTED_DUPLICATE_PARAMS: + raise Exception( + f"Duplicate flag names found in click command: {param_name}" + ) + else: + # Expected duplicate param from multi-level click command (ex: dbt --full_refresh run --full_refresh) + # Overwrite user-configured param with value from parent context + if ctx.get_parameter_source(param_name) != ParameterSource.DEFAULT: + object.__setattr__(self, param_name.upper(), param_value) + else: + object.__setattr__(self, param_name.upper(), param_value) + if ctx.get_parameter_source(param_name) == ParameterSource.DEFAULT: + params_assigned_from_default.add(param_name) + if ctx.parent: - assign_params(ctx.parent) + assign_params(ctx.parent, params_assigned_from_default) - assign_params(ctx) + params_assigned_from_default = set() # type: Set[str] + assign_params(ctx, params_assigned_from_default) + + # Get the invoked command flags + invoked_subcommand_name = ( + ctx.invoked_subcommand if hasattr(ctx, "invoked_subcommand") else None + ) + if invoked_subcommand_name is not None: + invoked_subcommand = getattr(import_module("dbt.cli.main"), invoked_subcommand_name) + invoked_subcommand.allow_extra_args = True + invoked_subcommand.ignore_unknown_options = True + invoked_subcommand_ctx = invoked_subcommand.make_context(None, sys.argv) + assign_params(invoked_subcommand_ctx, params_assigned_from_default) + + if not user_config: + profiles_dir = getattr(self, "PROFILES_DIR", None) + user_config = read_user_config(profiles_dir) if profiles_dir else None + + # Overwrite default assignments with user config if available + if user_config: + param_assigned_from_default_copy = params_assigned_from_default.copy() + for param_assigned_from_default in params_assigned_from_default: + user_config_param_value = getattr(user_config, param_assigned_from_default, None) + if user_config_param_value is not None: + object.__setattr__( + self, + param_assigned_from_default.upper(), + convert_config(param_assigned_from_default, user_config_param_value), + ) + param_assigned_from_default_copy.remove(param_assigned_from_default) + params_assigned_from_default = param_assigned_from_default_copy # Hard coded flags - object.__setattr__(self, "WHICH", ctx.info_name) + object.__setattr__(self, "WHICH", invoked_subcommand_name or ctx.info_name) object.__setattr__(self, "MP_CONTEXT", get_context("spawn")) + # Default LOG_PATH from PROJECT_DIR, if available. + if getattr(self, "LOG_PATH", None) is None: + log_path = "logs" + project_dir = getattr(self, "PROJECT_DIR", None) + # If available, set LOG_PATH from log-path in dbt_project.yml + # Known limitations: + # 1. Using PartialProject here, so no jinja rendering of log-path. + # 2. Programmatic invocations of the cli via dbtRunner may pass a Project object directly, + # which is not being used here to extract log-path. + if project_dir: + try: + partial = PartialProject.from_project_root( + project_dir, verify_version=getattr(self, "VERSION_CHECK", True) + ) + log_path = str(partial.project_dict.get("log-path", log_path)) + except DbtProjectError: + pass + + object.__setattr__(self, "LOG_PATH", log_path) + # Support console DO NOT TRACK initiave - if os.getenv("DO_NOT_TRACK", "").lower() in (1, "t", "true", "y", "yes"): - object.__setattr__(self, "ANONYMOUS_USAGE_STATS", False) + if os.getenv("DO_NOT_TRACK", "").lower() in ("1", "t", "true", "y", "yes"): + object.__setattr__(self, "SEND_ANONYMOUS_USAGE_STATS", False) + + # Check mutual exclusivity once all flags are set + self._assert_mutually_exclusive( + params_assigned_from_default, ["WARN_ERROR", "WARN_ERROR_OPTIONS"] + ) + + # Support lower cased access for legacy code + params = set( + x for x in dir(self) if not callable(getattr(self, x)) and not x.startswith("__") + ) + for param in params: + object.__setattr__(self, param.lower(), getattr(self, param)) def __str__(self) -> str: return str(pf(self.__dict__)) + + def _assert_mutually_exclusive( + self, params_assigned_from_default: Set[str], group: List[str] + ) -> None: + """ + Ensure no elements from group are simultaneously provided by a user, as inferred from params_assigned_from_default. + Raises click.UsageError if any two elements from group are simultaneously provided by a user. + """ + set_flag = None + for flag in group: + flag_set_by_user = flag.lower() not in params_assigned_from_default + if flag_set_by_user and set_flag: + raise BadOptionUsage( + flag.lower(), f"{flag.lower()}: not allowed with argument {set_flag.lower()}" + ) + elif flag_set_by_user: + set_flag = flag diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 9942db702ca..2e0ba71849b 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -1,22 +1,61 @@ -import inspect # This is temporary for RAT-ing from copy import copy -from pprint import pformat as pf # This is temporary for RAT-ing +from typing import List, Tuple, Optional import click -from dbt.adapters.factory import adapter_management -from dbt.cli import params as p -from dbt.cli.flags import Flags -from dbt.profiler import profiler - - -def cli_runner(): - # Alias "list" to "ls" - ls = copy(cli.commands["list"]) - ls.hidden = True - cli.add_command(ls, "ls") - - # Run the cli - cli() +from dbt.cli import requires, params as p +from dbt.config.project import Project +from dbt.config.profile import Profile +from dbt.contracts.graph.manifest import Manifest +from dbt.task.clean import CleanTask +from dbt.task.compile import CompileTask +from dbt.task.deps import DepsTask +from dbt.task.debug import DebugTask +from dbt.task.run import RunTask +from dbt.task.serve import ServeTask +from dbt.task.test import TestTask +from dbt.task.snapshot import SnapshotTask +from dbt.task.seed import SeedTask +from dbt.task.list import ListTask +from dbt.task.freshness import FreshnessTask +from dbt.task.run_operation import RunOperationTask +from dbt.task.build import BuildTask +from dbt.task.generate import GenerateTask +from dbt.task.init import InitTask + + +class dbtUsageException(Exception): + pass + + +class dbtInternalException(Exception): + pass + + +# Programmatic invocation +class dbtRunner: + def __init__( + self, project: Project = None, profile: Profile = None, manifest: Manifest = None + ): + self.project = project + self.profile = profile + self.manifest = manifest + + def invoke(self, args: List[str]) -> Tuple[Optional[List], bool]: + try: + dbt_ctx = cli.make_context(cli.name, args) + dbt_ctx.obj = { + "project": self.project, + "profile": self.profile, + "manifest": self.manifest, + } + return cli.invoke(dbt_ctx) + except click.exceptions.Exit as e: + # 0 exit code, expected for --version early exit + if str(e) == "0": + return [], True + raise dbtInternalException(f"unhandled exit code {str(e)}") + except (click.NoSuchOption, click.UsageError) as e: + raise dbtUsageException(e.message) # dbt @@ -27,19 +66,21 @@ def cli_runner(): epilog="Specify one of these sub-commands and you can find more help from there.", ) @click.pass_context -@p.anonymous_usage_stats +@p.send_anonymous_usage_stats @p.cache_selected_only @p.debug @p.enable_legacy_logger @p.fail_fast @p.log_cache_events @p.log_format +@p.log_path @p.macro_debugging @p.partial_parse @p.print @p.printer_width @p.quiet @p.record_timing_info +@p.single_threaded @p.static_parser @p.use_colors @p.use_experimental_parser @@ -52,21 +93,6 @@ def cli(ctx, **kwargs): """An ELT tool for managing your SQL transformations and data models. For more documentation on these commands, visit: docs.getdbt.com """ - incomplete_flags = Flags() - - # Profiling - if incomplete_flags.RECORD_TIMING_INFO: - ctx.with_resource(profiler(enable=True, outfile=incomplete_flags.RECORD_TIMING_INFO)) - - # Adapter management - ctx.with_resource(adapter_management()) - - # Version info - if incomplete_flags.VERSION: - click.echo(f"`version` called\n ctx.params: {pf(ctx.params)}") - return - else: - del ctx.params["version"] # dbt build @@ -75,13 +101,14 @@ def cli(ctx, **kwargs): @p.defer @p.exclude @p.fail_fast +@p.favor_state @p.full_refresh @p.indirect_selection -@p.log_path -@p.models @p.profile @p.profiles_dir @p.project_dir +@p.resource_type +@p.select @p.selector @p.show @p.state @@ -91,10 +118,22 @@ def cli(ctx, **kwargs): @p.threads @p.vars @p.version_check +@requires.preflight +@requires.profile +@requires.project +@requires.runtime_config +@requires.manifest def build(ctx, **kwargs): """Run all Seeds, Models, Snapshots, and tests in DAG order""" - flags = Flags() - click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") + task = BuildTask( + ctx.obj["flags"], + ctx.obj["runtime_config"], + ctx.obj["manifest"], + ) + + results = task.run() + success = task.interpret_results(results) + return results, success # dbt clean @@ -105,10 +144,16 @@ def build(ctx, **kwargs): @p.project_dir @p.target @p.vars +@requires.preflight +@requires.unset_profile +@requires.project def clean(ctx, **kwargs): """Delete all folders in the clean-targets list (usually the dbt_packages and target directories.)""" - flags = Flags() - click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") + task = CleanTask(ctx.obj["flags"], ctx.obj["project"]) + + results = task.run() + success = task.interpret_results(results) + return results, success # dbt docs @@ -124,11 +169,11 @@ def docs(ctx, **kwargs): @p.compile_docs @p.defer @p.exclude -@p.log_path -@p.models +@p.favor_state @p.profile @p.profiles_dir @p.project_dir +@p.select @p.selector @p.state @p.target @@ -136,10 +181,22 @@ def docs(ctx, **kwargs): @p.threads @p.vars @p.version_check +@requires.preflight +@requires.profile +@requires.project +@requires.runtime_config +@requires.manifest(write=False) def docs_generate(ctx, **kwargs): """Generate the documentation website for your project""" - flags = Flags() - click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") + task = GenerateTask( + ctx.obj["flags"], + ctx.obj["runtime_config"], + ctx.obj["manifest"], + ) + + results = task.run() + success = task.interpret_results(results) + return results, success # dbt docs serve @@ -152,10 +209,22 @@ def docs_generate(ctx, **kwargs): @p.project_dir @p.target @p.vars +@requires.preflight +@requires.profile +@requires.project +@requires.runtime_config +@requires.manifest def docs_serve(ctx, **kwargs): """Serve the documentation website for your project""" - flags = Flags() - click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") + task = ServeTask( + ctx.obj["flags"], + ctx.obj["runtime_config"], + ctx.obj["manifest"], + ) + + results = task.run() + success = task.interpret_results(results) + return results, success # dbt compile @@ -163,13 +232,13 @@ def docs_serve(ctx, **kwargs): @click.pass_context @p.defer @p.exclude +@p.favor_state @p.full_refresh -@p.log_path -@p.models @p.parse_only @p.profile @p.profiles_dir @p.project_dir +@p.select @p.selector @p.state @p.target @@ -177,10 +246,23 @@ def docs_serve(ctx, **kwargs): @p.threads @p.vars @p.version_check +@requires.preflight +@requires.profile +@requires.project +@requires.runtime_config +@requires.manifest def compile(ctx, **kwargs): - """Generates executable SQL from source, model, test, and analysis files. Compiled SQL files are written to the target/ directory.""" - flags = Flags() - click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") + """Generates executable SQL from source, model, test, and analysis files. Compiled SQL files are written to the + target/ directory.""" + task = CompileTask( + ctx.obj["flags"], + ctx.obj["runtime_config"], + ctx.obj["manifest"], + ) + + results = task.run() + success = task.interpret_results(results) + return results, success # dbt debug @@ -188,15 +270,22 @@ def compile(ctx, **kwargs): @click.pass_context @p.config_dir @p.profile -@p.profiles_dir +@p.profiles_dir_exists_false @p.project_dir @p.target @p.vars @p.version_check +@requires.preflight def debug(ctx, **kwargs): """Show some helpful information about dbt for debugging. Not to be confused with the --debug option which increases verbosity.""" - flags = Flags() - click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") + task = DebugTask( + ctx.obj["flags"], + None, + ) + + results = task.run() + success = task.interpret_results(results) + return results, success # dbt deps @@ -207,25 +296,36 @@ def debug(ctx, **kwargs): @p.project_dir @p.target @p.vars +@requires.preflight +@requires.unset_profile +@requires.project def deps(ctx, **kwargs): """Pull the most recent version of the dependencies listed in packages.yml""" - flags = Flags() - click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") + task = DepsTask(ctx.obj["flags"], ctx.obj["project"]) + results = task.run() + success = task.interpret_results(results) + return results, success # dbt init @cli.command("init") @click.pass_context +# for backwards compatibility, accept 'project_name' as an optional positional argument +@click.argument("project_name", required=False) @p.profile @p.profiles_dir @p.project_dir @p.skip_profile_setup @p.target @p.vars +@requires.preflight def init(ctx, **kwargs): - """Initialize a new DBT project.""" - flags = Flags() - click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") + """Initialize a new dbt project.""" + task = InitTask(ctx.obj["flags"], None) + + results = task.run() + success = task.interpret_results(results) + return results, success # dbt list @@ -240,21 +340,39 @@ def init(ctx, **kwargs): @p.profiles_dir @p.project_dir @p.resource_type +@p.raw_select @p.selector @p.state @p.target @p.vars +@requires.preflight +@requires.profile +@requires.project +@requires.runtime_config +@requires.manifest def list(ctx, **kwargs): """List the resources in your project""" - flags = Flags() - click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") + task = ListTask( + ctx.obj["flags"], + ctx.obj["runtime_config"], + ctx.obj["manifest"], + ) + + results = task.run() + success = task.interpret_results(results) + return results, success + + +# Alias "list" to "ls" +ls = copy(cli.commands["list"]) +ls.hidden = True +cli.add_command(ls, "ls") # dbt parse @cli.command("parse") @click.pass_context @p.compile_parse -@p.log_path @p.profile @p.profiles_dir @p.project_dir @@ -264,24 +382,29 @@ def list(ctx, **kwargs): @p.vars @p.version_check @p.write_manifest +@requires.preflight +@requires.profile +@requires.project +@requires.runtime_config +@requires.manifest(write_perf_info=True) def parse(ctx, **kwargs): """Parses the project and provides information on performance""" - flags = Flags() - click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") + # manifest generation and writing happens in @requires.manifest + return None, True # dbt run @cli.command("run") @click.pass_context @p.defer +@p.favor_state @p.exclude @p.fail_fast @p.full_refresh -@p.log_path -@p.models @p.profile @p.profiles_dir @p.project_dir +@p.select @p.selector @p.state @p.target @@ -289,25 +412,50 @@ def parse(ctx, **kwargs): @p.threads @p.vars @p.version_check +@requires.preflight +@requires.profile +@requires.project +@requires.runtime_config +@requires.manifest def run(ctx, **kwargs): """Compile SQL and execute against the current target database.""" - flags = Flags() - click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") + task = RunTask( + ctx.obj["flags"], + ctx.obj["runtime_config"], + ctx.obj["manifest"], + ) + + results = task.run() + success = task.interpret_results(results) + return results, success # dbt run operation @cli.command("run-operation") @click.pass_context +@click.argument("macro") @p.args @p.profile @p.profiles_dir @p.project_dir @p.target @p.vars +@requires.preflight +@requires.profile +@requires.project +@requires.runtime_config +@requires.manifest def run_operation(ctx, **kwargs): """Run the named macro with any supplied arguments.""" - flags = Flags() - click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") + task = RunOperationTask( + ctx.obj["flags"], + ctx.obj["runtime_config"], + ctx.obj["manifest"], + ) + + results = task.run() + success = task.interpret_results(results) + return results, success # dbt seed @@ -315,11 +463,10 @@ def run_operation(ctx, **kwargs): @click.pass_context @p.exclude @p.full_refresh -@p.log_path -@p.models @p.profile @p.profiles_dir @p.project_dir +@p.select @p.selector @p.show @p.state @@ -328,10 +475,21 @@ def run_operation(ctx, **kwargs): @p.threads @p.vars @p.version_check +@requires.preflight +@requires.profile +@requires.project +@requires.runtime_config +@requires.manifest def seed(ctx, **kwargs): """Load data from csv files into your data warehouse.""" - flags = Flags() - click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") + task = SeedTask( + ctx.obj["flags"], + ctx.obj["runtime_config"], + ctx.obj["manifest"], + ) + results = task.run() + success = task.interpret_results(results) + return results, success # dbt snapshot @@ -339,19 +497,32 @@ def seed(ctx, **kwargs): @click.pass_context @p.defer @p.exclude -@p.models +@p.favor_state @p.profile @p.profiles_dir @p.project_dir +@p.select @p.selector @p.state @p.target @p.threads @p.vars +@requires.preflight +@requires.profile +@requires.project +@requires.runtime_config +@requires.manifest def snapshot(ctx, **kwargs): """Execute snapshots defined in your project""" - flags = Flags() - click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") + task = SnapshotTask( + ctx.obj["flags"], + ctx.obj["runtime_config"], + ctx.obj["manifest"], + ) + + results = task.run() + success = task.interpret_results(results) + return results, success # dbt source @@ -365,20 +536,38 @@ def source(ctx, **kwargs): @source.command("freshness") @click.pass_context @p.exclude -@p.models @p.output_path # TODO: Is this ok to re-use? We have three different output params, how much can we consolidate? @p.profile @p.profiles_dir @p.project_dir +@p.select @p.selector @p.state @p.target @p.threads @p.vars +@requires.preflight +@requires.profile +@requires.project +@requires.runtime_config +@requires.manifest def freshness(ctx, **kwargs): - """Snapshots the current freshness of the project's sources""" - flags = Flags() - click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") + """check the current freshness of the project's sources""" + task = FreshnessTask( + ctx.obj["flags"], + ctx.obj["runtime_config"], + ctx.obj["manifest"], + ) + + results = task.run() + success = task.interpret_results(results) + return results, success + + +# Alias "source freshness" to "snapshot-freshness" +snapshot_freshness = copy(cli.commands["source"].commands["freshness"]) # type: ignore +snapshot_freshness.hidden = True +cli.commands["source"].add_command(snapshot_freshness, "snapshot-freshness") # type: ignore # dbt test @@ -387,12 +576,12 @@ def freshness(ctx, **kwargs): @p.defer @p.exclude @p.fail_fast +@p.favor_state @p.indirect_selection -@p.log_path -@p.models @p.profile @p.profiles_dir @p.project_dir +@p.select @p.selector @p.state @p.store_failures @@ -401,12 +590,24 @@ def freshness(ctx, **kwargs): @p.threads @p.vars @p.version_check +@requires.preflight +@requires.profile +@requires.project +@requires.runtime_config +@requires.manifest def test(ctx, **kwargs): """Runs tests on data in deployed models. Run this after `dbt run`""" - flags = Flags() - click.echo(f"`{inspect.stack()[0][3]}` called\n flags: {flags}") + task = TestTask( + ctx.obj["flags"], + ctx.obj["runtime_config"], + ctx.obj["manifest"], + ) + + results = task.run() + success = task.interpret_results(results) + return results, success # Support running as a module if __name__ == "__main__": - cli_runner() + cli() diff --git a/core/dbt/cli/option_types.py b/core/dbt/cli/option_types.py index f0c497b5bec..b5eea995624 100644 --- a/core/dbt/cli/option_types.py +++ b/core/dbt/cli/option_types.py @@ -1,5 +1,7 @@ -from click import ParamType -import yaml +from click import ParamType, Choice + +from dbt.config.utils import parse_cli_vars +from dbt.exceptions import ValidationError from dbt.helper_types import WarnErrorOptions @@ -14,8 +16,8 @@ def convert(self, value, param, ctx): if not isinstance(value, str): self.fail(f"Cannot load YAML from type {type(value)}", param, ctx) try: - return yaml.load(value, Loader=yaml.Loader) - except yaml.parser.ParserError: + return parse_cli_vars(value) + except ValidationError: self.fail(f"String '{value}' is not valid YAML", param, ctx) @@ -25,6 +27,7 @@ class WarnErrorOptionsType(YAML): name = "WarnErrorOptionsType" def convert(self, value, param, ctx): + # this function is being used by param in click include_exclude = super().convert(value, param, ctx) return WarnErrorOptions( @@ -46,3 +49,13 @@ def convert(self, value, param, ctx): return None else: return value + + +class ChoiceTuple(Choice): + name = "CHOICE_TUPLE" + + def convert(self, value, param, ctx): + for value_item in value: + super().convert(value_item, param, ctx) + + return value diff --git a/core/dbt/cli/options.py b/core/dbt/cli/options.py new file mode 100644 index 00000000000..4ac705dc140 --- /dev/null +++ b/core/dbt/cli/options.py @@ -0,0 +1,44 @@ +import click + + +# Implementation from: https://stackoverflow.com/a/48394004 +# Note MultiOption options must be specified with type=tuple or type=ChoiceTuple (https://github.com/pallets/click/issues/2012) +class MultiOption(click.Option): + def __init__(self, *args, **kwargs): + self.save_other_options = kwargs.pop("save_other_options", True) + nargs = kwargs.pop("nargs", -1) + assert nargs == -1, "nargs, if set, must be -1 not {}".format(nargs) + super(MultiOption, self).__init__(*args, **kwargs) + self._previous_parser_process = None + self._eat_all_parser = None + + def add_to_parser(self, parser, ctx): + def parser_process(value, state): + # method to hook to the parser.process + done = False + value = [value] + if self.save_other_options: + # grab everything up to the next option + while state.rargs and not done: + for prefix in self._eat_all_parser.prefixes: + if state.rargs[0].startswith(prefix): + done = True + if not done: + value.append(state.rargs.pop(0)) + else: + # grab everything remaining + value += state.rargs + state.rargs[:] = [] + value = tuple(value) + # call the actual process + self._previous_parser_process(value, state) + + retval = super(MultiOption, self).add_to_parser(parser, ctx) + for name in self.opts: + our_parser = parser._long_opt.get(name) or parser._short_opt.get(name) + if our_parser: + self._eat_all_parser = our_parser + self._previous_parser_process = our_parser.process + our_parser.process = parser_process + break + return retval diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index 3ad3747e962..ec3c1107fa9 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -1,17 +1,15 @@ from pathlib import Path, PurePath import click -from dbt.cli.option_types import YAML, WarnErrorOptionsType +from dbt.cli.options import MultiOption +from dbt.cli.option_types import YAML, ChoiceTuple, WarnErrorOptionsType from dbt.cli.resolvers import default_project_dir, default_profiles_dir +from dbt.version import get_version_information - -# TODO: The name (reflected in flags) is a correction! -# The original name was `SEND_ANONYMOUS_USAGE_STATS` and used an env var called "DBT_SEND_ANONYMOUS_USAGE_STATS" -# Both of which break existing naming conventions (doesn't match param flag). -# This will need to be fixed before use in the main codebase and communicated as a change to the community! -anonymous_usage_stats = click.option( - "--anonymous-usage-stats/--no-anonymous-usage-stats", - envvar="DBT_ANONYMOUS_USAGE_STATS", +# TODO: Rename this to meet naming conventions (the word "send" is redundant) +send_anonymous_usage_stats = click.option( + "--send-anonymous-usage-stats/--no-send-anonymous-usage-stats", + envvar="DBT_SEND_ANONYMOUS_USAGE_STATS", help="Send anonymous usage stats to dbt Labs.", default=True, ) @@ -80,7 +78,9 @@ hidden=True, ) -exclude = click.option("--exclude", envvar=None, help="Specify the nodes to exclude.") +exclude = click.option( + "--exclude", envvar=None, type=tuple, cls=MultiOption, help="Specify the nodes to exclude." +) fail_fast = click.option( "--fail-fast/--no-fail-fast", @@ -89,6 +89,12 @@ help="Stop execution on first failure.", ) +favor_state = click.option( + "--favor-state/--no-favor-state", + envvar="DBT_FAVOR_STATE", + help="If set, defer to the argument provided to the state flag for resolving unselected nodes, even if the node(s) exist as a database object in the current environment.", +) + full_refresh = click.option( "--full-refresh", "-f", @@ -101,7 +107,7 @@ "--indirect-selection", envvar="DBT_INDIRECT_SELECTION", help="Select all tests that are adjacent to selected resources, even if they those resources have been explicitly selected.", - type=click.Choice(["eager", "cautious"], case_sensitive=False), + type=click.Choice(["eager", "cautious", "buildable"], case_sensitive=False), default="eager", ) @@ -123,7 +129,8 @@ "--log-path", envvar="DBT_LOG_PATH", help="Configure the 'log-path'. Only applies this setting for the current run. Overrides the 'DBT_LOG_PATH' if it is set.", - type=click.Path(), + default=None, + type=click.Path(resolve_path=True, path_type=Path), ) macro_debugging = click.option( @@ -132,21 +139,12 @@ hidden=True, ) -models = click.option( - "-m", - "-s", - "models", - envvar=None, - help="Specify the nodes to include.", - multiple=True, -) - output = click.option( "--output", envvar=None, help="TODO: No current help text", type=click.Choice(["json", "name", "path", "selector"], case_sensitive=False), - default="name", + default="selector", ) output_keys = click.option( @@ -213,15 +211,24 @@ "--profiles-dir", envvar="DBT_PROFILES_DIR", help="Which directory to look in for the profiles.yml file. If not set, dbt will look in the current working directory first, then HOME/.dbt/", - default=default_profiles_dir(), + default=default_profiles_dir, type=click.Path(exists=True), ) +# `dbt debug` uses this because it implements custom behaviour for non-existent profiles.yml directories +profiles_dir_exists_false = click.option( + "--profiles-dir", + envvar="DBT_PROFILES_DIR", + help="Which directory to look in for the profiles.yml file. If not set, dbt will look in the current working directory first, then HOME/.dbt/", + default=default_profiles_dir, + type=click.Path(exists=False), +) + project_dir = click.option( "--project-dir", envvar=None, help="Which directory to look in for the dbt_project.yml file. Default is the current working directory and its parents.", - default=default_project_dir(), + default=default_project_dir, type=click.Path(exists=True), ) @@ -240,10 +247,11 @@ ) resource_type = click.option( + "--resource-types", "--resource-type", envvar=None, help="TODO: No current help text", - type=click.Choice( + type=ChoiceTuple( [ "metric", "source", @@ -258,8 +266,26 @@ ], case_sensitive=False, ), - default="default", -) + cls=MultiOption, + default=(), +) + +model_decls = ("-m", "--models", "--model") +select_decls = ("-s", "--select") +select_attrs = { + "envvar": None, + "help": "Specify the nodes to include.", + "cls": MultiOption, + "type": tuple, +} + +# `--select` and `--models` are analogous for most commands except `dbt list` for legacy reasons. +# Most CLI arguments should use the combined `select` option that aliases `--models` to `--select`. +# However, if you need to split out these separators (like `dbt ls`), use the `models` and `raw_select` options instead. +# See https://github.com/dbt-labs/dbt-core/pull/6774#issuecomment-1408476095 for more info. +models = click.option(*model_decls, **select_attrs) +raw_select = click.option(*select_decls, **select_attrs) +select = click.option(*select_decls, *model_decls, **select_attrs) selector = click.option( "--selector", envvar=None, help="The selector name to use, as defined in selectors.yml" @@ -269,6 +295,19 @@ "--show", envvar=None, help="Show a sample of the loaded data in the terminal", is_flag=True ) +# TODO: The env var is a correction! +# The original env var was `DBT_TEST_SINGLE_THREADED`. +# This broke the existing naming convention. +# This will need to be communicated as a change to the community! +# +# N.B. This flag is only used for testing, hence it's hidden from help text. +single_threaded = click.option( + "--single-threaded/--no-single-threaded", + envvar="DBT_SINGLE_THREADED", + default=False, + hidden=True, +) + skip_profile_setup = click.option( "--skip-profile-setup", "-s", envvar=None, help="Skip interactive profile setup.", is_flag=True ) @@ -283,10 +322,10 @@ help="If set, use the given directory as the source for json files to compare with this project.", type=click.Path( dir_okay=True, - exists=True, file_okay=False, readable=True, resolve_path=True, + path_type=Path, ), ) @@ -319,7 +358,7 @@ "--threads", envvar=None, help="Specify number of threads to use while executing models. Overrides settings in profiles.yml.", - default=1, + default=None, type=click.INT, ) @@ -341,12 +380,26 @@ envvar=None, help="Supply variables to the project. This argument overrides variables defined in your dbt_project.yml file. This argument should be a YAML string, eg. '{my_variable: my_value}'", type=YAML(), + default="{}", ) + +# TODO: when legacy flags are deprecated use +# click.version_option instead of a callback +def _version_callback(ctx, _param, value): + if not value or ctx.resilient_parsing: + return + click.echo(get_version_information()) + ctx.exit() + + version = click.option( "--version", + callback=_version_callback, envvar=None, + expose_value=False, help="Show version information", + is_eager=True, is_flag=True, ) @@ -362,13 +415,13 @@ envvar="DBT_WARN_ERROR", help="If dbt would normally warn, instead raise an exception. Examples include --select that selects nothing, deprecations, configurations with no associated models, invalid test configurations, and missing sources/refs in tests.", default=None, - flag_value=True, + is_flag=True, ) warn_error_options = click.option( "--warn-error-options", envvar="DBT_WARN_ERROR_OPTIONS", - default=None, + default="{}", help="""If dbt would normally warn, instead raise an exception based on include/exclude configuration. Examples include --select that selects nothing, deprecations, configurations with no associated models, invalid test configurations, and missing sources/refs in tests. This argument should be a YAML string, with keys 'include' or 'exclude'. eg. '{"include": "all", "exclude": ["NoNodesForSelectionCriteria"]}'""", type=WarnErrorOptionsType(), diff --git a/core/dbt/cli/requires.py b/core/dbt/cli/requires.py new file mode 100644 index 00000000000..b8fcec358b9 --- /dev/null +++ b/core/dbt/cli/requires.py @@ -0,0 +1,174 @@ +from dbt.adapters.factory import adapter_management, register_adapter +from dbt.flags import set_flags +from dbt.cli.flags import Flags +from dbt.config import RuntimeConfig +from dbt.config.runtime import load_project, load_profile, UnsetProfile +from dbt.events.functions import setup_event_logger +from dbt.exceptions import DbtProjectError +from dbt.parser.manifest import ManifestLoader, write_manifest +from dbt.profiler import profiler +from dbt.tracking import initialize_from_flags, track_run + +from click import Context +from functools import update_wrapper + + +def preflight(func): + def wrapper(*args, **kwargs): + ctx = args[0] + assert isinstance(ctx, Context) + ctx.obj = ctx.obj or {} + + # Flags + flags = Flags(ctx) + ctx.obj["flags"] = flags + set_flags(flags) + + # Tracking + initialize_from_flags(flags.SEND_ANONYMOUS_USAGE_STATS, flags.PROFILES_DIR) + ctx.with_resource(track_run(run_command=flags.WHICH)) + + # Logging + # N.B. Legacy logger is not supported + setup_event_logger( + flags.LOG_PATH, + flags.LOG_FORMAT, + flags.USE_COLORS, + flags.DEBUG, + ) + + # Profiling + if flags.RECORD_TIMING_INFO: + ctx.with_resource(profiler(enable=True, outfile=flags.RECORD_TIMING_INFO)) + + # Adapter management + ctx.with_resource(adapter_management()) + + return func(*args, **kwargs) + + return update_wrapper(wrapper, func) + + +# TODO: UnsetProfile is necessary for deps and clean to load a project. +# This decorator and its usage can be removed once https://github.com/dbt-labs/dbt-core/issues/6257 is closed. +def unset_profile(func): + def wrapper(*args, **kwargs): + ctx = args[0] + assert isinstance(ctx, Context) + + if ctx.obj.get("profile") is None: + profile = UnsetProfile() + ctx.obj["profile"] = profile + + return func(*args, **kwargs) + + return update_wrapper(wrapper, func) + + +def profile(func): + def wrapper(*args, **kwargs): + ctx = args[0] + assert isinstance(ctx, Context) + + if ctx.obj.get("profile") is None: + flags = ctx.obj["flags"] + # TODO: Generalize safe access to flags.THREADS: + # https://github.com/dbt-labs/dbt-core/issues/6259 + threads = getattr(flags, "THREADS", None) + profile = load_profile( + flags.PROJECT_DIR, flags.VARS, flags.PROFILE, flags.TARGET, threads + ) + ctx.obj["profile"] = profile + + return func(*args, **kwargs) + + return update_wrapper(wrapper, func) + + +def project(func): + def wrapper(*args, **kwargs): + ctx = args[0] + assert isinstance(ctx, Context) + + if ctx.obj.get("project") is None: + # TODO: Decouple target from profile, and remove the need for profile here: + # https://github.com/dbt-labs/dbt-core/issues/6257 + if not ctx.obj.get("profile"): + raise DbtProjectError("profile required for project") + + flags = ctx.obj["flags"] + project = load_project( + flags.PROJECT_DIR, flags.VERSION_CHECK, ctx.obj["profile"], flags.VARS + ) + ctx.obj["project"] = project + + return func(*args, **kwargs) + + return update_wrapper(wrapper, func) + + +def runtime_config(func): + """A decorator used by click command functions for generating a runtime + config given a profile and project. + """ + + def wrapper(*args, **kwargs): + ctx = args[0] + assert isinstance(ctx, Context) + + req_strs = ["profile", "project"] + reqs = [ctx.obj.get(req_str) for req_str in req_strs] + + if None in reqs: + raise DbtProjectError("profile and project required for runtime_config") + + ctx.obj["runtime_config"] = RuntimeConfig.from_parts( + ctx.obj["project"], + ctx.obj["profile"], + ctx.obj["flags"], + ) + + return func(*args, **kwargs) + + return update_wrapper(wrapper, func) + + +def manifest(*args0, write=True, write_perf_info=False): + """A decorator used by click command functions for generating a manifest + given a profile, project, and runtime config. This also registers the adaper + from the runtime config and conditionally writes the manifest to disc. + """ + + def outer_wrapper(func): + def wrapper(*args, **kwargs): + ctx = args[0] + assert isinstance(ctx, Context) + + req_strs = ["profile", "project", "runtime_config"] + reqs = [ctx.obj.get(dep) for dep in req_strs] + + if None in reqs: + raise DbtProjectError("profile, project, and runtime_config required for manifest") + + runtime_config = ctx.obj["runtime_config"] + register_adapter(runtime_config) + + # a manifest has already been set on the context, so don't overwrite it + if ctx.obj.get("manifest") is None: + manifest = ManifestLoader.get_full_manifest( + runtime_config, write_perf_info=write_perf_info + ) + + ctx.obj["manifest"] = manifest + if write and ctx.obj["flags"].write_json: + write_manifest(manifest, ctx.obj["runtime_config"].target_path) + + return func(*args, **kwargs) + + return update_wrapper(wrapper, func) + + # if there are no args, the decorator was used without params @decorator + # otherwise, the decorator was called with params @decorator(arg) + if len(args0) == 0: + return outer_wrapper + return outer_wrapper(args0[0]) diff --git a/core/dbt/clients/jinja.py b/core/dbt/clients/jinja.py index e9dcb45017b..ecf668c11e5 100644 --- a/core/dbt/clients/jinja.py +++ b/core/dbt/clients/jinja.py @@ -40,7 +40,7 @@ UndefinedCompilationError, UndefinedMacroError, ) -from dbt import flags +from dbt.flags import get_flags from dbt.node_types import ModelLanguage @@ -99,8 +99,9 @@ def _compile(self, source, filename): If the value is 'write', also write the files to disk. WARNING: This can write a ton of data if you aren't careful. """ - if filename == "