Skip to content

Commit

Permalink
fix(dbt): dbt test description extract (#19464)
Browse files Browse the repository at this point in the history
## Summary & Motivation
According to the problem [Adding descriptions to tests is no longer
possible from 0.20.0](dbt-labs/dbt-core#3599 )
and the source code
[SchemaGenericTestParser](https://github.com/dbt-labs/dbt-core/blob/main/core/dbt/parser/schema_generic_tests.py#L39)
the general test does not support the description in the schema.

This fix allows users to define a test description in `tests/<generic
test>/config/meta/description`, to display data in Dagster(UI) Asset
Check.

```yaml
models:
  - name: wildberries_incomes
    tests:
      - name: 'Equal_row_count_with_stg__wildberries_incomes'
        config: 
          meta:
            description: "Equal rows before join with mapping table"
        test_name: equal_rowcount
        compare_model: source('production', 'stg__wildberries_incomes')
        compare_model_final: True
```

Related: dbt-labs/dbt-core#2578


![Снимок экрана 2024-01-29 в 14 57
45](https://github.com/dagster-io/dagster/assets/34104577/2638a04c-935f-4368-b21b-04c5de740614)


## How I Tested These Changes
pytest

Co-authored-by: Rex Ledesma <[email protected]>
  • Loading branch information
2 people authored and OwenKephart committed Jan 31, 2024
1 parent c4d2358 commit dfe3e04
Show file tree
Hide file tree
Showing 4 changed files with 18 additions and 29 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,7 @@
from .utils import input_name_fn, output_name_fn

if TYPE_CHECKING:
from .dagster_dbt_translator import (
DagsterDbtTranslator,
DbtManifestWrapper,
)
from .dagster_dbt_translator import DagsterDbtTranslator, DbtManifestWrapper

MANIFEST_METADATA_KEY = "dagster_dbt/manifest"
DAGSTER_DBT_TRANSLATOR_METADATA_KEY = "dagster_dbt/dagster_dbt_translator"
Expand Down Expand Up @@ -557,7 +554,7 @@ def default_asset_check_fn(
return AssetCheckSpec(
name=test_resource_props["name"],
asset=asset_key,
description=test_resource_props["description"],
description=test_resource_props.get("meta", {}).get("description"),
additional_deps=additional_deps,
)

Expand Down

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,9 @@ models:
- accepted_values:
values:
["placed", "shipped", "completed", "return_pending", "returned"]
config:
meta:
description: "Status must be one of ['placed', 'shipped', 'completed', 'return_pending', or 'returned']"

- name: amount
description: Total amount (AUD) of the order
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,105 +73,94 @@ def my_dbt_assets_with_checks():
"customers_not_null_customers_customer_id": AssetCheckSpec(
name="not_null_customers_customer_id",
asset=AssetKey(["customers"]),
description="",
),
"customers_unique_customers_customer_id": AssetCheckSpec(
name="unique_customers_customer_id",
asset=AssetKey(["customers"]),
description="",
),
"orders_accepted_values_orders_status__placed__shipped__completed__return_pending__returned": AssetCheckSpec(
name="accepted_values_orders_status__placed__shipped__completed__return_pending__returned",
asset=AssetKey(["orders"]),
description="",
description="Status must be one of ['placed', 'shipped', 'completed', 'return_pending', or 'returned']",
),
"orders_not_null_orders_amount": AssetCheckSpec(
name="not_null_orders_amount", asset=AssetKey(["orders"]), description=""
name="not_null_orders_amount",
asset=AssetKey(["orders"]),
),
"orders_not_null_orders_bank_transfer_amount": AssetCheckSpec(
name="not_null_orders_bank_transfer_amount",
asset=AssetKey(["orders"]),
description="",
),
"orders_not_null_orders_coupon_amount": AssetCheckSpec(
name="not_null_orders_coupon_amount", asset=AssetKey(["orders"]), description=""
name="not_null_orders_coupon_amount",
asset=AssetKey(["orders"]),
),
"orders_not_null_orders_credit_card_amount": AssetCheckSpec(
name="not_null_orders_credit_card_amount",
asset=AssetKey(["orders"]),
description="",
),
"orders_not_null_orders_customer_id": AssetCheckSpec(
name="not_null_orders_customer_id", asset=AssetKey(["orders"]), description=""
name="not_null_orders_customer_id",
asset=AssetKey(["orders"]),
),
"orders_not_null_orders_gift_card_amount": AssetCheckSpec(
name="not_null_orders_gift_card_amount",
asset=AssetKey(["orders"]),
description="",
),
"orders_not_null_orders_order_id": AssetCheckSpec(
name="not_null_orders_order_id", asset=AssetKey(["orders"]), description=""
name="not_null_orders_order_id",
asset=AssetKey(["orders"]),
),
"orders_relationships_orders_customer_id__customer_id__ref_customers_": AssetCheckSpec(
name="relationships_orders_customer_id__customer_id__ref_customers_",
asset=AssetKey(["orders"]),
description="",
additional_deps=[
AssetKey(["customers"]),
],
),
"orders_unique_orders_order_id": AssetCheckSpec(
name="unique_orders_order_id", asset=AssetKey(["orders"]), description=""
name="unique_orders_order_id",
asset=AssetKey(["orders"]),
),
"stg_customers_not_null_stg_customers_customer_id": AssetCheckSpec(
name="not_null_stg_customers_customer_id",
asset=AssetKey(["stg_customers"]),
description="",
),
"stg_customers_unique_stg_customers_customer_id": AssetCheckSpec(
name="unique_stg_customers_customer_id",
asset=AssetKey(["stg_customers"]),
description="",
),
"stg_orders_accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned": AssetCheckSpec(
name="accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned",
asset=AssetKey(["stg_orders"]),
description="",
),
"stg_orders_not_null_stg_orders_order_id": AssetCheckSpec(
name="not_null_stg_orders_order_id",
asset=AssetKey(["stg_orders"]),
description="",
),
"stg_orders_unique_stg_orders_order_id": AssetCheckSpec(
name="unique_stg_orders_order_id",
asset=AssetKey(["stg_orders"]),
description="",
),
"stg_payments_accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card": AssetCheckSpec(
name="accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card",
asset=AssetKey(["stg_payments"]),
description="",
),
"stg_payments_not_null_stg_payments_payment_id": AssetCheckSpec(
name="not_null_stg_payments_payment_id",
asset=AssetKey(["stg_payments"]),
description="",
),
"stg_payments_unique_stg_payments_payment_id": AssetCheckSpec(
name="unique_stg_payments_payment_id",
asset=AssetKey(["stg_payments"]),
description="",
),
"fail_tests_model_accepted_values_fail_tests_model_first_name__foo__bar__baz": AssetCheckSpec(
name="accepted_values_fail_tests_model_first_name__foo__bar__baz",
asset=AssetKey(["fail_tests_model"]),
description="",
),
"fail_tests_model_unique_fail_tests_model_id": AssetCheckSpec(
name="unique_fail_tests_model_id",
asset=AssetKey(["fail_tests_model"]),
description="",
),
}

Expand Down Expand Up @@ -246,7 +235,7 @@ def my_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource):
def test_materialize_no_selection(dbt_commands: List[List[str]]) -> None:
result = _materialize_dbt_assets(dbt_commands, selection=None, raise_on_error=False)
assert not result.success # fail_tests_model fails
assert len(result.get_asset_materialization_events()) == 9
assert len(result.get_asset_materialization_events()) == 10
assert len(result.get_asset_check_evaluations()) == 22


Expand Down

0 comments on commit dfe3e04

Please sign in to comment.