diff --git a/core/dbt/artifacts/README.md b/core/dbt/artifacts/README.md index dd8990e28cd..fcb2360b7b9 100644 --- a/core/dbt/artifacts/README.md +++ b/core/dbt/artifacts/README.md @@ -46,9 +46,12 @@ These types of minor, non-breaking changes are tested by [tests/unit/artifacts/t #### Updating [schemas.getdbt.com](https://schemas.getdbt.com) Non-breaking changes to artifact schemas require an update to the corresponding jsonschemas published to [schemas.getdbt.com](https://schemas.getdbt.com), which are defined in https://github.com/dbt-labs/schemas.getdbt.com. To do so: -1. Create a PR in https://github.com/dbt-labs/schemas.getdbt.com which reflects the schema changes to the artifact. The schema can be updated in-place for non-breaking changes. Example PR: https://github.com/dbt-labs/schemas.getdbt.com/pull/39 -2. Merge the https://github.com/dbt-labs/schemas.getdbt.com PR -3. Observe the `Artifact Schema Check` CI check pass on the `dbt-core` PR that updates the artifact schemas, and merge the `dbt-core` PR! + +1. Run `scripts/collect-artifact-schema.py --path schemas --artifact [manifest | catalog | run-results | sources]` to generate the jsonschema inclusive of your changes. +2. Copy the generated schema into the corresponding artifact file in the schemas.getdbt.com repository. (e.g. `cp schemas/dbt/manifest/v12.json ../../schemas.getdbt.com/dbt/manifest/`) +3. Create a PR in https://github.com/dbt-labs/schemas.getdbt.com which reflects the schema changes to the artifact. The schema can be updated in-place for non-breaking changes. Example PR: https://github.com/dbt-labs/schemas.getdbt.com/pull/39 +4. Merge the https://github.com/dbt-labs/schemas.getdbt.com PR +5. Observe the `Artifact Schema Check` CI check pass on the `dbt-core` PR that updates the artifact schemas, and merge the `dbt-core` PR! Note: Although `jsonschema` validation using the schemas in [schemas.getdbt.com](https://schemas.getdbt.com) is not encouraged or formally supported, `jsonschema` validation should still continue to work once the schemas are updated because they are forward-compatible and can therefore be used to validate previous minor versions of the schema. diff --git a/core/dbt/artifacts/resources/v1/exposure.py b/core/dbt/artifacts/resources/v1/exposure.py index 00f3c8b89e1..9724afe366b 100644 --- a/core/dbt/artifacts/resources/v1/exposure.py +++ b/core/dbt/artifacts/resources/v1/exposure.py @@ -3,7 +3,7 @@ from typing import Any, Dict, List, Literal, Optional from dbt.artifacts.resources.base import GraphResource -from dbt.artifacts.resources.types import NodeType +from dbt.artifacts.resources.types import AccessType, NodeType from dbt.artifacts.resources.v1.components import DependsOn, RefArgs from dbt.artifacts.resources.v1.owner import Owner from dbt_common.contracts.config.base import BaseConfig @@ -47,3 +47,5 @@ class Exposure(GraphResource): sources: List[List[str]] = field(default_factory=list) metrics: List[List[str]] = field(default_factory=list) created_at: float = field(default_factory=lambda: time.time()) + access: AccessType = AccessType.Public + group: Optional[str] = None diff --git a/core/dbt/artifacts/resources/v1/source_definition.py b/core/dbt/artifacts/resources/v1/source_definition.py index ac0fcfca1b2..31701920c54 100644 --- a/core/dbt/artifacts/resources/v1/source_definition.py +++ b/core/dbt/artifacts/resources/v1/source_definition.py @@ -3,7 +3,7 @@ from typing import Any, Dict, List, Literal, Optional, Union from dbt.artifacts.resources.base import GraphResource -from dbt.artifacts.resources.types import NodeType +from dbt.artifacts.resources.types import AccessType, NodeType from dbt.artifacts.resources.v1.components import ( ColumnInfo, FreshnessThreshold, @@ -70,3 +70,5 @@ class SourceDefinition(ParsedSourceMandatory): unrendered_config: Dict[str, Any] = field(default_factory=dict) relation_name: Optional[str] = None created_at: float = field(default_factory=lambda: time.time()) + access: AccessType = AccessType.Protected + group: Optional[str] = None diff --git a/core/dbt/contracts/graph/manifest.py b/core/dbt/contracts/graph/manifest.py index 358ef84db63..317d3bc3a8f 100644 --- a/core/dbt/contracts/graph/manifest.py +++ b/core/dbt/contracts/graph/manifest.py @@ -643,6 +643,8 @@ class Disabled(Generic[D]): MaybeNonSource = Optional[Union[ManifestNode, Disabled[ManifestNode]]] +MaybeSource = Optional[Union[SourceDefinition, Disabled[SourceDefinition]]] + T = TypeVar("T", bound=GraphMemberNode) @@ -1467,6 +1469,36 @@ def is_invalid_protected_ref( node.package_name != target_model.package_name and restrict_package_access ) + def is_invalid_private_source(self, node: GraphMemberNode, target_source: MaybeSource) -> bool: + if not isinstance(target_source, SourceDefinition): + return False + + is_private_source = target_source.access == AccessType.Private + + return is_private_source and ( + not hasattr(node, "group") or not node.group or node.group != target_source.group + ) + + def is_invalid_protected_source( + self, node: GraphMemberNode, target_source: MaybeSource, dependencies: Optional[Mapping] + ) -> bool: + dependencies = dependencies or {} + if not isinstance(target_source, SourceDefinition): + return False + + is_protected_source = ( + target_source.access == AccessType.Protected + # don't raise this reference error for ad hoc 'preview' queries + and node.resource_type != NodeType.SqlOperation + and node.resource_type != NodeType.RPCCall # TODO: rm + ) + target_dependency = dependencies.get(target_source.package_name) + restrict_package_access = target_dependency.restrict_access if target_dependency else False + + return is_protected_source and ( + node.package_name != target_source.package_name and restrict_package_access + ) + # Called in GraphRunnableTask.before_run, RunTask.before_run, CloneTask.before_run def merge_from_artifact(self, other: "Manifest") -> None: """Update this manifest by adding the 'defer_relation' attribute to all nodes diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index da42fb7d766..7e399c3eca2 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -1232,6 +1232,9 @@ def same_config(self, old: "SourceDefinition") -> bool: old.unrendered_config, ) + def same_source_representation(self, old: "SourceDefinition") -> bool: + return self.access == old.access and self.group == old.group + def same_contents(self, old: Optional["SourceDefinition"]) -> bool: # existing when it didn't before is a change! if old is None: @@ -1252,6 +1255,7 @@ def same_contents(self, old: Optional["SourceDefinition"]) -> bool: and self.same_quoting(old) and self.same_freshness(old) and self.same_external(old) + and self.same_source_representation(old) and True ) @@ -1297,10 +1301,6 @@ def has_freshness(self) -> bool: def search_name(self): return f"{self.source_name}.{self.name}" - @property - def group(self): - return None - # ==================================== # Exposure node @@ -1342,6 +1342,9 @@ def same_exposure_type(self, old: "Exposure") -> bool: def same_url(self, old: "Exposure") -> bool: return self.url == old.url + def same_group(self, old: "Exposure") -> bool: + return self.group == old.group + def same_config(self, old: "Exposure") -> bool: return self.config.same_contents( self.unrendered_config, @@ -1364,13 +1367,10 @@ def same_contents(self, old: Optional["Exposure"]) -> bool: and self.same_label(old) and self.same_depends_on(old) and self.same_config(old) + and self.same_group(old) and True ) - @property - def group(self): - return None - # ==================================== # Metric node diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index a64223df3b6..683dd6ba90d 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -302,6 +302,8 @@ class UnparsedSourceDefinition(dbtClassMixin): tables: List[UnparsedSourceTableDefinition] = field(default_factory=list) tags: List[str] = field(default_factory=list) config: Dict[str, Any] = field(default_factory=dict) + group: Optional[str] = None + access: Optional[str] = None @classmethod def validate(cls, data): @@ -462,6 +464,7 @@ class UnparsedExposure(dbtClassMixin): url: Optional[str] = None depends_on: List[str] = field(default_factory=list) config: Dict[str, Any] = field(default_factory=dict) + group: Optional[str] = None @classmethod def validate(cls, data): diff --git a/core/dbt/parser/manifest.py b/core/dbt/parser/manifest.py index 7e37a0e5417..a67edaf2fc0 100644 --- a/core/dbt/parser/manifest.py +++ b/core/dbt/parser/manifest.py @@ -458,7 +458,7 @@ def load(self) -> Manifest: # These check the created_at time on the nodes to # determine whether they need processing. start_process = time.perf_counter() - self.process_sources(self.root_project.project_name) + self.process_sources(self.root_project.project_name, self.root_project.dependencies) self.process_refs(self.root_project.project_name, self.root_project.dependencies) self.process_unit_tests(self.root_project.project_name) self.process_docs(self.root_project) @@ -1246,18 +1246,18 @@ def process_docs(self, config: RuntimeConfig): # Loops through all nodes and exposures, for each element in # 'sources' array finds the source node and updates the # 'depends_on.nodes' array with the unique id - def process_sources(self, current_project: str): + def process_sources(self, current_project: str, dependencies: Optional[Mapping[str, Project]]): for node in self.manifest.nodes.values(): if node.resource_type == NodeType.Source: continue assert not isinstance(node, SourceDefinition) if node.created_at < self.started_at: continue - _process_sources_for_node(self.manifest, current_project, node) + _process_sources_for_node(self.manifest, current_project, node, dependencies) for exposure in self.manifest.exposures.values(): if exposure.created_at < self.started_at: continue - _process_sources_for_exposure(self.manifest, current_project, exposure) + _process_sources_for_exposure(self.manifest, current_project, exposure, dependencies) # Loops through all nodes, for each element in # 'unit_test' array finds the node and updates the @@ -1793,7 +1793,12 @@ def remove_dependent_project_references(manifest, external_node_unique_id): node.created_at = time.time() -def _process_sources_for_exposure(manifest: Manifest, current_project: str, exposure: Exposure): +def _process_sources_for_exposure( + manifest: Manifest, + current_project: str, + exposure: Exposure, + dependencies: Optional[Mapping[str, Project]], +): target_source: Optional[Union[Disabled, SourceDefinition]] = None for source_name, table_name in exposure.sources: target_source = manifest.resolve_source( @@ -1811,6 +1816,21 @@ def _process_sources_for_exposure(manifest: Manifest, current_project: str, expo disabled=(isinstance(target_source, Disabled)), ) continue + + if manifest.is_invalid_private_source(exposure, target_source): + raise dbt.exceptions.DbtReferenceError( + unique_id=exposure.unique_id, + ref_unique_id=target_source.unique_id, + access=AccessType.Private, + scope=dbt_common.utils.cast_to_str(target_source.group), + ) + elif manifest.is_invalid_protected_source(exposure, target_source, dependencies): + raise dbt.exceptions.DbtReferenceError( + unique_id=exposure.unique_id, + ref_unique_id=target_source.unique_id, + access=AccessType.Protected, + scope=exposure.package_name, + ) target_source_id = target_source.unique_id exposure.depends_on.add_node(target_source_id) @@ -1837,7 +1857,12 @@ def _process_sources_for_metric(manifest: Manifest, current_project: str, metric metric.depends_on.add_node(target_source_id) -def _process_sources_for_node(manifest: Manifest, current_project: str, node: ManifestNode): +def _process_sources_for_node( + manifest: Manifest, + current_project: str, + node: ManifestNode, + dependencies: Optional[Mapping[str, Project]], +): if isinstance(node, SeedNode): return @@ -1851,6 +1876,31 @@ def _process_sources_for_node(manifest: Manifest, current_project: str, node: Ma ) if target_source is None or isinstance(target_source, Disabled): + node.config.enabled = False + invalid_target_fail_unless_test( + node=node, + target_name=f"{source_name}.{table_name}", + target_kind="source", + disabled=(isinstance(target_source, Disabled)), + ) + continue + + if manifest.is_invalid_private_source(node, target_source): + raise dbt.exceptions.DbtReferenceError( + unique_id=node.unique_id, + ref_unique_id=target_source.unique_id, + access=AccessType.Private, + scope=dbt_common.utils.cast_to_str(target_source.group), + ) + elif manifest.is_invalid_protected_source(node, target_source, dependencies): + raise dbt.exceptions.DbtReferenceError( + unique_id=node.unique_id, + ref_unique_id=target_source.unique_id, + access=AccessType.Protected, + scope=node.package_name, + ) + + elif target_source is None or isinstance(target_source, Disabled): # this follows the same pattern as refs node.config.enabled = False invalid_target_fail_unless_test( @@ -1879,7 +1929,7 @@ def process_macro(config: RuntimeConfig, manifest: Manifest, macro: Macro) -> No # This is called in task.rpc.sql_commands when a "dynamic" node is # created in the manifest, in 'add_refs' def process_node(config: RuntimeConfig, manifest: Manifest, node: ManifestNode): - _process_sources_for_node(manifest, config.project_name, node) + _process_sources_for_node(manifest, config.project_name, node, config.dependencies) _process_refs(manifest, config.project_name, node, config.dependencies) ctx = generate_runtime_docs_context(config, node, manifest, config.project_name) _process_docs_for_node(ctx, node) diff --git a/core/dbt/parser/schema_yaml_readers.py b/core/dbt/parser/schema_yaml_readers.py index 20e0a44bc08..793358eb2ae 100644 --- a/core/dbt/parser/schema_yaml_readers.py +++ b/core/dbt/parser/schema_yaml_readers.py @@ -123,6 +123,7 @@ def parse_exposure(self, unparsed: UnparsedExposure) -> None: unique_id=unique_id, fqn=fqn, name=unparsed.name, + group=unparsed.group, type=unparsed.type, url=unparsed.url, meta=unparsed.meta, diff --git a/core/dbt/parser/sources.py b/core/dbt/parser/sources.py index 68dbed94ce5..cd8f1d9e973 100644 --- a/core/dbt/parser/sources.py +++ b/core/dbt/parser/sources.py @@ -6,6 +6,7 @@ from dbt.adapters.capability import Capability from dbt.adapters.factory import get_adapter from dbt.artifacts.resources import FreshnessThreshold, SourceConfig, Time +from dbt.artifacts.resources.types import AccessType from dbt.config import RuntimeConfig from dbt.context.context_config import ( BaseContextConfigGenerator, @@ -26,6 +27,7 @@ UnparsedSourceTableDefinition, ) from dbt.events.types import FreshnessConfigProblem, UnusedTables +from dbt.exceptions import InvalidAccessTypeError from dbt.node_types import NodeType from dbt.parser.common import ParserRef from dbt.parser.schema_generic_tests import SchemaGenericTestParser @@ -127,6 +129,19 @@ def parse_source(self, target: UnpatchedSourceDefinition) -> SourceDefinition: unique_id = target.unique_id description = table.description or "" source_description = source.description or "" + group = source.group or None + access = AccessType.Protected + # make sure sources are not public and are a valid access type + if source.access: + if ( + not AccessType.is_valid(source.access) + or AccessType(source.access) == AccessType.Public + ): + raise InvalidAccessTypeError( + unique_id=unique_id, + field_value=source.access, + ) + access = AccessType(source.access) # We need to be able to tell the difference between explicitly setting the loaded_at_field to None/null # and when it's simply not set. This allows a user to override the source level loaded_at_field so that @@ -190,6 +205,8 @@ def parse_source(self, target: UnpatchedSourceDefinition) -> SourceDefinition: tags=tags, config=config, unrendered_config=unrendered_config, + group=group, + access=access, ) if ( diff --git a/schemas/dbt/manifest/v12.json b/schemas/dbt/manifest/v12.json index 0ec6daae1f1..547bfe3b2bd 100644 --- a/schemas/dbt/manifest/v12.json +++ b/schemas/dbt/manifest/v12.json @@ -162,10 +162,7 @@ } }, "additionalProperties": false, - "required": [ - "name", - "checksum" - ] + "required": ["name", "checksum"] }, "config": { "type": "object", @@ -291,9 +288,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "pre-hook": { @@ -322,9 +317,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "quoting": { @@ -379,11 +372,7 @@ "default": "ignore" }, "on_configuration_change": { - "enum": [ - "apply", - "continue", - "fail" - ] + "enum": ["apply", "continue", "fail"] }, "grants": { "type": "object", @@ -540,9 +529,7 @@ } }, "additionalProperties": false, - "required": [ - "type" - ] + "required": ["type"] } }, "quote": { @@ -570,9 +557,7 @@ } }, "additionalProperties": true, - "required": [ - "name" - ] + "required": ["name"] }, "propertyNames": { "type": "string" @@ -901,9 +886,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "pre-hook": { @@ -932,9 +915,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "quoting": { @@ -989,11 +970,7 @@ "default": "ignore" }, "on_configuration_change": { - "enum": [ - "apply", - "continue", - "fail" - ] + "enum": ["apply", "continue", "fail"] }, "grants": { "type": "object", @@ -1146,10 +1123,7 @@ } }, "additionalProperties": false, - "required": [ - "name", - "checksum" - ] + "required": ["name", "checksum"] }, "config": { "type": "object", @@ -1275,9 +1249,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "pre-hook": { @@ -1306,9 +1278,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "quoting": { @@ -1363,11 +1333,7 @@ "default": "ignore" }, "on_configuration_change": { - "enum": [ - "apply", - "continue", - "fail" - ] + "enum": ["apply", "continue", "fail"] }, "grants": { "type": "object", @@ -1509,9 +1475,7 @@ } }, "additionalProperties": false, - "required": [ - "type" - ] + "required": ["type"] } }, "quote": { @@ -1539,9 +1503,7 @@ } }, "additionalProperties": true, - "required": [ - "name" - ] + "required": ["name"] }, "propertyNames": { "type": "string" @@ -1678,9 +1640,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] } }, "sources": { @@ -1764,10 +1724,7 @@ } }, "additionalProperties": false, - "required": [ - "id", - "sql" - ] + "required": ["id", "sql"] } }, "_pre_injected_sql": { @@ -1879,10 +1836,7 @@ } }, "additionalProperties": false, - "required": [ - "name", - "checksum" - ] + "required": ["name", "checksum"] }, "config": { "type": "object", @@ -2117,9 +2071,7 @@ } }, "additionalProperties": false, - "required": [ - "type" - ] + "required": ["type"] } }, "quote": { @@ -2147,9 +2099,7 @@ } }, "additionalProperties": true, - "required": [ - "name" - ] + "required": ["name"] }, "propertyNames": { "type": "string" @@ -2286,9 +2236,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] } }, "sources": { @@ -2372,10 +2320,7 @@ } }, "additionalProperties": false, - "required": [ - "id", - "sql" - ] + "required": ["id", "sql"] } }, "_pre_injected_sql": { @@ -2487,10 +2432,7 @@ } }, "additionalProperties": false, - "required": [ - "name", - "checksum" - ] + "required": ["name", "checksum"] }, "config": { "type": "object", @@ -2616,9 +2558,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "pre-hook": { @@ -2647,9 +2587,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "quoting": { @@ -2704,11 +2642,7 @@ "default": "ignore" }, "on_configuration_change": { - "enum": [ - "apply", - "continue", - "fail" - ] + "enum": ["apply", "continue", "fail"] }, "grants": { "type": "object", @@ -2850,9 +2784,7 @@ } }, "additionalProperties": false, - "required": [ - "type" - ] + "required": ["type"] } }, "quote": { @@ -2880,9 +2812,7 @@ } }, "additionalProperties": true, - "required": [ - "name" - ] + "required": ["name"] }, "propertyNames": { "type": "string" @@ -3019,9 +2949,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] } }, "sources": { @@ -3105,10 +3033,7 @@ } }, "additionalProperties": false, - "required": [ - "id", - "sql" - ] + "required": ["id", "sql"] } }, "_pre_injected_sql": { @@ -3231,10 +3156,7 @@ } }, "additionalProperties": false, - "required": [ - "name", - "checksum" - ] + "required": ["name", "checksum"] }, "config": { "type": "object", @@ -3360,9 +3282,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "pre-hook": { @@ -3391,9 +3311,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "quoting": { @@ -3448,11 +3366,7 @@ "default": "ignore" }, "on_configuration_change": { - "enum": [ - "apply", - "continue", - "fail" - ] + "enum": ["apply", "continue", "fail"] }, "grants": { "type": "object", @@ -3504,11 +3418,7 @@ "additionalProperties": false }, "access": { - "enum": [ - "private", - "protected", - "public" - ], + "enum": ["private", "protected", "public"], "default": "protected" } }, @@ -3602,9 +3512,7 @@ } }, "additionalProperties": false, - "required": [ - "type" - ] + "required": ["type"] } }, "quote": { @@ -3632,9 +3540,7 @@ } }, "additionalProperties": true, - "required": [ - "name" - ] + "required": ["name"] }, "propertyNames": { "type": "string" @@ -3771,9 +3677,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] } }, "sources": { @@ -3857,10 +3761,7 @@ } }, "additionalProperties": false, - "required": [ - "id", - "sql" - ] + "required": ["id", "sql"] } }, "_pre_injected_sql": { @@ -3901,11 +3802,7 @@ "additionalProperties": false }, "access": { - "enum": [ - "private", - "protected", - "public" - ], + "enum": ["private", "protected", "public"], "default": "protected" }, "constraints": { @@ -3962,9 +3859,7 @@ } }, "additionalProperties": false, - "required": [ - "type" - ] + "required": ["type"] } }, "version": { @@ -4214,9 +4109,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "pre-hook": { @@ -4245,9 +4138,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "quoting": { @@ -4302,11 +4193,7 @@ "default": "ignore" }, "on_configuration_change": { - "enum": [ - "apply", - "continue", - "fail" - ] + "enum": ["apply", "continue", "fail"] }, "grants": { "type": "object", @@ -4465,10 +4352,7 @@ } }, "additionalProperties": false, - "required": [ - "name", - "checksum" - ] + "required": ["name", "checksum"] }, "config": { "type": "object", @@ -4594,9 +4478,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "pre-hook": { @@ -4625,9 +4507,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "quoting": { @@ -4682,11 +4562,7 @@ "default": "ignore" }, "on_configuration_change": { - "enum": [ - "apply", - "continue", - "fail" - ] + "enum": ["apply", "continue", "fail"] }, "grants": { "type": "object", @@ -4828,9 +4704,7 @@ } }, "additionalProperties": false, - "required": [ - "type" - ] + "required": ["type"] } }, "quote": { @@ -4858,9 +4732,7 @@ } }, "additionalProperties": true, - "required": [ - "name" - ] + "required": ["name"] }, "propertyNames": { "type": "string" @@ -4997,9 +4869,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] } }, "sources": { @@ -5083,10 +4953,7 @@ } }, "additionalProperties": false, - "required": [ - "id", - "sql" - ] + "required": ["id", "sql"] } }, "_pre_injected_sql": { @@ -5198,10 +5065,7 @@ } }, "additionalProperties": false, - "required": [ - "name", - "checksum" - ] + "required": ["name", "checksum"] }, "config": { "type": "object", @@ -5436,9 +5300,7 @@ } }, "additionalProperties": false, - "required": [ - "type" - ] + "required": ["type"] } }, "quote": { @@ -5466,9 +5328,7 @@ } }, "additionalProperties": true, - "required": [ - "name" - ] + "required": ["name"] }, "propertyNames": { "type": "string" @@ -5605,9 +5465,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] } }, "sources": { @@ -5691,10 +5549,7 @@ } }, "additionalProperties": false, - "required": [ - "id", - "sql" - ] + "required": ["id", "sql"] } }, "_pre_injected_sql": { @@ -5867,10 +5722,7 @@ } }, "additionalProperties": false, - "required": [ - "name", - "checksum" - ] + "required": ["name", "checksum"] }, "config": { "type": "object", @@ -5996,9 +5848,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "pre-hook": { @@ -6027,9 +5877,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "quoting": { @@ -6078,11 +5926,7 @@ "default": "ignore" }, "on_configuration_change": { - "enum": [ - "apply", - "continue", - "fail" - ] + "enum": ["apply", "continue", "fail"] }, "grants": { "type": "object", @@ -6285,9 +6129,7 @@ } }, "additionalProperties": false, - "required": [ - "type" - ] + "required": ["type"] } }, "quote": { @@ -6315,9 +6157,7 @@ } }, "additionalProperties": true, - "required": [ - "name" - ] + "required": ["name"] }, "propertyNames": { "type": "string" @@ -6454,9 +6294,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] } }, "sources": { @@ -6540,10 +6378,7 @@ } }, "additionalProperties": false, - "required": [ - "id", - "sql" - ] + "required": ["id", "sql"] } }, "_pre_injected_sql": { @@ -6791,9 +6626,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "pre-hook": { @@ -6822,9 +6655,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "quoting": { @@ -6879,11 +6710,7 @@ "default": "ignore" }, "on_configuration_change": { - "enum": [ - "apply", - "continue", - "fail" - ] + "enum": ["apply", "continue", "fail"] }, "grants": { "type": "object", @@ -7131,11 +6958,7 @@ "period": { "anyOf": [ { - "enum": [ - "minute", - "hour", - "day" - ] + "enum": ["minute", "hour", "day"] }, { "type": "null" @@ -7171,11 +6994,7 @@ "period": { "anyOf": [ { - "enum": [ - "minute", - "hour", - "day" - ] + "enum": ["minute", "hour", "day"] }, { "type": "null" @@ -7406,9 +7225,7 @@ } }, "additionalProperties": false, - "required": [ - "type" - ] + "required": ["type"] } }, "quote": { @@ -7436,9 +7253,7 @@ } }, "additionalProperties": true, - "required": [ - "name" - ] + "required": ["name"] }, "propertyNames": { "type": "string" @@ -7509,6 +7324,21 @@ }, "created_at": { "type": "number" + }, + "access": { + "enum": ["private", "protected", "public"], + "default": "protected" + }, + "group": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": false, @@ -7642,9 +7472,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] } }, "created_at": { @@ -7655,10 +7483,7 @@ { "type": "array", "items": { - "enum": [ - "python", - "sql" - ] + "enum": ["python", "sql"] } }, { @@ -7759,13 +7584,7 @@ } }, "type": { - "enum": [ - "dashboard", - "notebook", - "analysis", - "ml", - "application" - ] + "enum": ["dashboard", "notebook", "analysis", "ml", "application"] }, "owner": { "type": "object", @@ -7820,11 +7639,7 @@ "maturity": { "anyOf": [ { - "enum": [ - "low", - "medium", - "high" - ] + "enum": ["low", "medium", "high"] }, { "type": "null" @@ -7933,9 +7748,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] } }, "sources": { @@ -7958,6 +7771,21 @@ }, "created_at": { "type": "number" + }, + "access": { + "enum": ["private", "protected", "public"], + "default": "public" + }, + "group": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": false, @@ -8015,13 +7843,7 @@ "type": "string" }, "type": { - "enum": [ - "simple", - "ratio", - "cumulative", - "derived", - "conversion" - ] + "enum": ["simple", "ratio", "cumulative", "derived", "conversion"] }, "type_params": { "type": "object", @@ -8053,16 +7875,12 @@ } }, "additionalProperties": false, - "required": [ - "where_sql_template" - ] + "required": ["where_sql_template"] } } }, "additionalProperties": false, - "required": [ - "where_filters" - ] + "required": ["where_filters"] }, { "type": "null" @@ -8098,9 +7916,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] }, { "type": "null" @@ -8134,16 +7950,12 @@ } }, "additionalProperties": false, - "required": [ - "where_sql_template" - ] + "required": ["where_sql_template"] } } }, "additionalProperties": false, - "required": [ - "where_filters" - ] + "required": ["where_filters"] }, { "type": "null" @@ -8179,9 +7991,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] } }, "numerator": { @@ -8210,16 +8020,12 @@ } }, "additionalProperties": false, - "required": [ - "where_sql_template" - ] + "required": ["where_sql_template"] } } }, "additionalProperties": false, - "required": [ - "where_filters" - ] + "required": ["where_filters"] }, { "type": "null" @@ -8264,10 +8070,7 @@ } }, "additionalProperties": false, - "required": [ - "count", - "granularity" - ] + "required": ["count", "granularity"] }, { "type": "null" @@ -8300,9 +8103,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] }, { "type": "null" @@ -8336,16 +8137,12 @@ } }, "additionalProperties": false, - "required": [ - "where_sql_template" - ] + "required": ["where_sql_template"] } } }, "additionalProperties": false, - "required": [ - "where_filters" - ] + "required": ["where_filters"] }, { "type": "null" @@ -8390,10 +8187,7 @@ } }, "additionalProperties": false, - "required": [ - "count", - "granularity" - ] + "required": ["count", "granularity"] }, { "type": "null" @@ -8426,9 +8220,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] }, { "type": "null" @@ -8473,10 +8265,7 @@ } }, "additionalProperties": false, - "required": [ - "count", - "granularity" - ] + "required": ["count", "granularity"] }, { "type": "null" @@ -8535,16 +8324,12 @@ } }, "additionalProperties": false, - "required": [ - "where_sql_template" - ] + "required": ["where_sql_template"] } } }, "additionalProperties": false, - "required": [ - "where_filters" - ] + "required": ["where_filters"] }, { "type": "null" @@ -8589,10 +8374,7 @@ } }, "additionalProperties": false, - "required": [ - "count", - "granularity" - ] + "required": ["count", "granularity"] }, { "type": "null" @@ -8625,9 +8407,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] } }, { @@ -8666,16 +8446,12 @@ } }, "additionalProperties": false, - "required": [ - "where_sql_template" - ] + "required": ["where_sql_template"] } } }, "additionalProperties": false, - "required": [ - "where_filters" - ] + "required": ["where_filters"] }, { "type": "null" @@ -8711,9 +8487,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] }, "conversion_measure": { "type": "object", @@ -8739,16 +8513,12 @@ } }, "additionalProperties": false, - "required": [ - "where_sql_template" - ] + "required": ["where_sql_template"] } } }, "additionalProperties": false, - "required": [ - "where_filters" - ] + "required": ["where_filters"] }, { "type": "null" @@ -8784,18 +8554,13 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] }, "entity": { "type": "string" }, "calculation": { - "enum": [ - "conversions", - "conversion_rate" - ], + "enum": ["conversions", "conversion_rate"], "default": "conversion_rate" }, "window": { @@ -8824,10 +8589,7 @@ } }, "additionalProperties": false, - "required": [ - "count", - "granularity" - ] + "required": ["count", "granularity"] }, { "type": "null" @@ -8865,11 +8627,7 @@ } }, "additionalProperties": false, - "required": [ - "base_measure", - "conversion_measure", - "entity" - ] + "required": ["base_measure", "conversion_measure", "entity"] }, { "type": "null" @@ -8909,10 +8667,7 @@ } }, "additionalProperties": false, - "required": [ - "count", - "granularity" - ] + "required": ["count", "granularity"] }, { "type": "null" @@ -8944,11 +8699,7 @@ "default": null }, "period_agg": { - "enum": [ - "first", - "last", - "average" - ], + "enum": ["first", "last", "average"], "default": "first" } }, @@ -8980,16 +8731,12 @@ } }, "additionalProperties": false, - "required": [ - "where_sql_template" - ] + "required": ["where_sql_template"] } } }, "additionalProperties": false, - "required": [ - "where_filters" - ] + "required": ["where_filters"] }, { "type": "null" @@ -9033,10 +8780,7 @@ } }, "additionalProperties": false, - "required": [ - "repo_file_path", - "file_slice" - ] + "required": ["repo_file_path", "file_slice"] }, { "type": "null" @@ -9183,9 +8927,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] } }, "metrics": { @@ -9379,10 +9121,7 @@ } }, "additionalProperties": false, - "required": [ - "name", - "checksum" - ] + "required": ["name", "checksum"] }, "config": { "type": "object", @@ -9508,9 +9247,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "pre-hook": { @@ -9539,9 +9276,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "quoting": { @@ -9596,11 +9331,7 @@ "default": "ignore" }, "on_configuration_change": { - "enum": [ - "apply", - "continue", - "fail" - ] + "enum": ["apply", "continue", "fail"] }, "grants": { "type": "object", @@ -9757,9 +9488,7 @@ } }, "additionalProperties": false, - "required": [ - "type" - ] + "required": ["type"] } }, "quote": { @@ -9787,9 +9516,7 @@ } }, "additionalProperties": true, - "required": [ - "name" - ] + "required": ["name"] }, "propertyNames": { "type": "string" @@ -10118,9 +9845,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "pre-hook": { @@ -10149,9 +9874,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "quoting": { @@ -10206,11 +9929,7 @@ "default": "ignore" }, "on_configuration_change": { - "enum": [ - "apply", - "continue", - "fail" - ] + "enum": ["apply", "continue", "fail"] }, "grants": { "type": "object", @@ -10363,10 +10082,7 @@ } }, "additionalProperties": false, - "required": [ - "name", - "checksum" - ] + "required": ["name", "checksum"] }, "config": { "type": "object", @@ -10492,9 +10208,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "pre-hook": { @@ -10523,9 +10237,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "quoting": { @@ -10580,11 +10292,7 @@ "default": "ignore" }, "on_configuration_change": { - "enum": [ - "apply", - "continue", - "fail" - ] + "enum": ["apply", "continue", "fail"] }, "grants": { "type": "object", @@ -10726,9 +10434,7 @@ } }, "additionalProperties": false, - "required": [ - "type" - ] + "required": ["type"] } }, "quote": { @@ -10756,9 +10462,7 @@ } }, "additionalProperties": true, - "required": [ - "name" - ] + "required": ["name"] }, "propertyNames": { "type": "string" @@ -10895,9 +10599,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] } }, "sources": { @@ -10981,10 +10683,7 @@ } }, "additionalProperties": false, - "required": [ - "id", - "sql" - ] + "required": ["id", "sql"] } }, "_pre_injected_sql": { @@ -11096,10 +10795,7 @@ } }, "additionalProperties": false, - "required": [ - "name", - "checksum" - ] + "required": ["name", "checksum"] }, "config": { "type": "object", @@ -11334,9 +11030,7 @@ } }, "additionalProperties": false, - "required": [ - "type" - ] + "required": ["type"] } }, "quote": { @@ -11364,9 +11058,7 @@ } }, "additionalProperties": true, - "required": [ - "name" - ] + "required": ["name"] }, "propertyNames": { "type": "string" @@ -11503,9 +11195,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] } }, "sources": { @@ -11589,10 +11279,7 @@ } }, "additionalProperties": false, - "required": [ - "id", - "sql" - ] + "required": ["id", "sql"] } }, "_pre_injected_sql": { @@ -11704,10 +11391,7 @@ } }, "additionalProperties": false, - "required": [ - "name", - "checksum" - ] + "required": ["name", "checksum"] }, "config": { "type": "object", @@ -11833,9 +11517,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "pre-hook": { @@ -11864,9 +11546,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "quoting": { @@ -11921,11 +11601,7 @@ "default": "ignore" }, "on_configuration_change": { - "enum": [ - "apply", - "continue", - "fail" - ] + "enum": ["apply", "continue", "fail"] }, "grants": { "type": "object", @@ -12067,9 +11743,7 @@ } }, "additionalProperties": false, - "required": [ - "type" - ] + "required": ["type"] } }, "quote": { @@ -12097,9 +11771,7 @@ } }, "additionalProperties": true, - "required": [ - "name" - ] + "required": ["name"] }, "propertyNames": { "type": "string" @@ -12236,9 +11908,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] } }, "sources": { @@ -12322,10 +11992,7 @@ } }, "additionalProperties": false, - "required": [ - "id", - "sql" - ] + "required": ["id", "sql"] } }, "_pre_injected_sql": { @@ -12448,10 +12115,7 @@ } }, "additionalProperties": false, - "required": [ - "name", - "checksum" - ] + "required": ["name", "checksum"] }, "config": { "type": "object", @@ -12577,9 +12241,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "pre-hook": { @@ -12608,9 +12270,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "quoting": { @@ -12665,11 +12325,7 @@ "default": "ignore" }, "on_configuration_change": { - "enum": [ - "apply", - "continue", - "fail" - ] + "enum": ["apply", "continue", "fail"] }, "grants": { "type": "object", @@ -12721,11 +12377,7 @@ "additionalProperties": false }, "access": { - "enum": [ - "private", - "protected", - "public" - ], + "enum": ["private", "protected", "public"], "default": "protected" } }, @@ -12819,9 +12471,7 @@ } }, "additionalProperties": false, - "required": [ - "type" - ] + "required": ["type"] } }, "quote": { @@ -12849,9 +12499,7 @@ } }, "additionalProperties": true, - "required": [ - "name" - ] + "required": ["name"] }, "propertyNames": { "type": "string" @@ -12988,9 +12636,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] } }, "sources": { @@ -13074,10 +12720,7 @@ } }, "additionalProperties": false, - "required": [ - "id", - "sql" - ] + "required": ["id", "sql"] } }, "_pre_injected_sql": { @@ -13118,11 +12761,7 @@ "additionalProperties": false }, "access": { - "enum": [ - "private", - "protected", - "public" - ], + "enum": ["private", "protected", "public"], "default": "protected" }, "constraints": { @@ -13179,9 +12818,7 @@ } }, "additionalProperties": false, - "required": [ - "type" - ] + "required": ["type"] } }, "version": { @@ -13431,9 +13068,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "pre-hook": { @@ -13462,9 +13097,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "quoting": { @@ -13519,11 +13152,7 @@ "default": "ignore" }, "on_configuration_change": { - "enum": [ - "apply", - "continue", - "fail" - ] + "enum": ["apply", "continue", "fail"] }, "grants": { "type": "object", @@ -13682,10 +13311,7 @@ } }, "additionalProperties": false, - "required": [ - "name", - "checksum" - ] + "required": ["name", "checksum"] }, "config": { "type": "object", @@ -13811,9 +13437,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "pre-hook": { @@ -13842,9 +13466,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "quoting": { @@ -13899,11 +13521,7 @@ "default": "ignore" }, "on_configuration_change": { - "enum": [ - "apply", - "continue", - "fail" - ] + "enum": ["apply", "continue", "fail"] }, "grants": { "type": "object", @@ -14045,9 +13663,7 @@ } }, "additionalProperties": false, - "required": [ - "type" - ] + "required": ["type"] } }, "quote": { @@ -14075,9 +13691,7 @@ } }, "additionalProperties": true, - "required": [ - "name" - ] + "required": ["name"] }, "propertyNames": { "type": "string" @@ -14214,9 +13828,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] } }, "sources": { @@ -14300,10 +13912,7 @@ } }, "additionalProperties": false, - "required": [ - "id", - "sql" - ] + "required": ["id", "sql"] } }, "_pre_injected_sql": { @@ -14415,10 +14024,7 @@ } }, "additionalProperties": false, - "required": [ - "name", - "checksum" - ] + "required": ["name", "checksum"] }, "config": { "type": "object", @@ -14653,9 +14259,7 @@ } }, "additionalProperties": false, - "required": [ - "type" - ] + "required": ["type"] } }, "quote": { @@ -14683,9 +14287,7 @@ } }, "additionalProperties": true, - "required": [ - "name" - ] + "required": ["name"] }, "propertyNames": { "type": "string" @@ -14822,9 +14424,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] } }, "sources": { @@ -14908,10 +14508,7 @@ } }, "additionalProperties": false, - "required": [ - "id", - "sql" - ] + "required": ["id", "sql"] } }, "_pre_injected_sql": { @@ -15084,10 +14681,7 @@ } }, "additionalProperties": false, - "required": [ - "name", - "checksum" - ] + "required": ["name", "checksum"] }, "config": { "type": "object", @@ -15213,9 +14807,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "pre-hook": { @@ -15244,9 +14836,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "quoting": { @@ -15295,11 +14885,7 @@ "default": "ignore" }, "on_configuration_change": { - "enum": [ - "apply", - "continue", - "fail" - ] + "enum": ["apply", "continue", "fail"] }, "grants": { "type": "object", @@ -15502,9 +15088,7 @@ } }, "additionalProperties": false, - "required": [ - "type" - ] + "required": ["type"] } }, "quote": { @@ -15532,9 +15116,7 @@ } }, "additionalProperties": true, - "required": [ - "name" - ] + "required": ["name"] }, "propertyNames": { "type": "string" @@ -15671,9 +15253,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] } }, "sources": { @@ -15757,10 +15337,7 @@ } }, "additionalProperties": false, - "required": [ - "id", - "sql" - ] + "required": ["id", "sql"] } }, "_pre_injected_sql": { @@ -16008,9 +15585,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "pre-hook": { @@ -16039,9 +15614,7 @@ } }, "additionalProperties": false, - "required": [ - "sql" - ] + "required": ["sql"] } }, "quoting": { @@ -16096,11 +15669,7 @@ "default": "ignore" }, "on_configuration_change": { - "enum": [ - "apply", - "continue", - "fail" - ] + "enum": ["apply", "continue", "fail"] }, "grants": { "type": "object", @@ -16339,11 +15908,7 @@ "period": { "anyOf": [ { - "enum": [ - "minute", - "hour", - "day" - ] + "enum": ["minute", "hour", "day"] }, { "type": "null" @@ -16379,11 +15944,7 @@ "period": { "anyOf": [ { - "enum": [ - "minute", - "hour", - "day" - ] + "enum": ["minute", "hour", "day"] }, { "type": "null" @@ -16614,9 +16175,7 @@ } }, "additionalProperties": false, - "required": [ - "type" - ] + "required": ["type"] } }, "quote": { @@ -16644,9 +16203,7 @@ } }, "additionalProperties": true, - "required": [ - "name" - ] + "required": ["name"] }, "propertyNames": { "type": "string" @@ -16717,6 +16274,21 @@ }, "created_at": { "type": "number" + }, + "access": { + "enum": ["private", "protected", "public"], + "default": "protected" + }, + "group": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": false, @@ -16826,11 +16398,7 @@ "maturity": { "anyOf": [ { - "enum": [ - "low", - "medium", - "high" - ] + "enum": ["low", "medium", "high"] }, { "type": "null" @@ -16939,9 +16507,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] } }, "sources": { @@ -16964,6 +16530,21 @@ }, "created_at": { "type": "number" + }, + "access": { + "enum": ["private", "protected", "public"], + "default": "public" + }, + "group": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null } }, "additionalProperties": false, @@ -17052,16 +16633,12 @@ } }, "additionalProperties": false, - "required": [ - "where_sql_template" - ] + "required": ["where_sql_template"] } } }, "additionalProperties": false, - "required": [ - "where_filters" - ] + "required": ["where_filters"] }, { "type": "null" @@ -17097,9 +16674,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] }, { "type": "null" @@ -17133,16 +16708,12 @@ } }, "additionalProperties": false, - "required": [ - "where_sql_template" - ] + "required": ["where_sql_template"] } } }, "additionalProperties": false, - "required": [ - "where_filters" - ] + "required": ["where_filters"] }, { "type": "null" @@ -17178,9 +16749,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] } }, "numerator": { @@ -17209,16 +16778,12 @@ } }, "additionalProperties": false, - "required": [ - "where_sql_template" - ] + "required": ["where_sql_template"] } } }, "additionalProperties": false, - "required": [ - "where_filters" - ] + "required": ["where_filters"] }, { "type": "null" @@ -17263,10 +16828,7 @@ } }, "additionalProperties": false, - "required": [ - "count", - "granularity" - ] + "required": ["count", "granularity"] }, { "type": "null" @@ -17299,9 +16861,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] }, { "type": "null" @@ -17335,16 +16895,12 @@ } }, "additionalProperties": false, - "required": [ - "where_sql_template" - ] + "required": ["where_sql_template"] } } }, "additionalProperties": false, - "required": [ - "where_filters" - ] + "required": ["where_filters"] }, { "type": "null" @@ -17389,10 +16945,7 @@ } }, "additionalProperties": false, - "required": [ - "count", - "granularity" - ] + "required": ["count", "granularity"] }, { "type": "null" @@ -17425,9 +16978,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] }, { "type": "null" @@ -17472,10 +17023,7 @@ } }, "additionalProperties": false, - "required": [ - "count", - "granularity" - ] + "required": ["count", "granularity"] }, { "type": "null" @@ -17534,16 +17082,12 @@ } }, "additionalProperties": false, - "required": [ - "where_sql_template" - ] + "required": ["where_sql_template"] } } }, "additionalProperties": false, - "required": [ - "where_filters" - ] + "required": ["where_filters"] }, { "type": "null" @@ -17588,10 +17132,7 @@ } }, "additionalProperties": false, - "required": [ - "count", - "granularity" - ] + "required": ["count", "granularity"] }, { "type": "null" @@ -17624,9 +17165,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] } }, { @@ -17672,9 +17211,7 @@ } }, "additionalProperties": false, - "required": [ - "where_filters" - ] + "required": ["where_filters"] }, { "type": "null" @@ -17710,9 +17247,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] }, "conversion_measure": { "type": "object", @@ -17745,9 +17280,7 @@ } }, "additionalProperties": false, - "required": [ - "where_filters" - ] + "required": ["where_filters"] }, { "type": "null" @@ -17783,18 +17316,13 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] }, "entity": { "type": "string" }, "calculation": { - "enum": [ - "conversions", - "conversion_rate" - ], + "enum": ["conversions", "conversion_rate"], "default": "conversion_rate" }, "window": { @@ -17823,10 +17351,7 @@ } }, "additionalProperties": false, - "required": [ - "count", - "granularity" - ] + "required": ["count", "granularity"] }, { "type": "null" @@ -17908,10 +17433,7 @@ } }, "additionalProperties": false, - "required": [ - "count", - "granularity" - ] + "required": ["count", "granularity"] }, { "type": "null" @@ -17943,11 +17465,7 @@ "default": null }, "period_agg": { - "enum": [ - "first", - "last", - "average" - ], + "enum": ["first", "last", "average"], "default": "first" } }, @@ -17979,16 +17497,12 @@ } }, "additionalProperties": false, - "required": [ - "where_sql_template" - ] + "required": ["where_sql_template"] } } }, "additionalProperties": false, - "required": [ - "where_filters" - ] + "required": ["where_filters"] }, { "type": "null" @@ -18032,10 +17546,7 @@ } }, "additionalProperties": false, - "required": [ - "repo_file_path", - "file_slice" - ] + "required": ["repo_file_path", "file_slice"] }, { "type": "null" @@ -18182,9 +17693,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] } }, "metrics": { @@ -18287,16 +17796,12 @@ } }, "additionalProperties": false, - "required": [ - "where_sql_template" - ] + "required": ["where_sql_template"] } } }, "additionalProperties": false, - "required": [ - "where_filters" - ] + "required": ["where_filters"] }, { "type": "null" @@ -18305,11 +17810,7 @@ } }, "additionalProperties": false, - "required": [ - "metrics", - "group_by", - "where" - ] + "required": ["metrics", "group_by", "where"] }, "exports": { "type": "array", @@ -18325,10 +17826,7 @@ "title": "ExportConfig", "properties": { "export_as": { - "enum": [ - "table", - "view" - ] + "enum": ["table", "view"] }, "schema_name": { "anyOf": [ @@ -18365,16 +17863,11 @@ } }, "additionalProperties": false, - "required": [ - "export_as" - ] + "required": ["export_as"] } }, "additionalProperties": false, - "required": [ - "name", - "config" - ] + "required": ["name", "config"] } }, "description": { @@ -18435,10 +17928,7 @@ } }, "additionalProperties": false, - "required": [ - "repo_file_path", - "file_slice" - ] + "required": ["repo_file_path", "file_slice"] }, { "type": "null" @@ -18480,10 +17970,7 @@ "export_as": { "anyOf": [ { - "enum": [ - "table", - "view" - ] + "enum": ["table", "view"] }, { "type": "null" @@ -18591,9 +18078,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] } } }, @@ -18696,10 +18181,7 @@ } }, "additionalProperties": false, - "required": [ - "alias", - "schema_name" - ] + "required": ["alias", "schema_name"] }, { "type": "null" @@ -18764,12 +18246,7 @@ "type": "string" }, "type": { - "enum": [ - "foreign", - "natural", - "primary", - "unique" - ] + "enum": ["foreign", "natural", "primary", "unique"] }, "description": { "anyOf": [ @@ -18817,10 +18294,7 @@ } }, "additionalProperties": false, - "required": [ - "name", - "type" - ] + "required": ["name", "type"] } }, "measures": { @@ -18971,10 +18445,7 @@ } }, "additionalProperties": false, - "required": [ - "name", - "agg" - ] + "required": ["name", "agg"] } }, "dimensions": { @@ -18987,10 +18458,7 @@ "type": "string" }, "type": { - "enum": [ - "categorical", - "time" - ] + "enum": ["categorical", "time"] }, "description": { "anyOf": [ @@ -19064,9 +18532,7 @@ } }, "additionalProperties": false, - "required": [ - "time_granularity" - ] + "required": ["time_granularity"] }, { "type": "null" @@ -19121,10 +18587,7 @@ } }, "additionalProperties": false, - "required": [ - "repo_file_path", - "file_slice" - ] + "required": ["repo_file_path", "file_slice"] }, { "type": "null" @@ -19134,10 +18597,7 @@ } }, "additionalProperties": false, - "required": [ - "name", - "type" - ] + "required": ["name", "type"] } }, "metadata": { @@ -19176,10 +18636,7 @@ } }, "additionalProperties": false, - "required": [ - "repo_file_path", - "file_slice" - ] + "required": ["repo_file_path", "file_slice"] }, { "type": "null" @@ -19242,9 +18699,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] } }, "created_at": { @@ -19363,11 +18818,7 @@ "default": null }, "format": { - "enum": [ - "csv", - "dict", - "sql" - ], + "enum": ["csv", "dict", "sql"], "default": "dict" }, "fixture": { @@ -19383,9 +18834,7 @@ } }, "additionalProperties": false, - "required": [ - "input" - ] + "required": ["input"] } }, "expect": { @@ -19413,11 +18862,7 @@ "default": null }, "format": { - "enum": [ - "csv", - "dict", - "sql" - ], + "enum": ["csv", "dict", "sql"], "default": "dict" }, "fixture": { @@ -19812,16 +19257,12 @@ } }, "additionalProperties": false, - "required": [ - "where_sql_template" - ] + "required": ["where_sql_template"] } } }, "additionalProperties": false, - "required": [ - "where_filters" - ] + "required": ["where_filters"] }, { "type": "null" @@ -19830,11 +19271,7 @@ } }, "additionalProperties": false, - "required": [ - "metrics", - "group_by", - "where" - ] + "required": ["metrics", "group_by", "where"] }, "exports": { "type": "array", @@ -19850,10 +19287,7 @@ "title": "ExportConfig", "properties": { "export_as": { - "enum": [ - "table", - "view" - ] + "enum": ["table", "view"] }, "schema_name": { "anyOf": [ @@ -19890,16 +19324,11 @@ } }, "additionalProperties": false, - "required": [ - "export_as" - ] + "required": ["export_as"] } }, "additionalProperties": false, - "required": [ - "name", - "config" - ] + "required": ["name", "config"] } }, "description": { @@ -19960,10 +19389,7 @@ } }, "additionalProperties": false, - "required": [ - "repo_file_path", - "file_slice" - ] + "required": ["repo_file_path", "file_slice"] }, { "type": "null" @@ -20005,10 +19431,7 @@ "export_as": { "anyOf": [ { - "enum": [ - "table", - "view" - ] + "enum": ["table", "view"] }, { "type": "null" @@ -20116,9 +19539,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] } } }, @@ -20228,10 +19649,7 @@ } }, "additionalProperties": false, - "required": [ - "alias", - "schema_name" - ] + "required": ["alias", "schema_name"] }, { "type": "null" @@ -20296,12 +19714,7 @@ "type": "string" }, "type": { - "enum": [ - "foreign", - "natural", - "primary", - "unique" - ] + "enum": ["foreign", "natural", "primary", "unique"] }, "description": { "anyOf": [ @@ -20349,10 +19762,7 @@ } }, "additionalProperties": false, - "required": [ - "name", - "type" - ] + "required": ["name", "type"] } }, "measures": { @@ -20478,11 +19888,7 @@ } }, "additionalProperties": false, - "required": [ - "name", - "window_choice", - "window_groupings" - ] + "required": ["name", "window_choice", "window_groupings"] }, { "type": "null" @@ -20503,10 +19909,7 @@ } }, "additionalProperties": false, - "required": [ - "name", - "agg" - ] + "required": ["name", "agg"] } }, "dimensions": { @@ -20519,10 +19922,7 @@ "type": "string" }, "type": { - "enum": [ - "categorical", - "time" - ] + "enum": ["categorical", "time"] }, "description": { "anyOf": [ @@ -20596,9 +19996,7 @@ } }, "additionalProperties": false, - "required": [ - "time_granularity" - ] + "required": ["time_granularity"] }, { "type": "null" @@ -20653,10 +20051,7 @@ } }, "additionalProperties": false, - "required": [ - "repo_file_path", - "file_slice" - ] + "required": ["repo_file_path", "file_slice"] }, { "type": "null" @@ -20666,10 +20061,7 @@ } }, "additionalProperties": false, - "required": [ - "name", - "type" - ] + "required": ["name", "type"] } }, "metadata": { @@ -20708,10 +20100,7 @@ } }, "additionalProperties": false, - "required": [ - "repo_file_path", - "file_slice" - ] + "required": ["repo_file_path", "file_slice"] }, { "type": "null" @@ -20774,9 +20163,7 @@ } }, "additionalProperties": false, - "required": [ - "name" - ] + "required": ["name"] } }, "created_at": { @@ -20902,11 +20289,7 @@ "default": null }, "format": { - "enum": [ - "csv", - "dict", - "sql" - ], + "enum": ["csv", "dict", "sql"], "default": "dict" }, "fixture": { @@ -20922,9 +20305,7 @@ } }, "additionalProperties": false, - "required": [ - "input" - ] + "required": ["input"] } }, "expect": { @@ -20952,11 +20333,7 @@ "default": null }, "format": { - "enum": [ - "csv", - "dict", - "sql" - ], + "enum": ["csv", "dict", "sql"], "default": "dict" }, "fixture": { diff --git a/tests/functional/access/test_access.py b/tests/functional/access/test_access.py index 5ef10c0e100..b94d9e1da59 100644 --- a/tests/functional/access/test_access.py +++ b/tests/functional/access/test_access.py @@ -12,6 +12,51 @@ yet_another_model_sql = "select 999 as weird" +stg_model_sql = """ +-- depends_on: {{ source('my_source', 'my_table') }} + +select 999 as weird +""" + +stg_model_yml = """ +models: + - name: stg_model + description: "my model" + group: analytics +""" + +invalid_access_source_yml = """ +sources: + - name: my_source + access: invalid + tables: + - name: my_table +""" + +non_grouped_source_yml = """ +sources: + - name: my_source + tables: + - name: my_table +""" + +protected_analytics_source_yml = """ +sources: + - name: my_source + group: analytics + tables: + - name: my_table +""" + +private_marts_source_yml = """ +sources: + - name: my_source + group: marts + access: private + tables: + - name: my_table +""" + schema_yml = """ models: - name: my_model @@ -314,6 +359,34 @@ def test_access_attribute(self, project): # verify it works again manifest = run_dbt(["parse"]) assert len(manifest.nodes) == 3 + + # add source with invalid access + write_file(invalid_access_source_yml, project.project_root, "models", "my_source.yml") + with pytest.raises(InvalidAccessTypeError): + run_dbt(["parse"]) + + # add source without group, new model in group analytics that depends on it + write_file(non_grouped_source_yml, project.project_root, "models", "my_source.yml") + write_file(stg_model_yml, project.project_root, "models", "stg_model.yml") + write_file(stg_model_sql, project.project_root, "models", "stg_model.sql") + manifest = run_dbt(["parse"]) + assert len(manifest.nodes) == 4 + assert len(manifest.sources) == 1 + + # add source to group analytics, implicitly set it to protected + write_file(protected_analytics_source_yml, project.project_root, "models", "my_source.yml") + manifest = run_dbt(["parse"]) + assert len(manifest.nodes) == 4 + assert len(manifest.sources) == 1 + + # add source to group mart, explicitly set it to private + write_file(private_marts_source_yml, project.project_root, "models", "my_source.yml") + with pytest.raises(DbtReferenceError): + run_dbt(["parse"]) + + # revert source to no group, protected access + write_file(non_grouped_source_yml, project.project_root, "models", "my_source.yml") + # Write out exposure refing private my_model write_file(simple_exposure_yml, project.project_root, "models", "simple_exposure.yml") # Fails with reference error @@ -329,14 +402,14 @@ def test_access_attribute(self, project): write_file(metricflow_time_spine_sql, "models", "metricflow_time_spine.sql") # Should succeed manifest = run_dbt(["parse"]) - assert len(manifest.nodes) == 5 + assert len(manifest.nodes) == 6 metric_id = "metric.test.number_of_people" assert manifest.metrics[metric_id].group == "analytics" # Use access and group in config write_file(v5_schema_yml, project.project_root, "models", "schema.yml") manifest = run_dbt(["parse"]) - assert len(manifest.nodes) == 5 + assert len(manifest.nodes) == 6 assert manifest.nodes["model.test.my_model"].access == AccessType.Private assert manifest.nodes["model.test.my_model"].group == "analytics" assert manifest.nodes["model.test.ref_my_model"].access == AccessType.Protected diff --git a/tests/functional/artifacts/expected_manifest.py b/tests/functional/artifacts/expected_manifest.py index 68e7799182b..11c2f63c4cd 100644 --- a/tests/functional/artifacts/expected_manifest.py +++ b/tests/functional/artifacts/expected_manifest.py @@ -770,6 +770,8 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "unique_id": "source.test.my_source.my_table", "fqn": ["test", "my_source", "my_table"], "unrendered_config": {}, + "group": None, + "access": "protected", }, }, "exposures": { @@ -804,6 +806,8 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "unique_id": "exposure.test.notebook_exposure", "url": "http://example.com/notebook/1", "unrendered_config": {}, + "group": None, + "access": "public", }, "exposure.test.simple_exposure": { "created_at": ANY, @@ -836,6 +840,8 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False): "meta": {}, "tags": [], "unrendered_config": {}, + "group": None, + "access": "public", }, }, "metrics": {}, @@ -1258,6 +1264,8 @@ def expected_references_manifest(project): "unique_id": "source.test.my_source.my_table", "fqn": ["test", "my_source", "my_table"], "unrendered_config": {}, + "group": None, + "access": "protected", }, }, "exposures": { @@ -1289,6 +1297,8 @@ def expected_references_manifest(project): "unique_id": "exposure.test.notebook_exposure", "url": "http://example.com/notebook/1", "unrendered_config": {}, + "group": None, + "access": "public", }, }, "metrics": {}, @@ -1859,6 +1869,8 @@ def expected_versions_manifest(project): "unique_id": "exposure.test.notebook_exposure", "url": None, "unrendered_config": {}, + "group": None, + "access": "public", }, }, "metrics": {}, diff --git a/tests/functional/partial_parsing/fixtures.py b/tests/functional/partial_parsing/fixtures.py index e419a3dfdbd..0326c0f4077 100644 --- a/tests/functional/partial_parsing/fixtures.py +++ b/tests/functional/partial_parsing/fixtures.py @@ -164,6 +164,62 @@ """ + +schema_sources_add_group_yml = """ + +groups: + - name: dave_land + owner: + name: dave + + +sources: + - name: seed_sources + schema: "{{ target.schema }}" + group: dave_land + tables: + - name: raw_customers + columns: + - name: id + data_tests: + - not_null: + severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" + - unique + - every_value_is_blue + - name: first_name + - name: last_name + - name: email +""" + + +schema_sources_add_access_yml = """ + +groups: + - name: dave_land + owner: + name: dave + + +sources: + - name: seed_sources + schema: "{{ target.schema }}" + group: dave_land + access: private + tables: + - name: raw_customers + columns: + - name: id + data_tests: + - not_null: + severity: "{{ 'error' if target.name == 'prod' else 'warn' }}" + - unique + - every_value_is_blue + - name: first_name + - name: last_name + - name: email + +""" + env_var_schema_yml = """ models: diff --git a/tests/functional/partial_parsing/test_partial_parsing.py b/tests/functional/partial_parsing/test_partial_parsing.py index 13bb3d5daae..64e804ee5c3 100644 --- a/tests/functional/partial_parsing/test_partial_parsing.py +++ b/tests/functional/partial_parsing/test_partial_parsing.py @@ -9,7 +9,7 @@ import dbt.flags as flags from dbt.contracts.files import ParseFileType from dbt.contracts.results import TestStatus -from dbt.exceptions import CompilationError +from dbt.exceptions import CompilationError, DbtReferenceError from dbt.plugins.manifest import ModelNodeArgs, PluginNodes from dbt.tests.fixtures.project import write_project_files from dbt.tests.util import ( @@ -72,6 +72,8 @@ schema_sources3_yml, schema_sources4_yml, schema_sources5_yml, + schema_sources_add_access_yml, + schema_sources_add_group_yml, snapshot2_sql, snapshot_sql, sources_tests1_sql, @@ -428,6 +430,18 @@ def test_pp_sources(self, project): write_file(sources_tests2_sql, project.project_root, "macros", "tests.sql") results = run_dbt(["--partial-parse", "run"]) + # Add group and add source to group + write_file(schema_sources_add_group_yml, project.project_root, "models", "sources.yml") + results = run_dbt(["--partial-parse", "run"]) + manifest = get_manifest(project.project_root) + assert len(manifest.sources) == 1 + assert manifest.sources["source.test.seed_sources.raw_customers"].group == "dave_land" + + # Add group and add source to group + write_file(schema_sources_add_access_yml, project.project_root, "models", "sources.yml") + with pytest.raises(DbtReferenceError): + run_dbt(["--partial-parse", "run"]) + class TestPartialParsingDependency: @pytest.fixture(scope="class") diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 7c14e8dee5b..be2ca3a4de8 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -34,4 +34,6 @@ def basic_parsed_source_definition_object(): unique_id="test.source.my_source.my_source_table", tags=[], config=SourceConfig(), + group="", + access=AccessType.Protected, # noqa ) diff --git a/tests/unit/contracts/graph/test_nodes_parsed.py b/tests/unit/contracts/graph/test_nodes_parsed.py index ebbe2443771..1dca6e568b6 100644 --- a/tests/unit/contracts/graph/test_nodes_parsed.py +++ b/tests/unit/contracts/graph/test_nodes_parsed.py @@ -1817,6 +1817,8 @@ def minimum_parsed_source_definition_dict(): "identifier": "my_source_table", "resource_type": str(NodeType.Source), "unique_id": "test.source.my_source.my_source_table", + "group": "", + "access": AccessType.Protected.value, } @@ -1847,6 +1849,8 @@ def basic_parsed_source_definition_dict(): "enabled": True, }, "unrendered_config": {}, + "group": "", + "access": AccessType.Protected.value, } @@ -1879,6 +1883,8 @@ def complex_parsed_source_definition_dict(): "freshness": {"warn_after": {"period": "hour", "count": 1}, "error_after": {}}, "loaded_at_field": "loaded_at", "unrendered_config": {}, + "group": "", + "access": AccessType.Protected.value, } @@ -1905,6 +1911,8 @@ def complex_parsed_source_definition_object(): config=SourceConfig(), freshness=FreshnessThreshold(warn_after=Time(period=TimePeriod.hour, count=1)), loaded_at_field="loaded_at", + group="", + access=AccessType.Protected, ) @@ -2018,6 +2026,8 @@ def minimal_parsed_exposure_dict(): "description": "", "created_at": 1.0, "resource_type": "exposure", + "group": "", + "access": AccessType.Public.value, } @@ -2050,6 +2060,8 @@ def basic_parsed_exposure_dict(): "enabled": True, }, "unrendered_config": {}, + "access": AccessType.Public.value, + "group": "", } @@ -2070,6 +2082,8 @@ def basic_parsed_exposure_object(): tags=[], config=ExposureConfig(), unrendered_config={}, + group="", + access=AccessType.Public, ) @@ -2105,6 +2119,8 @@ def complex_parsed_exposure_dict(): "enabled": True, }, "unrendered_config": {}, + "group": "", + "access": AccessType.Public.value, } @@ -2128,6 +2144,8 @@ def complex_parsed_exposure_object(): original_file_path="models/something.yml", config=ExposureConfig(), unrendered_config={}, + group="", + access=AccessType.Public, )