commit ff096b51b5c306d62eded24098af2b402a4fecb2 Author: M7XBeast Date: Sun Oct 11 17:37:40 2026 +0300 Initial publish of dbt_f2dq (f2dq) package v0.12.1. Clean package repo for Gitea installs via packages.yml git URL. Co-authored-by: Cursor diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..534c261 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,23 @@ +--- +name: Bug report +about: Create a report to help us improve +title: "[BUG]" +labels: bug +assignees: '' + +--- + +**Describe the bug** +Description of what the bug is. + +**Expected behavior** +Description of what you expected to happen. + +**To Reproduce** +Steps to reproduce the behavior: + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Logs and additional context** +If application, any other context, logs etc.here diff --git a/.github/ISSUE_TEMPLATE/documentation-request.md b/.github/ISSUE_TEMPLATE/documentation-request.md new file mode 100644 index 0000000..483998e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation-request.md @@ -0,0 +1,12 @@ +--- +name: Documentation request +about: Describe this issue template's purpose here. +title: "[DOCUMENTATION]" +labels: documentation +assignees: '' + +--- + +**Tell us about the documentation you'd like us to add or update** + +**Is anything not clear or outdates in the current version of docs?** diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..946d347 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: "[FEATURE]" +labels: '' +assignees: '' + +--- + +**Tell us about the problem you're trying to solve** +What are you trying to do, and why is it hard? A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you’d like** +A clear and concise description of what you want to see happen, or the change you would like to see + +**Describe the alternative you’ve considered or used** +A clear and concise description of any alternative solutions or features you've considered or are using today. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..a91d94a --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,6 @@ +## What +*Describe what the change is solving* +*It helps to add screenshots if it affects the frontend.* + +## How +*Describe the solution* diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml new file mode 100644 index 0000000..1330e2c --- /dev/null +++ b/.github/workflows/generate-docs.yml @@ -0,0 +1,55 @@ +name: Generate Documentation + +on: [push] + +env: + DBT_PROFILES_DIR: ./ + DBT_VERSION: 1.7 + PYTHON_VERSION: "3.8.x" + +jobs: + generate-docs: + runs-on: ubuntu-latest + if: github.event_name == 'push' + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + # Maps tcp port 5432 on service container to the host + - 5432:5432 + steps: + - name: Check out + uses: actions/checkout@v2 + + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install dependencies and run + run: | + pip install protobuf==4.25.3 dbt-postgres==$DBT_VERSION + dbt deps + dbt run + + - name: Generate Documentation + run: dbt docs generate + + - name: Copy files + if: github.ref == 'refs/heads/main' + run: 'mkdir docs && cp target/{catalog.json,index.html,manifest.json,run_results.json} docs/' + shell: bash + + - name: Deploy + uses: peaceiris/actions-gh-pages@v3 + if: github.ref == 'refs/heads/main' && !env.ACT + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./docs diff --git a/.github/workflows/run-db-tests.yml b/.github/workflows/run-db-tests.yml new file mode 100644 index 0000000..610de31 --- /dev/null +++ b/.github/workflows/run-db-tests.yml @@ -0,0 +1,144 @@ +name: Run package tests for all dbs + +on: [push, delete] + +env: + DBT_PROFILES_DIR: ${{ github.workspace }}/ + SNOWFLAKE_F2DQ_TESTING_ACCOUNT: ${{ secrets.SNOWFLAKE_F2DQ_TESTING_ACCOUNT }} + F2DQ_TESTING_USER: ${{ secrets.F2DQ_TESTING_USER }} + F2DQ_TESTING_PASSWORD: ${{ secrets.F2DQ_TESTING_PASSWORD }} + REDSHIFT_F2DQ_TESTING_HOST: ${{ secrets.REDSHIFT_F2DQ_TESTING_HOST }} + BIGQUERY_TESTING_TYPE: ${{ secrets.BIGQUERY_TESTING_TYPE }} + BIGQUERY_TESTING_PROJECT_ID: ${{ secrets.BIGQUERY_TESTING_PROJECT_ID }} + BIGQUERY_TESTING_PRIVATE_KEY_ID: ${{ secrets.BIGQUERY_TESTING_PRIVATE_KEY_ID }} + BIGQUERY_TESTING_PRIVATE_KEY: ${{ secrets.BIGQUERY_TESTING_PRIVATE_KEY }} + BIGQUERY_TESTING_CLIENT_EMAIL: ${{ secrets.BIGQUERY_TESTING_CLIENT_EMAIL }} + BIGQUERY_TESTING_CLIENT_ID: ${{ secrets.BIGQUERY_TESTING_CLIENT_ID }} + BIGQUERY_TESTING_AUTH_URI: ${{ secrets.BIGQUERY_TESTING_AUTH_URI }} + BIGQUERY_TESTING_TOKEN_URI: ${{ secrets.BIGQUERY_TESTING_TOKEN_URI }} + BIGQUERY_TESTING_AUTH_PROVIDER_X509_CERT_URL: ${{ secrets.BIGQUERY_TESTING_AUTH_PROVIDER_X509_CERT_URL }} + BIGQUERY_TESTING_CLIENT_X509_CERT_URL: ${{ secrets.BIGQUERY_TESTING_CLIENT_X509_CERT_URL }} + DBT_VERSION: 1.7 + PYTHON_VERSION: "3.8.x" + +jobs: + test-postgres: + runs-on: ubuntu-latest + if: github.event_name == 'push' + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + # Maps tcp port 5432 on service container to the host + - 5432:5432 + steps: + - name: Check out + uses: actions/checkout@v2 + + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install dependencies + working-directory: ./integration_tests + run: | + pip install -r requirements.txt + pip install dbt-postgres==$DBT_VERSION + dbt deps + + - name: Test DB + working-directory: ./integration_tests/python_tests + run: pytest --db postgres --source_schema dq + + test-other-dbs: + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.repository == 're-data/dbt-re-data' && github.ref == 'refs/heads/main' + strategy: + fail-fast: false + matrix: + database: [snowflake, bigquery, redshift] + steps: + - name: Check out + uses: actions/checkout@v2 + + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v3.x + + - name: Set the DQ_SCHEMA environment variable + shell: bash + run: | + echo "DQ_SCHEMA=dq_${GITHUB_REF_SLUG//[^[:alnum:]]/_}" >> $GITHUB_ENV + + - name: Print DQ_SCHEMA + run: | + echo $DQ_SCHEMA + + - name: Install dependencies + working-directory: ./integration_tests + run: | + pip install -r requirements.txt + pip install dbt-${{ matrix.database }}==$DBT_VERSION + dbt deps + + - name: Drop schemas + working-directory: ./integration_tests + run: | + dbt run-operation drop_all_schemas --args "{ schema_name: ${{ env.DQ_SCHEMA }} }" --profile f2dq_${{ matrix.database }} --vars "{ source_schema: ${{ env.DQ_SCHEMA }} }" + + - name: Create Schemas if needed + if: matrix.database == 'redshift' + working-directory: ./integration_tests + run: | + dbt run-operation create_required_schemas --args "{ schema_name: ${{ env.DQ_SCHEMA }} }" --profile f2dq_${{ matrix.database }} --vars "{ source_schema: ${{ env.DQ_SCHEMA }} }" + + - name: Test DB + working-directory: ./integration_tests/python_tests + run: | + pytest --db ${{ matrix.database }} --source_schema ${{ env.DQ_SCHEMA }} + + clean-up-schemas: + runs-on: ubuntu-latest + if: github.event_name == 'delete' && github.repository == 're-data/dbt-re-data' && github.ref == 'refs/heads/main' + strategy: + fail-fast: false + matrix: + database: [snowflake, bigquery, redshift] + steps: + - name: Check out + uses: actions/checkout@v2 + + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v3.x + + - name: Set the DQ_SCHEMA environment variable + shell: bash + run: | + echo "DQ_SCHEMA=dq_${GITHUB_EVENT_REF_SLUG//[^[:alnum:]]/_}" >> $GITHUB_ENV + + - name: Print DQ_SCHEMA + run: | + echo $DQ_SCHEMA + + - name: Install dependencies and drop branch schema + working-directory: ./integration_tests + run: | + pip install -r requirements.txt + pip install dbt-${{ matrix.database }}==$DBT_VERSION + dbt deps + dbt run-operation drop_all_schemas --args "{ schema_name: ${{ env.DQ_SCHEMA }} }" --profile f2dq_${{ matrix.database }} --vars "{ source_schema: ${{ env.DQ_SCHEMA }} }" \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4e9bf5e --- /dev/null +++ b/.gitignore @@ -0,0 +1,24 @@ +# dbt build artefacts +target/ +dbt_modules/ +dbt_packages/ +logs/ +# Keep package model sources under models/logs/ (not the dbt logs/ dir) + +# Local / secrets +.user.yml +.env +.env.* +*.pem +*.key + +# Python / IDE +*.pyc +__pycache__/ +*.egg-info +.python-version +venv/ +.venv/ +.idea/ +.vscode/ +.DS_Store diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..6292c40 --- /dev/null +++ b/LICENSE @@ -0,0 +1,38 @@ +MIT License + +Copyright (c) 2020 redata-team (original re_data project) +Copyright (c) 2026 fast2.dev (f2dq fork) + +This package is a fork of re_data (https://github.com/re-data/re-data). +The fork is maintained by fast2.dev and ships under the same MIT license +as the upstream. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +======================================================================== +Third party Apache 2.0 licenses +======================================================================== + +The following files contain significant parts of code licensed under third +party Apache 2.0 License (inherited from the upstream re_data project): + +macros/public/validating/regex_dict.sql +macros/utils/fivetran_utils/percentile.sql +macros/utils/fivetran_utils/json_extract.sql diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..e3341de --- /dev/null +++ b/Makefile @@ -0,0 +1,34 @@ + +# Maybe change this if you are not running on a Mac +CONTAINER_ARCH = linux/amd64 + +.PHONY: help run-all-ci test-postgres generate-docs + +help: + $(info ${HELP_MESSAGE}) + @exit 0 + + +# Run GitHub Actions CI jobs locally +run-all-ci: test-postgres generate-docs + @echo "All CI steps completed." + +test-postgres: + @echo "Running test-postgres job..." + act -j test-postgres --container-architecture $(CONTAINER_ARCH) + +generate-docs: + @echo "Running generate-docs job..." + act -j generate-docs --container-architecture $(CONTAINER_ARCH) + + +define HELP_MESSAGE +Usage: $ make [TARGETS] + +TARGETS + help Shows this help message + run-all-ci Runs all CI steps + test-postgres Runs test-postgres job + generate-docs Generates documentation + +endef diff --git a/README.md b/README.md new file mode 100644 index 0000000..2b948c5 --- /dev/null +++ b/README.md @@ -0,0 +1,60 @@ +# dbt_f2dq (`f2dq`) + +Modernized fork of [re_data](https://github.com/re-data/re-data) for fast2.dev Data Quality (f2dq). + +- **dbt package name:** `f2dq` (what `dbt_project.yml` / `detect_namespace()` use) +- **Hub-style identity:** `dbt_f2dq` +- **Version:** `0.12.1` +- **Requires:** dbt-core `>=1.0,<2.0`, `dbt_utils==1.4.1` + +## Install (git — recommended) + +```yaml +packages: + - git: "https://gitea.m7xlab.top/fast2dev/dbt-f2dq.git" + revision: main # or a tag like v0.12.1 +``` + +Then: + +```bash +dbt deps +``` + +Pin a release tag in production; use `main` only if you want floating tip. + +## Install (local path — active development) + +Prefer a local path while editing the package itself: + +```yaml +packages: + - local: /path/to/dbt_f2dq +``` + +## Dual-read (migration) + +Node config keys accept either prefix: + +- `+f2dq_monitored` / `+re_data_monitored` +- `+f2dq_time_filter` / `+re_data_time_filter` +- (same for metrics, columns, anomaly_detector, owners) + +Vars: `f2dq:` preferred; boolean enable flags also honour `re_data:` +(`store_table_samples`, `save_test_history`, etc.) via `f2dq.f2dq_var()`. + +## Artefacts + +Exports default to `target/f2dq/*.json` (`overview`, `monitored`, `alerts`, +`tests_history`, `table_samples`). Enable samples + test history: + +```yaml +vars: + f2dq:store_table_samples: true + f2dq:save_test_history: true +``` + +## Repository + +- Public clone URL: `https://gitea.m7xlab.top/fast2dev/dbt-f2dq.git` +- SSH (when port 22 is reachable): `git@gitea.m7xlab.top:fast2dev/dbt-f2dq.git` diff --git a/analysis/.gitkeep b/analysis/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/dbt_project.yml b/dbt_project.yml new file mode 100644 index 0000000..db121f2 --- /dev/null +++ b/dbt_project.yml @@ -0,0 +1,73 @@ +name: "f2dq" +version: "0.12.1" # dbt_f2dq: modernized fork (dbt_utils 1.4.1); dual-read re_data_* keys +config-version: 2 + +require-dbt-version: [">=1.0.0", "<2.0.0"] + +profile: "f2dq_postgres" + +target-path: "target" # directory which will store compiled SQL files +clean-targets: ["target", "dbt_modules", "dbt_packages"] + +on-run-end: + # Dual-read: f2dq:save_test_history or legacy re_data:save_test_history + - "{% if var('f2dq:save_test_history') or var('re_data:save_test_history', false) %} {{ f2dq.save_test_history(results) }} {% endif %}" + +vars: + f2dq:max_columns_in_query: 10 + f2dq:time_window_end: '{{ run_started_at.strftime("%Y-%m-%d 00:00:00") }}' + f2dq:time_window_start: '{{ (run_started_at - modules.datetime.timedelta(1)).strftime("%Y-%m-%d 00:00:00") }}' + f2dq:anomaly_detection_look_back_days: 30 + f2dq:select: null + f2dq:anomalies_filtered: f2dq_anomalies + + f2dq:alerting_z_score: 3 + + f2dq:save_test_history: false + f2dq:show_warns_as_alerts: false + + f2dq:anomaly_detector: + name: modified_z_score + threshold: 3 + + f2dq:store_table_samples: false + + f2dq:metrics_groups: + table_metrics: + table: + - row_count + - freshness + + column_metrics: + column: + numeric: + - min + - max + - avg + - stddev + - variance + - nulls_count + - nulls_percent + text: + - min_length + - max_length + - avg_length + - nulls_count + - missing_count + - nulls_percent + - missing_percent + boolean: + - count_true + - count_false + - nulls_count + - nulls_percent + + f2dq:default_metrics: + - table_metrics + - column_metrics + +models: + f2dq: + +schema: f2 + internal: + +schema: f2_internal diff --git a/integration_tests/.gitignore b/integration_tests/.gitignore new file mode 100644 index 0000000..eb35edc --- /dev/null +++ b/integration_tests/.gitignore @@ -0,0 +1,7 @@ + +target/ +dbt_modules/ +dbt_packages/ +logs/ +.user.yml +package-lock.yml diff --git a/integration_tests/README.md b/integration_tests/README.md new file mode 100644 index 0000000..bcea1f9 --- /dev/null +++ b/integration_tests/README.md @@ -0,0 +1 @@ +f2dq integration tests — dbt project for running f2dq dbt-package integration tests diff --git a/integration_tests/analysis/.gitkeep b/integration_tests/analysis/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml new file mode 100644 index 0000000..cd32cd1 --- /dev/null +++ b/integration_tests/dbt_project.yml @@ -0,0 +1,116 @@ +name: "f2dq_integration_tests" +version: "1.0" +config-version: 2 + +# Changed when running tests +profile: "f2dq_postgres" + +target-path: "target" +clean-targets: ["target", "dbt_modules", "dbt_packages"] + +models: + f2dq: + enabled: true + +sources: + f2dq_integration_tests: + +f2dq_monitored: true + +f2dq_time_filter: null + +vars: + f2dq:store_table_samples: true + f2dq:anomaly_detector: + name: modified_z_score + threshold: 0.6 + f2dq:max_columns_in_query: 1 + + f2dq:select: + - tag:testtag + - sample_table + - sample_without_time_filter + - sample_with_anomaly + - f2dq_source_test_table + + f2dq:metrics_groups: + integration_test_group: + table: + - row_count + - freshness + - my_distinct_table_rows + + column: + numeric: + - min + - max + - avg + - stddev + - variance + - nulls_count + - nulls_percent + - diff # my own custom metric + + text: + - min_length + - max_length + - avg_length + - nulls_count + - nulls_percent + - missing_percent + - missing_count + + f2dq:default_metrics: + - integration_test_group + +seeds: + +schema: seeds + +quote_columns: false + + f2dq_integration_tests: + monitoring: + sample_with_anomaly: + +f2dq_monitored: true + +f2dq_time_filter: creation_time + + sample_without_time_filter: + +f2dq_monitored: true + +f2dq_time_filter: null + + sample_table: + +f2dq_monitored: true + +f2dq_time_filter: creation_time + + +f2dq_columns: + - event_type + - value1 + - value2 + - null_value + + +f2dq_metrics: + table: + - my_custom_table_metric # my own custom metric + - distinct_table_rows + column: + event_type: + - regex_test: + regex: ([A-Za-z0-9]+) + - match_regex: + regex: ^sell + - match_regex_percent: + regex: ^sell + - not_match_regex: + regex: ^buy + - not_match_regex_percent: + regex: ^buy + - distinct_values + - duplicate_values + - duplicate_rows + - unique_rows + + expected_z_score: + +column_types: + time_window_end: "TIMESTAMP" + + expected_metrics: + +column_types: + time_window_start: "TIMESTAMP" + time_window_end: "TIMESTAMP" diff --git a/integration_tests/macros/.gitkeep b/integration_tests/macros/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/integration_tests/macros/create_test_source_tables.sql b/integration_tests/macros/create_test_source_tables.sql new file mode 100644 index 0000000..ef22220 --- /dev/null +++ b/integration_tests/macros/create_test_source_tables.sql @@ -0,0 +1,28 @@ + + +{% macro create_test_source_tables() %} + + {% set create_table %} + CREATE SCHEMA IF NOT EXISTS {{target.schema}}; + DROP TABLE IF EXISTS {{target.schema}}.f2dq_source_test_table; + CREATE TABLE IF NOT EXISTS {{target.schema}}.f2dq_source_test_table ( + number {{ f2dq.integer_type() }}, + description {{ f2dq.string_type() }}, + created_at {{ f2dq.timestamp_type() }} + ); + INSERT INTO {{target.schema}}.f2dq_source_test_table (number, description, created_at) VALUES + (1, 'one', current_timestamp), + (2, 'two', current_timestamp), + (3, 'three', current_timestamp), + (4, 'four', current_timestamp), + (5, 'five', current_timestamp), + (6, 'six', current_timestamp), + (7, 'seven', current_timestamp), + (8, 'eight', current_timestamp), + (9, 'nine', current_timestamp), + (10, 'ten', current_timestamp + ); + {% endset %} + {% do run_query(create_table) %} + +{% endmacro %} \ No newline at end of file diff --git a/integration_tests/macros/drop_all_schemas.sql b/integration_tests/macros/drop_all_schemas.sql new file mode 100644 index 0000000..bc8f391 --- /dev/null +++ b/integration_tests/macros/drop_all_schemas.sql @@ -0,0 +1,48 @@ +{% macro get_schemas_used(schema_name) %} + {% set schemas = [ + schema_name, + schema_name + '_re', + schema_name + '_re_internal', + schema_name + '_raw', + schema_name + '_expected', + schema_name + '_dbt_test__audit', + schema_name + '_seeds' + ] %} + {{ return (schemas) }} +{% endmacro %} + +{% macro drop_all_schemas(schema_name) %} + {% set schemas_to_drop = get_schemas_used(schema_name) %} + {{ adapter.dispatch('drop_all_schemas')(schemas_to_drop) }} +{% endmacro %} + +{% macro default__drop_all_schemas(schemas_to_drop) %} + {% for schema in schemas_to_drop %} + {% set relation = api.Relation.create(database=target.database, schema=schema) %} + {% do adapter.drop_schema(relation) %} + {% endfor %} +{% endmacro %} + +{% macro redshift__drop_all_schemas(schemas_to_drop) %} + {# + dropping schemas with adapter.drop_schema doesn't seem to work with redshift + so we default to issuing DDL commands to redshift + #} + {% set drop_query %} + {% for schema in schemas_to_drop %} + drop schema if exists {{schema}} cascade; + {% endfor %} + {% endset %} + {% do run_query(drop_query) %} +{% endmacro %} + +{% macro create_required_schemas(schema_name) %} + {# required to manually create schemas used for redshift tests #} + {% set schemas_to_drop = get_schemas_used(schema_name) %} + {% set create_query %} + {% for schema in schemas_to_drop %} + create schema if not exists {{schema}}; + {% endfor %} + {% endset %} + {% do run_query(create_query) %} +{% endmacro %} \ No newline at end of file diff --git a/integration_tests/macros/my_metrics.sql b/integration_tests/macros/my_metrics.sql new file mode 100644 index 0000000..77754e6 --- /dev/null +++ b/integration_tests/macros/my_metrics.sql @@ -0,0 +1,29 @@ +{% macro f2dq_metric_diff(context) %} + max({{context.column_name}}) - min({{context.column_name}}) +{% endmacro %} + +{% macro f2dq_metric_my_custom_table_metric(context) %} + 1000 +{% endmacro %} + +{% macro f2dq_metric_regex_test(context) %} + {{ regex_test(context.column_name, context.config.regex) }} +{% endmacro %} + +{% macro regex_test(column_name, pattern) %} + coalesce( + sum( + case when {{ f2dq.regex_match_expression(column_name, pattern) }} + then 1 + else 0 + end + ), 0 + ) +{% endmacro %} + +{% macro f2dq_metric_my_distinct_table_rows(context) %} + with temp_table AS ( + select distinct * from {{ context.table_name }} + ) + select coalesce(count(*), 0) FROM temp_table +{% endmacro %} \ No newline at end of file diff --git a/integration_tests/macros/test_utils.sql b/integration_tests/macros/test_utils.sql new file mode 100644 index 0000000..87ae590 --- /dev/null +++ b/integration_tests/macros/test_utils.sql @@ -0,0 +1,20 @@ + +{% macro to_big_integer(field) %} + cast (round({{field}} * 1000) as integer) as {{field}} +{% endmacro %} + +{% macro clean_table_name(field) %} + upper( + {{- + f2dq.clean_blacklist( + f2dq.split_and_return_nth_value(field, '.', 3), + ['"', '`'], + '' + ) + -}} + ) +{% endmacro %} + +{% macro clean_column_name(field) %} + case when ({{ field }} = '' or {{ field }} is null ) then '---' else upper({{field}}) end +{% endmacro %} diff --git a/integration_tests/macros/trigger_schema_change.sql b/integration_tests/macros/trigger_schema_change.sql new file mode 100644 index 0000000..4273f3e --- /dev/null +++ b/integration_tests/macros/trigger_schema_change.sql @@ -0,0 +1,15 @@ + +{% macro schema_change_buy_events_add_column() %} + {% set alter_table %} + alter table {{ ref('buy_events')}} add column sample_column boolean + {% endset %} + {% do run_query(alter_table) %} +{% endmacro %} + + +{% macro schema_change_buy_events_drop_column() %} + {% set alter_table %} + alter table {{ ref('buy_events')}} drop column sample_column + {% endset %} + {% do run_query(alter_table) %} +{% endmacro %} \ No newline at end of file diff --git a/integration_tests/models/metrics/f2dq_metrics.yml b/integration_tests/models/metrics/f2dq_metrics.yml new file mode 100644 index 0000000..d306a0d --- /dev/null +++ b/integration_tests/models/metrics/f2dq_metrics.yml @@ -0,0 +1,30 @@ + +version: 2 + +models: + - name: f2dq_metrics + tests: + - f2dq.metric_expression_is_true: + table: ref('buy_events') + metric: max_length + column_name: event_type + expression: value = 3 + + - f2dq.metric_equal_to: + table: ref('buy_events') + metric: max_length + column_name: event_type + value: 3 + + - f2dq.metric_in_range: + table: ref('buy_events') + metric: max_length + column_name: event_type + min_value: 3 + max_value: 3 + + - f2dq.metric_expression_is_true: + table: ref('buy_events') + metric: row_count + expression: value > 0 and value < 10 + condition: time_window_start >= '2021-05-02' \ No newline at end of file diff --git a/integration_tests/models/monitoring/schema.yml b/integration_tests/models/monitoring/schema.yml new file mode 100644 index 0000000..6612d9f --- /dev/null +++ b/integration_tests/models/monitoring/schema.yml @@ -0,0 +1,27 @@ +version: 2 + +models: + - name: test_f2dq_metrics + tests: + - dbt_utils.equality: + compare_model: ref('expected_metrics') + + - name: test_f2dq_z_score + tests: + - dbt_utils.equality: + compare_model: ref('expected_z_score') + + - name: test_f2dq_anomalies + tests: + - dbt_utils.equality: + compare_model: ref('expected_anomalies') + + - name: test_f2dq_test_history + tests: + - dbt_utils.equality: + compare_model: ref('expected_test_history') + + - name: test_f2dq_table_samples + tests: + - dbt_utils.equality: + compare_model: ref('expected_table_samples') \ No newline at end of file diff --git a/integration_tests/models/monitoring/test_f2dq_anomalies.sql b/integration_tests/models/monitoring/test_f2dq_anomalies.sql new file mode 100644 index 0000000..0205867 --- /dev/null +++ b/integration_tests/models/monitoring/test_f2dq_anomalies.sql @@ -0,0 +1,9 @@ + +select + {{ clean_table_name('table_name') }} as table_name, + {{ clean_column_name('column_name') }} as column_name, + metric, + anomaly_detector, + interval_length_sec + +from {{ ref('f2dq_anomalies') }} \ No newline at end of file diff --git a/integration_tests/models/monitoring/test_f2dq_metrics.sql b/integration_tests/models/monitoring/test_f2dq_metrics.sql new file mode 100644 index 0000000..5fa5773 --- /dev/null +++ b/integration_tests/models/monitoring/test_f2dq_metrics.sql @@ -0,0 +1,11 @@ + +select + {{ clean_table_name('table_name') }} as table_name, + {{ clean_column_name('column_name') }} as column_name, + metric, + time_window_start, + time_window_end, + {{ to_big_integer('value') }}, + interval_length_sec + +from {{ ref('f2dq_metrics') }} \ No newline at end of file diff --git a/integration_tests/models/monitoring/test_f2dq_table_samples.sql b/integration_tests/models/monitoring/test_f2dq_table_samples.sql new file mode 100644 index 0000000..1c2dc19 --- /dev/null +++ b/integration_tests/models/monitoring/test_f2dq_table_samples.sql @@ -0,0 +1,9 @@ + +select + {{ clean_table_name('table_name') }} as table_name, + length(sample_data) as sample_data_length +from {{ ref('f2dq_table_samples') }} +where {{ clean_table_name('table_name') }} != 'SAMPLE_WITHOUT_TIME_FILTER' + +-- SAMPLE_WITHOUT_TIME_FILTER because this table doesn't have a time filter, it's not possible to say how +-- exactly the sampel of it should look like. \ No newline at end of file diff --git a/integration_tests/models/monitoring/test_f2dq_test_history.sql b/integration_tests/models/monitoring/test_f2dq_test_history.sql new file mode 100644 index 0000000..a87bd54 --- /dev/null +++ b/integration_tests/models/monitoring/test_f2dq_test_history.sql @@ -0,0 +1,10 @@ + +select + {{ clean_table_name('table_name') }} as table_name, + {{ clean_column_name('column_name') }} as column_name, + right(test_name, 15) as test_name, + status, + {{ clean_column_name('message') }} as message, + cast (failures_count as integer) as failures_count, + severity +from {{ ref('f2dq_test_history') }} \ No newline at end of file diff --git a/integration_tests/models/monitoring/test_f2dq_z_score.sql b/integration_tests/models/monitoring/test_f2dq_z_score.sql new file mode 100644 index 0000000..03b0c8b --- /dev/null +++ b/integration_tests/models/monitoring/test_f2dq_z_score.sql @@ -0,0 +1,22 @@ + +{% set values_compare = [ + 'z_score_value', + 'modified_z_score_value', + 'last_value', + 'last_avg', + 'last_stddev', + 'last_median', + 'last_iqr', + 'last_median_absolute_deviation', + 'last_mean_absolute_deviation', +] %} + +select + {{ clean_table_name('table_name') }} as table_name, + {{ clean_column_name('column_name') }} as column_name, + metric, + time_window_end, + {% for col in values_compare %}{{ to_big_integer(col) }},{% endfor %} + interval_length_sec + +from {{ ref('f2dq_z_score') }} \ No newline at end of file diff --git a/integration_tests/models/public_macros/cleaning/sanitized_user_data.sql b/integration_tests/models/public_macros/cleaning/sanitized_user_data.sql new file mode 100644 index 0000000..2ee0021 --- /dev/null +++ b/integration_tests/models/public_macros/cleaning/sanitized_user_data.sql @@ -0,0 +1,4 @@ +select + {{ f2dq.clean_capitalize_words(f2dq.clean_additional_whitespaces('full_name')) }} as full_name, + {{ f2dq.clean_blacklist('email', ['^[a-zA-Z0-9_.+-]+'], '*****') }} as email +from {{ ref('sample_user_data') }} diff --git a/integration_tests/models/public_macros/cleaning/schema.yml b/integration_tests/models/public_macros/cleaning/schema.yml new file mode 100644 index 0000000..2e08bb1 --- /dev/null +++ b/integration_tests/models/public_macros/cleaning/schema.yml @@ -0,0 +1,7 @@ +version: 2 + +models: + - name: sanitized_user_data + tests: + - dbt_utils.equality: + compare_model: ref('expected_sample_user_data') \ No newline at end of file diff --git a/integration_tests/models/public_macros/filtering/deduplicated.sql b/integration_tests/models/public_macros/filtering/deduplicated.sql new file mode 100644 index 0000000..4807131 --- /dev/null +++ b/integration_tests/models/public_macros/filtering/deduplicated.sql @@ -0,0 +1,16 @@ +with x as + {{ f2dq.filter_remove_duplicates( + ref('duplicated'), ['transaction_id'], ['creation_time']) }} + +select *, 'take_first' as use_case from x + +union all + +select *, 'take_last' as use_case from {{ f2dq.filter_remove_duplicates( + ref('duplicated'), ['transaction_id'], ['creation_time desc']) }} duplicates + + +union all + +select *, 'take_all_statuses' as use_case from {{ f2dq.filter_remove_duplicates( + ref('duplicated'), ['transaction_id', 'status'], ['creation_time desc']) }} duplicates \ No newline at end of file diff --git a/integration_tests/models/public_macros/filtering/duplicates.sql b/integration_tests/models/public_macros/filtering/duplicates.sql new file mode 100644 index 0000000..42e2459 --- /dev/null +++ b/integration_tests/models/public_macros/filtering/duplicates.sql @@ -0,0 +1,2 @@ + {{ f2dq.filter_get_duplicates( + ref('duplicated'), ['transaction_id'], ['creation_time']) }} diff --git a/integration_tests/models/public_macros/filtering/schema.yml b/integration_tests/models/public_macros/filtering/schema.yml new file mode 100644 index 0000000..cf92f9b --- /dev/null +++ b/integration_tests/models/public_macros/filtering/schema.yml @@ -0,0 +1,11 @@ +version: 2 + +models: + - name: deduplicated + tests: + - dbt_utils.equality: + compare_model: ref('expected_deduplicated') + - name: duplicates + tests: + - dbt_utils.equality: + compare_model: ref('expected_duplicates') \ No newline at end of file diff --git a/integration_tests/models/public_macros/normalizing/schema.yml b/integration_tests/models/public_macros/normalizing/schema.yml new file mode 100644 index 0000000..6c66f79 --- /dev/null +++ b/integration_tests/models/public_macros/normalizing/schema.yml @@ -0,0 +1,7 @@ +version: 2 + +models: + - name: us_states_normalized + tests: + - dbt_utils.equality: + compare_model: ref('expected_us_states_normalized') \ No newline at end of file diff --git a/integration_tests/models/public_macros/normalizing/us_states_normalized.sql b/integration_tests/models/public_macros/normalizing/us_states_normalized.sql new file mode 100644 index 0000000..1381443 --- /dev/null +++ b/integration_tests/models/public_macros/normalizing/us_states_normalized.sql @@ -0,0 +1,29 @@ +with us_states_normalization_cte as ( + select source, target from {{ ref('us_states_normalization') }} +) + +{% set us_states_mapping = {'Ala.': 'Alabama', 'Alaska': 'Alaska', 'Ariz.': 'Arizona', 'Ark.': 'Arkansas', 'Calif.': 'California', 'Colo.': 'Colorado', 'Conn.': 'Connecticut', + 'Del.': 'Delaware', 'D.C.': 'District of Columbia', 'Fla.': 'Florida', 'Ga.': 'Georgia', 'Hawaii': 'Hawaii', 'Idaho': 'Idaho', 'Ill.': 'Illinois', 'Ind.': 'Indiana', + 'Iowa': 'Iowa', 'Kans.': 'Kansas', 'Ky.': 'Kentucky', 'La.': 'Louisiana', 'Maine': 'Maine', 'Md.': 'Maryland', 'Mass.': 'Massachusetts', 'Mich.': 'Michigan', + 'Minn.': 'Minnesota', 'Miss.': 'Mississippi', 'Mo.': 'Missouri', 'Mont.': 'Montana', 'Nebr.': 'Nebraska', 'Nev.': 'Nevada', 'N.H.': 'New Hampshire', 'N.J.': 'New Jersey', + 'N.M.': 'New Mexico', 'N.Y.': 'New York', 'N.C.': 'North Carolina', 'N.D.': 'North Dakota', 'Ohio': 'Ohio', 'Okla.': 'Oklahoma', 'Ore.': 'Oregon', 'Pa.': 'Pennsylvania', + 'R.I.': 'Rhode Island', 'S.C.': 'South Carolina', 'S.D.': 'South Dakota', 'Tenn.': 'Tennessee', 'Tex.': 'Texas', 'Utah': 'Utah', 'Vt.': 'Vermont', 'Va.': 'Virginia', + 'Wash.': 'Washington', 'W.Va.': 'West Virginia', 'Wis.': 'Wisconsin', 'Wyo.': 'Wyoming'} +%} + + +{# + We have three ways of passing the source used for normalization + 1. passing a dbt model using ref('') which is of type Relation. + 2. passing a common table expression that contains the source mapping + Note: model or cte must include "source" and "target" column names used for normalization in 1. & 2. repectively + 3. passing a dictionary of values that map from source -> target ie {[source]: [target]} + #} + +select distinct * from ( + select state, code, state__normalized from {{ f2dq.normalize_values(ref('abbreviated_us_states'), 'state', ref('us_states_normalization')) }} s + union all + select state, code, state__normalized from {{ f2dq.normalize_values(ref('abbreviated_us_states'), 'state', 'us_states_normalization_cte') }} s + union all + select state, code, state__normalized from {{ f2dq.normalize_values(ref('abbreviated_us_states'), 'state', us_states_mapping) }} s +) as normalized \ No newline at end of file diff --git a/integration_tests/models/public_macros/validating/schema.yml b/integration_tests/models/public_macros/validating/schema.yml new file mode 100644 index 0000000..50a3a5d --- /dev/null +++ b/integration_tests/models/public_macros/validating/schema.yml @@ -0,0 +1,27 @@ +version: 2 + +models: + - name: validated_emails + tests: + - dbt_utils.equality: + compare_model: ref('expected_validated_emails') + + - name: validated_numbers + tests: + - dbt_utils.equality: + compare_model: ref('expected_validated_numbers') + + - name: validated_date_and_time + tests: + - dbt_utils.equality: + compare_model: ref('expected_validated_date_and_time') + + - name: validated_ips + tests: + - dbt_utils.equality: + compare_model: ref('expected_validated_ips') + + - name: validated_uuids + tests: + - dbt_utils.equality: + compare_model: ref('expected_validated_uuids') \ No newline at end of file diff --git a/integration_tests/models/public_macros/validating/validated_date_and_time.sql b/integration_tests/models/public_macros/validating/validated_date_and_time.sql new file mode 100644 index 0000000..8f54c9b --- /dev/null +++ b/integration_tests/models/public_macros/validating/validated_date_and_time.sql @@ -0,0 +1,14 @@ +with + all_rows as ( + select * from {{ ref('validate_date_and_time') }} + ) + +select *, + case when {{ f2dq.valid_date_eu('date_time') }} then 1 else 0 end as valid_date_eu, + case when {{ f2dq.valid_date_us('date_time') }} then 1 else 0 end as valid_date_us, + case when {{ f2dq.valid_date_inverse('date_time') }} then 1 else 0 end as valid_date_inverse, + case when {{ f2dq.valid_date_iso_8601('date_time') }} then 1 else 0 end as valid_date_iso_8601, + case when {{ f2dq.valid_time_24h('date_time') }} then 1 else 0 end as valid_time_24h, + case when {{ f2dq.valid_time_12h('date_time') }} then 1 else 0 end as valid_time_12h, + case when {{ f2dq.valid_time('date_time') }} then 1 else 0 end as valid_time + from all_rows diff --git a/integration_tests/models/public_macros/validating/validated_emails.sql b/integration_tests/models/public_macros/validating/validated_emails.sql new file mode 100644 index 0000000..aaed247 --- /dev/null +++ b/integration_tests/models/public_macros/validating/validated_emails.sql @@ -0,0 +1,7 @@ +with + all_emails as ( + select * from {{ ref('validate_emails') }} + ) + +select *, case when {{ f2dq.valid_email('email') }} then 1 else 0 end as email_valid + from all_emails diff --git a/integration_tests/models/public_macros/validating/validated_ips.sql b/integration_tests/models/public_macros/validating/validated_ips.sql new file mode 100644 index 0000000..9d8425b --- /dev/null +++ b/integration_tests/models/public_macros/validating/validated_ips.sql @@ -0,0 +1,10 @@ +with + all_rows as ( + select * from {{ ref('validate_ip') }} + ) + +select *, + case when {{ f2dq.valid_ip_v4('ip_address') }} then 1 else 0 end as valid_ip_v4, + case when {{ f2dq.valid_ip_v6('ip_address') }} then 1 else 0 end as valid_ip_v6, + case when {{ f2dq.valid_ip('ip_address') }} then 1 else 0 end as valid_ip + from all_rows diff --git a/integration_tests/models/public_macros/validating/validated_numbers.sql b/integration_tests/models/public_macros/validating/validated_numbers.sql new file mode 100644 index 0000000..a4c9c63 --- /dev/null +++ b/integration_tests/models/public_macros/validating/validated_numbers.sql @@ -0,0 +1,13 @@ +with + all_num_rows as ( + select * from {{ ref('validate_numbers') }} + ) + +select *, + case when {{ f2dq.valid_number('number') }} then 1 else 0 end as is_number, + case when {{ f2dq.valid_number_decimal_point('number') }} then 1 else 0 end as is_number_decimal_point, + case when {{ f2dq.valid_number_decimal_comma('number') }} then 1 else 0 end as is_number_decimal_comma, + case when {{ f2dq.valid_number_percentage('number') }} then 1 else 0 end as is_percentage, + case when {{ f2dq.valid_number_percentage_point('number') }} then 1 else 0 end as is_percentage_decimal_point, + case when {{ f2dq.valid_number_percentage_comma('number') }} then 1 else 0 end as is_percentage_decimal_comma + from all_num_rows diff --git a/integration_tests/models/public_macros/validating/validated_uuids.sql b/integration_tests/models/public_macros/validating/validated_uuids.sql new file mode 100644 index 0000000..711a041 --- /dev/null +++ b/integration_tests/models/public_macros/validating/validated_uuids.sql @@ -0,0 +1,8 @@ +with + all_rows as ( + select * from {{ ref('validate_uuid') }} + ) + +select *, + case when {{ f2dq.valid_uuid('uuid') }} then 1 else 0 end as valid_uuid + from all_rows diff --git a/integration_tests/models/sources/schema.yml b/integration_tests/models/sources/schema.yml new file mode 100644 index 0000000..26cec14 --- /dev/null +++ b/integration_tests/models/sources/schema.yml @@ -0,0 +1,18 @@ +version: 2 + +sources: + - name: "{{ var('source_schema', target.schema) }}" + tables: + - name: f2dq_source_test_table + columns: + - name: number + tests: + - not_null + - unique + + - name: description + tests: + - not_null + - unique + + \ No newline at end of file diff --git a/integration_tests/models/transformed/buy_events.sql b/integration_tests/models/transformed/buy_events.sql new file mode 100644 index 0000000..46800b6 --- /dev/null +++ b/integration_tests/models/transformed/buy_events.sql @@ -0,0 +1,12 @@ +{{ + config( + f2dq_monitored=true, + f2dq_time_filter='creation_time', + f2dq_anomaly_detector={'name': 'z_score', 'threshold': 0.5}, + materialized='table', + tags=['testtag'] + ) +}} +select * +from {{ ref('sample_with_anomaly') }} +where event_type = 'buy' \ No newline at end of file diff --git a/integration_tests/models/transformed/schema.yml b/integration_tests/models/transformed/schema.yml new file mode 100644 index 0000000..125dbe9 --- /dev/null +++ b/integration_tests/models/transformed/schema.yml @@ -0,0 +1,49 @@ +version: 2 + +models: + - name: buy_events + tests: + - f2dq.assert_in_range: + metric: row_count + min_value: 0 + max_value: 10 + + - f2dq.assert_equal: + metric: row_count + value: 1 + + - f2dq.assert_false: + metric: freshness + expression: value is null + + - f2dq.assert_greater_equal: + metric: my_distinct_table_rows + value: 10 + + columns: + - name: value1 + tests: + - f2dq.assert_in_range: + metric: nulls_percent + min_value: 0 + max_value: 10 + + - f2dq.assert_true: + metric: nulls_percent + expression: value = 0 + + - f2dq.assert_less: + metric: min + value: 100 + condition: time_window_start = '2021-05-02' + + - f2dq.assert_less_equal: + metric: min + value: 107 + + - name: value2 + tests: + - f2dq.assert_greater_equal: + metric: min + value: 200 + condition: time_window_start = '2021-05-02' diff --git a/integration_tests/packages.yml b/integration_tests/packages.yml new file mode 100644 index 0000000..891773a --- /dev/null +++ b/integration_tests/packages.yml @@ -0,0 +1,5 @@ +packages: + - local: ../ + + - package: dbt-labs/dbt_utils + version: [">=1.0.0", "<1.2.0"] diff --git a/integration_tests/pytest.ini b/integration_tests/pytest.ini new file mode 100644 index 0000000..b80dabf --- /dev/null +++ b/integration_tests/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +norecursedirs = dbt_modules models target logs data analysis macros snapshots tests +python_files = test_*.py +python_functions = test* +addopts = --capture=no --durations=0 \ No newline at end of file diff --git a/integration_tests/python_tests/__init__.py b/integration_tests/python_tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/integration_tests/python_tests/conftest.py b/integration_tests/python_tests/conftest.py new file mode 100644 index 0000000..222cda2 --- /dev/null +++ b/integration_tests/python_tests/conftest.py @@ -0,0 +1,14 @@ +import pytest + +def pytest_addoption(parser): + parser.addoption("--db", action="store") + parser.addoption("--source_schema", action="store") + + +@pytest.fixture() +def db(pytestconfig): + return pytestconfig.getoption("db") + +@pytest.fixture() +def source_schema(pytestconfig): + return pytestconfig.getoption("source_schema") \ No newline at end of file diff --git a/integration_tests/python_tests/run_all_dbs.sh b/integration_tests/python_tests/run_all_dbs.sh new file mode 100755 index 0000000..e271867 --- /dev/null +++ b/integration_tests/python_tests/run_all_dbs.sh @@ -0,0 +1,5 @@ +pytest --db postgres $@ & +pytest --db snowflake $@ & +pytest --db bigquery $@ & +pytest --db redshift $@ & +wait diff --git a/integration_tests/python_tests/test_cleaners.py b/integration_tests/python_tests/test_cleaners.py new file mode 100644 index 0000000..7fc9bbe --- /dev/null +++ b/integration_tests/python_tests/test_cleaners.py @@ -0,0 +1,14 @@ +from .utils.run import dbt_seed, dbt_run, dbt_test + +def test_cleaners(db, source_schema, debug=True): + dbt_vars = { + 'source_schema': source_schema + } + + print (f"Running setup and tests for {db}") + + dbt_seed(f'--select public_macros.cleaning', db, dbt_vars) + dbt_run(f'--select sanitized_user_data+', db, dbt_vars) + dbt_test(f'--select sanitized_user_data', db, dbt_vars) + + print (f"Running tests completed for {db}") diff --git a/integration_tests/python_tests/test_filters.py b/integration_tests/python_tests/test_filters.py new file mode 100644 index 0000000..92764e1 --- /dev/null +++ b/integration_tests/python_tests/test_filters.py @@ -0,0 +1,25 @@ +from .utils.run import dbt_seed, dbt_run, dbt_test + +def test_deduplication(db, source_schema, debug=True): + dbt_vars = { + 'source_schema': source_schema + } + + print (f"Running setup and tests for {db}") + + dbt_seed(f'--select public_macros.filtering', db, dbt_vars) + dbt_run(f'--select deduplicated', db, dbt_vars) + dbt_test(f'--select deduplicated', db, dbt_vars) + +def test_get_duplicates(db, source_schema, debug=True): + dbt_vars = { + 'source_schema': source_schema + } + + print (f"Running setup and tests for {db}") + + dbt_seed(f'--select public_macros.filtering', db, dbt_vars) + dbt_run(f'--select duplicates', db, dbt_vars) + dbt_test(f'--select duplicates', db, dbt_vars) + + print (f"Running tests completed for {db}") diff --git a/integration_tests/python_tests/test_monitoring.py b/integration_tests/python_tests/test_monitoring.py new file mode 100644 index 0000000..a8a4a21 --- /dev/null +++ b/integration_tests/python_tests/test_monitoring.py @@ -0,0 +1,88 @@ +import os +import copy +import yaml +import json +from datetime import datetime, timedelta +from .utils.run import dbt_seed, dbt_run, dbt_test, dbt_command, dbt_build + +RUN_TIME = datetime(2021, 5, 2, 0, 0, 0) + +DBT_VARS = { + 'f2dq:time_window_start': (RUN_TIME - timedelta(days=1)).strftime("%Y-%m-%d %H:%M:%S"), + 'f2dq:time_window_end': RUN_TIME.strftime("%Y-%m-%d %H:%M:%S"), + 'f2dq:save_test_history': True +} + +def test_monitoring(db, source_schema): + DBT_VARS.update({'source_schema': source_schema}) + + load_deps = 'dbt deps' + assert os.system(load_deps) == 0 + + dbt_vars = copy.deepcopy(DBT_VARS) + + print (f"Running setup and tests for {db}") + + dbt_seed('--select monitoring', db, dbt_vars) + dbt_run('--models transformed', db, dbt_vars) + dbt_command( + f'dbt run-operation create_test_source_tables', + db, dbt_vars + ) + + print (f"Computing f2dq metrics for {db}") + dbt_run('--select package:f2dq', db, dbt_vars) + + dbt_command( + f'dbt run-operation schema_change_buy_events_add_column', + db, dbt_vars + ) + + # update dbts_vars to run dbt for next day of data + dbt_vars['f2dq:time_window_start'] = dbt_vars['f2dq:time_window_end'] + dbt_vars['f2dq:time_window_end'] = (RUN_TIME + timedelta(days=1)).strftime("%Y-%m-%d %H:%M:%S") + + dbt_command( + 'dbt run --select package:f2dq --fail-fast', + db, dbt_vars + ) + + dbt_command( + 'dbt run --select monitoring.*', db, dbt_vars + ) + + dbt_test('--select test_f2dq_anomalies test_f2dq_metrics test_f2dq_z_score test_f2dq_table_samples f2dq_metrics transformed', db, dbt_vars) + # dbt build will "duplicate" saved test result history + dbt_build('--select test_f2dq_anomalies test_f2dq_metrics test_f2dq_z_score test_f2dq_table_samples f2dq_metrics transformed', db, dbt_vars) + + # tests test_history seperately, because those are actually added to DB after running + # dbt test command + dbt_test('--select test_f2dq_test_history', db, dbt_vars) + + op_vars = { + 'start_date': RUN_TIME.strftime("%Y-%m-%d"), + 'end_date': (RUN_TIME + timedelta(days=1)).strftime("%Y-%m-%d"), + 'interval': 'days:1' + } + op_vars = yaml.dump(op_vars) + + dbt_command( + f'dbt run-operation generate_overview --args "{op_vars}"', + db, dbt_vars + ) + + overview = json.load(open(f'../target/f2dq/overview.json')) + expected_types = ['metric', 'schema_change', 'schema', 'alert', 'anomaly'] + all_types = set() + + # some simple check for now + for obj in overview: + all_types.add(obj['type']) + assert obj['table_name'] + assert 'column_name' in obj + assert 'computed_on' in obj + + assert len(overview) > 100 + assert sorted(all_types) == sorted(expected_types) + + print (f"Running tests completed for {db}") diff --git a/integration_tests/python_tests/test_normalizers.py b/integration_tests/python_tests/test_normalizers.py new file mode 100644 index 0000000..32c53d0 --- /dev/null +++ b/integration_tests/python_tests/test_normalizers.py @@ -0,0 +1,14 @@ +from .utils.run import dbt_seed, dbt_run, dbt_test + +def test_normalizers(db, source_schema, debug=True): + dbt_vars = { + 'source_schema': source_schema + } + + print (f"Running setup and tests for {db}") + + dbt_seed(f'--select public_macros.normalizing', db, dbt_vars) + dbt_run(f'--select us_states_normalized+', db, dbt_vars) + dbt_test(f'--models us_states_normalized', db, dbt_vars) + + print (f"Running tests completed for {db}") diff --git a/integration_tests/python_tests/test_validate.py b/integration_tests/python_tests/test_validate.py new file mode 100644 index 0000000..b5a4f95 --- /dev/null +++ b/integration_tests/python_tests/test_validate.py @@ -0,0 +1,17 @@ +from .utils.run import dbt_seed, dbt_run, dbt_test + +def test_validate_regex(db, source_schema, debug=True): + dbt_vars = { + 'source_schema': source_schema + } + + print (f"Running setup and tests for {db}") + + dbt_seed( + f'--select public_macros.validating', db, dbt_vars + ) + + dbt_run(f'--select public_macros.validating', db, dbt_vars) + dbt_test(f'--select public_macros.validating', db, dbt_vars) + + print (f"Running tests completed for {db}") \ No newline at end of file diff --git a/integration_tests/python_tests/utils/__init__.py b/integration_tests/python_tests/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/integration_tests/python_tests/utils/run.py b/integration_tests/python_tests/utils/run.py new file mode 100644 index 0000000..1c5a8e0 --- /dev/null +++ b/integration_tests/python_tests/utils/run.py @@ -0,0 +1,23 @@ +import os +import yaml + +def dbt_command(command, for_db, dbt_vars, threads=None): + debug = 'DBT_MACRO_DEBUGGING=1 ' + profile_part = f' --profile f2dq_{for_db}' + yaml_vars = yaml.dump(dbt_vars) + cmd = f'{debug} {command} --vars "{yaml_vars}" {profile_part}' + if threads: + cmd += f' --threads {threads}' + assert os.system(cmd) == 0 + +def dbt_seed(args, for_db, dbt_vars): + dbt_command(f'dbt seed --full-refresh {args}', for_db, dbt_vars, threads=4) + +def dbt_run(args, for_db, dbt_vars): + dbt_command(f'dbt run --full-refresh --fail-fast {args}', for_db, dbt_vars, threads=4) + +def dbt_test(args, for_db, dbt_vars): + dbt_command(f'dbt test --store-failures --fail-fast {args}', for_db, dbt_vars, threads=4) + +def dbt_build(args, for_db, dbt_vars): + dbt_command(f'dbt build --full-refresh --store-failures --fail-fast {args}', for_db, dbt_vars, threads=4) diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt new file mode 100644 index 0000000..7bc7cc6 --- /dev/null +++ b/integration_tests/requirements.txt @@ -0,0 +1,3 @@ +protobuf==4.25.3 +pytest==6.2.5 +pyyaml==6.0 diff --git a/integration_tests/seeds/.gitkeep b/integration_tests/seeds/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/integration_tests/seeds/monitoring/expected_anomalies.csv b/integration_tests/seeds/monitoring/expected_anomalies.csv new file mode 100644 index 0000000..07aeb7f --- /dev/null +++ b/integration_tests/seeds/monitoring/expected_anomalies.csv @@ -0,0 +1,29 @@ +table_name,column_name,metric,anomaly_detector,interval_length_sec +BUY_EVENTS,VALUE2,min,"{""name"": ""z_score"", ""threshold"": 0.5}",86400 +BUY_EVENTS,VALUE2,avg,"{""name"": ""z_score"", ""threshold"": 0.5}",86400 +BUY_EVENTS,---,freshness,"{""name"": ""z_score"", ""threshold"": 0.5}",86400 +BUY_EVENTS,VALUE1,min,"{""name"": ""z_score"", ""threshold"": 0.5}",86400 +BUY_EVENTS,VALUE1,max,"{""name"": ""z_score"", ""threshold"": 0.5}",86400 +BUY_EVENTS,VALUE1,avg,"{""name"": ""z_score"", ""threshold"": 0.5}",86400 +BUY_EVENTS,VALUE2,max,"{""name"": ""z_score"", ""threshold"": 0.5}",86400 +SAMPLE_TABLE,VALUE1,avg,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400 +SAMPLE_TABLE,EVENT_TYPE,unique_rows,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400 +SAMPLE_TABLE,EVENT_TYPE,duplicate_rows,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400 +SAMPLE_TABLE,VALUE1,max,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400 +SAMPLE_TABLE,EVENT_TYPE,match_regex_percent,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400 +SAMPLE_TABLE,VALUE1,diff,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400 +SAMPLE_TABLE,VALUE1,stddev,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400 +SAMPLE_TABLE,EVENT_TYPE,not_match_regex,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400 +SAMPLE_TABLE,EVENT_TYPE,avg_length,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400 +SAMPLE_TABLE,EVENT_TYPE,not_match_regex_percent,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400 +SAMPLE_TABLE,EVENT_TYPE,match_regex,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400 +SAMPLE_TABLE,VALUE1,variance,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400 +SAMPLE_TABLE,EVENT_TYPE,max_length,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400 +SAMPLE_TABLE,EVENT_TYPE,distinct_values,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400 +SAMPLE_WITH_ANOMALY,---,freshness,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400 +SAMPLE_WITH_ANOMALY,VALUE2,avg,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400 +SAMPLE_WITH_ANOMALY,VALUE1,max,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400 +SAMPLE_WITH_ANOMALY,VALUE2,min,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400 +SAMPLE_WITH_ANOMALY,VALUE1,avg,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400 +SAMPLE_WITH_ANOMALY,VALUE2,max,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400 +SAMPLE_WITH_ANOMALY,VALUE1,min,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400 diff --git a/integration_tests/seeds/monitoring/expected_metrics.csv b/integration_tests/seeds/monitoring/expected_metrics.csv new file mode 100644 index 0000000..3f8b738 --- /dev/null +++ b/integration_tests/seeds/monitoring/expected_metrics.csv @@ -0,0 +1,281 @@ +table_name,column_name,metric,time_window_start,time_window_end,value,interval_length_sec +SAMPLE_WITH_ANOMALY,VALUE1,min,2021-05-01 00:00:00,2021-05-02 00:00:00,107000,86400 +SAMPLE_WITH_ANOMALY,VALUE1,max,2021-05-01 00:00:00,2021-05-02 00:00:00,107000,86400 +SAMPLE_WITH_ANOMALY,VALUE1,avg,2021-05-01 00:00:00,2021-05-02 00:00:00,107000,86400 +SAMPLE_WITH_ANOMALY,VALUE1,stddev,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400 +SAMPLE_WITH_ANOMALY,VALUE1,variance,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400 +SAMPLE_WITH_ANOMALY,VALUE1,nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_WITH_ANOMALY,VALUE1,nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_WITH_ANOMALY,VALUE1,diff,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_WITH_ANOMALY,VALUE2,min,2021-05-01 00:00:00,2021-05-02 00:00:00,200000,86400 +SAMPLE_WITH_ANOMALY,VALUE2,max,2021-05-01 00:00:00,2021-05-02 00:00:00,200000,86400 +SAMPLE_WITH_ANOMALY,VALUE2,avg,2021-05-01 00:00:00,2021-05-02 00:00:00,200000,86400 +SAMPLE_WITH_ANOMALY,VALUE2,stddev,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400 +SAMPLE_WITH_ANOMALY,VALUE2,variance,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400 +SAMPLE_WITH_ANOMALY,VALUE2,nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_WITH_ANOMALY,VALUE2,nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_WITH_ANOMALY,VALUE2,diff,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_WITH_ANOMALY,EVENT_TYPE,min_length,2021-05-01 00:00:00,2021-05-02 00:00:00,3000,86400 +SAMPLE_WITH_ANOMALY,EVENT_TYPE,max_length,2021-05-01 00:00:00,2021-05-02 00:00:00,3000,86400 +SAMPLE_WITH_ANOMALY,EVENT_TYPE,avg_length,2021-05-01 00:00:00,2021-05-02 00:00:00,3000,86400 +SAMPLE_WITH_ANOMALY,EVENT_TYPE,nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_WITH_ANOMALY,EVENT_TYPE,nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_WITH_ANOMALY,EVENT_TYPE,missing_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_WITH_ANOMALY,EVENT_TYPE,missing_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_WITH_ANOMALY,---,row_count,2021-05-01 00:00:00,2021-05-02 00:00:00,1000,86400 +SAMPLE_WITH_ANOMALY,---,freshness,2021-05-01 00:00:00,2021-05-02 00:00:00,41308000,86400 +SAMPLE_WITH_ANOMALY,---,my_distinct_table_rows,2021-05-01 00:00:00,2021-05-02 00:00:00,12000,86400 +BUY_EVENTS,VALUE1,min,2021-05-01 00:00:00,2021-05-02 00:00:00,107000,86400 +BUY_EVENTS,VALUE1,max,2021-05-01 00:00:00,2021-05-02 00:00:00,107000,86400 +BUY_EVENTS,VALUE1,avg,2021-05-01 00:00:00,2021-05-02 00:00:00,107000,86400 +BUY_EVENTS,VALUE1,stddev,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400 +BUY_EVENTS,VALUE1,variance,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400 +BUY_EVENTS,VALUE1,nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +BUY_EVENTS,VALUE1,nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +BUY_EVENTS,VALUE1,diff,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +BUY_EVENTS,VALUE2,min,2021-05-01 00:00:00,2021-05-02 00:00:00,200000,86400 +BUY_EVENTS,VALUE2,max,2021-05-01 00:00:00,2021-05-02 00:00:00,200000,86400 +BUY_EVENTS,VALUE2,avg,2021-05-01 00:00:00,2021-05-02 00:00:00,200000,86400 +BUY_EVENTS,VALUE2,stddev,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400 +BUY_EVENTS,VALUE2,variance,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400 +BUY_EVENTS,VALUE2,nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +BUY_EVENTS,VALUE2,nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +BUY_EVENTS,VALUE2,diff,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +BUY_EVENTS,EVENT_TYPE,min_length,2021-05-01 00:00:00,2021-05-02 00:00:00,3000,86400 +BUY_EVENTS,EVENT_TYPE,max_length,2021-05-01 00:00:00,2021-05-02 00:00:00,3000,86400 +BUY_EVENTS,EVENT_TYPE,avg_length,2021-05-01 00:00:00,2021-05-02 00:00:00,3000,86400 +BUY_EVENTS,EVENT_TYPE,nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +BUY_EVENTS,EVENT_TYPE,nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +BUY_EVENTS,EVENT_TYPE,missing_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +BUY_EVENTS,EVENT_TYPE,missing_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +BUY_EVENTS,---,row_count,2021-05-01 00:00:00,2021-05-02 00:00:00,1000,86400 +BUY_EVENTS,---,freshness,2021-05-01 00:00:00,2021-05-02 00:00:00,41308000,86400 +BUY_EVENTS,---,my_distinct_table_rows,2021-05-01 00:00:00,2021-05-02 00:00:00,10000,86400 +SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__min,2021-05-01 00:00:00,2021-05-02 00:00:00,990,86400 +SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__max,2021-05-01 00:00:00,2021-05-02 00:00:00,4990,86400 +SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__avg,2021-05-01 00:00:00,2021-05-02 00:00:00,3306,86400 +SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__stddev,2021-05-01 00:00:00,2021-05-02 00:00:00,1668,86400 +SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__variance,2021-05-01 00:00:00,2021-05-02 00:00:00,2784,86400 +SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__diff,2021-05-01 00:00:00,2021-05-02 00:00:00,4000,86400 +SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__min_length,2021-05-01 00:00:00,2021-05-02 00:00:00,10000,86400 +SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__max_length,2021-05-01 00:00:00,2021-05-02 00:00:00,17000,86400 +SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__avg_length,2021-05-01 00:00:00,2021-05-02 00:00:00,14105,86400 +SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__missing_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__missing_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,RATING,global__min_length,2021-05-01 00:00:00,2021-05-02 00:00:00,1000,86400 +SAMPLE_WITHOUT_TIME_FILTER,RATING,global__max_length,2021-05-01 00:00:00,2021-05-02 00:00:00,5000,86400 +SAMPLE_WITHOUT_TIME_FILTER,RATING,global__avg_length,2021-05-01 00:00:00,2021-05-02 00:00:00,3053,86400 +SAMPLE_WITHOUT_TIME_FILTER,RATING,global__nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,RATING,global__nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,RATING,global__missing_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,RATING,global__missing_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,---,global__row_count,2021-05-01 00:00:00,2021-05-02 00:00:00,19000,86400 +SAMPLE_WITHOUT_TIME_FILTER,---,global__freshness,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400 +SAMPLE_WITHOUT_TIME_FILTER,---,global__my_distinct_table_rows,2021-05-01 00:00:00,2021-05-02 00:00:00,19000,86400 +F2DQ_SOURCE_TEST_TABLE,NUMBER,global__min,2021-05-01 00:00:00,2021-05-02 00:00:00,1000,86400 +F2DQ_SOURCE_TEST_TABLE,NUMBER,global__max,2021-05-01 00:00:00,2021-05-02 00:00:00,10000,86400 +F2DQ_SOURCE_TEST_TABLE,NUMBER,global__avg,2021-05-01 00:00:00,2021-05-02 00:00:00,5500,86400 +F2DQ_SOURCE_TEST_TABLE,NUMBER,global__stddev,2021-05-01 00:00:00,2021-05-02 00:00:00,3028,86400 +F2DQ_SOURCE_TEST_TABLE,NUMBER,global__variance,2021-05-01 00:00:00,2021-05-02 00:00:00,9167,86400 +F2DQ_SOURCE_TEST_TABLE,NUMBER,global__nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +F2DQ_SOURCE_TEST_TABLE,NUMBER,global__nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +F2DQ_SOURCE_TEST_TABLE,NUMBER,global__diff,2021-05-01 00:00:00,2021-05-02 00:00:00,9000,86400 +F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__min_length,2021-05-01 00:00:00,2021-05-02 00:00:00,3000,86400 +F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__max_length,2021-05-01 00:00:00,2021-05-02 00:00:00,5000,86400 +F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__avg_length,2021-05-01 00:00:00,2021-05-02 00:00:00,3900,86400 +F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__missing_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__missing_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +F2DQ_SOURCE_TEST_TABLE,---,global__row_count,2021-05-01 00:00:00,2021-05-02 00:00:00,10000,86400 +F2DQ_SOURCE_TEST_TABLE,---,global__freshness,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400 +F2DQ_SOURCE_TEST_TABLE,---,global__my_distinct_table_rows,2021-05-01 00:00:00,2021-05-02 00:00:00,10000,86400 +SAMPLE_TABLE,VALUE1,min,2021-05-01 00:00:00,2021-05-02 00:00:00,100000,86400 +SAMPLE_TABLE,VALUE1,max,2021-05-01 00:00:00,2021-05-02 00:00:00,200000,86400 +SAMPLE_TABLE,VALUE1,avg,2021-05-01 00:00:00,2021-05-02 00:00:00,127500,86400 +SAMPLE_TABLE,VALUE1,stddev,2021-05-01 00:00:00,2021-05-02 00:00:00,48563,86400 +SAMPLE_TABLE,VALUE1,variance,2021-05-01 00:00:00,2021-05-02 00:00:00,2358333,86400 +SAMPLE_TABLE,VALUE1,nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_TABLE,VALUE1,nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_TABLE,VALUE1,diff,2021-05-01 00:00:00,2021-05-02 00:00:00,100000,86400 +SAMPLE_TABLE,VALUE2,min,2021-05-01 00:00:00,2021-05-02 00:00:00,109000,86400 +SAMPLE_TABLE,VALUE2,max,2021-05-01 00:00:00,2021-05-02 00:00:00,209000,86400 +SAMPLE_TABLE,VALUE2,avg,2021-05-01 00:00:00,2021-05-02 00:00:00,180750,86400 +SAMPLE_TABLE,VALUE2,stddev,2021-05-01 00:00:00,2021-05-02 00:00:00,47975,86400 +SAMPLE_TABLE,VALUE2,variance,2021-05-01 00:00:00,2021-05-02 00:00:00,2301583,86400 +SAMPLE_TABLE,VALUE2,nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_TABLE,VALUE2,nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_TABLE,VALUE2,diff,2021-05-01 00:00:00,2021-05-02 00:00:00,100000,86400 +SAMPLE_TABLE,NULL_VALUE,min,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400 +SAMPLE_TABLE,NULL_VALUE,max,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400 +SAMPLE_TABLE,NULL_VALUE,avg,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400 +SAMPLE_TABLE,NULL_VALUE,stddev,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400 +SAMPLE_TABLE,NULL_VALUE,variance,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400 +SAMPLE_TABLE,NULL_VALUE,nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,4000,86400 +SAMPLE_TABLE,NULL_VALUE,nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,100000,86400 +SAMPLE_TABLE,NULL_VALUE,diff,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400 +SAMPLE_TABLE,EVENT_TYPE,min_length,2021-05-01 00:00:00,2021-05-02 00:00:00,3000,86400 +SAMPLE_TABLE,EVENT_TYPE,max_length,2021-05-01 00:00:00,2021-05-02 00:00:00,4000,86400 +SAMPLE_TABLE,EVENT_TYPE,avg_length,2021-05-01 00:00:00,2021-05-02 00:00:00,3250,86400 +SAMPLE_TABLE,EVENT_TYPE,nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_TABLE,EVENT_TYPE,nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_TABLE,EVENT_TYPE,missing_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_TABLE,EVENT_TYPE,missing_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400 +SAMPLE_TABLE,EVENT_TYPE,regex_test,2021-05-01 00:00:00,2021-05-02 00:00:00,4000,86400 +SAMPLE_TABLE,EVENT_TYPE,match_regex,2021-05-01 00:00:00,2021-05-02 00:00:00,1000,86400 +SAMPLE_TABLE,EVENT_TYPE,match_regex_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,25000,86400 +SAMPLE_TABLE,EVENT_TYPE,not_match_regex,2021-05-01 00:00:00,2021-05-02 00:00:00,1000,86400 +SAMPLE_TABLE,EVENT_TYPE,not_match_regex_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,25000,86400 +SAMPLE_TABLE,EVENT_TYPE,distinct_values,2021-05-01 00:00:00,2021-05-02 00:00:00,2000,86400 +SAMPLE_TABLE,EVENT_TYPE,duplicate_values,2021-05-01 00:00:00,2021-05-02 00:00:00,1000,86400 +SAMPLE_TABLE,EVENT_TYPE,duplicate_rows,2021-05-01 00:00:00,2021-05-02 00:00:00,3000,86400 +SAMPLE_TABLE,EVENT_TYPE,unique_rows,2021-05-01 00:00:00,2021-05-02 00:00:00,1000,86400 +SAMPLE_TABLE,---,row_count,2021-05-01 00:00:00,2021-05-02 00:00:00,4000,86400 +SAMPLE_TABLE,---,freshness,2021-05-01 00:00:00,2021-05-02 00:00:00,40765000,86400 +SAMPLE_TABLE,---,my_distinct_table_rows,2021-05-01 00:00:00,2021-05-02 00:00:00,9000,86400 +SAMPLE_TABLE,---,my_custom_table_metric,2021-05-01 00:00:00,2021-05-02 00:00:00,1000000,86400 +SAMPLE_TABLE,---,distinct_table_rows,2021-05-01 00:00:00,2021-05-02 00:00:00,4000,86400 +SAMPLE_WITH_ANOMALY,VALUE1,min,2021-05-02 00:00:00,2021-05-03 00:00:00,98000,86400 +SAMPLE_WITH_ANOMALY,VALUE1,max,2021-05-02 00:00:00,2021-05-03 00:00:00,98000,86400 +SAMPLE_WITH_ANOMALY,VALUE1,avg,2021-05-02 00:00:00,2021-05-03 00:00:00,98000,86400 +SAMPLE_WITH_ANOMALY,VALUE1,stddev,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400 +SAMPLE_WITH_ANOMALY,VALUE1,variance,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400 +SAMPLE_WITH_ANOMALY,VALUE1,nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_WITH_ANOMALY,VALUE1,nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_WITH_ANOMALY,VALUE1,diff,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_WITH_ANOMALY,VALUE2,min,2021-05-02 00:00:00,2021-05-03 00:00:00,205000,86400 +SAMPLE_WITH_ANOMALY,VALUE2,max,2021-05-02 00:00:00,2021-05-03 00:00:00,205000,86400 +SAMPLE_WITH_ANOMALY,VALUE2,avg,2021-05-02 00:00:00,2021-05-03 00:00:00,205000,86400 +SAMPLE_WITH_ANOMALY,VALUE2,stddev,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400 +SAMPLE_WITH_ANOMALY,VALUE2,variance,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400 +SAMPLE_WITH_ANOMALY,VALUE2,nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_WITH_ANOMALY,VALUE2,nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_WITH_ANOMALY,VALUE2,diff,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_WITH_ANOMALY,EVENT_TYPE,min_length,2021-05-02 00:00:00,2021-05-03 00:00:00,3000,86400 +SAMPLE_WITH_ANOMALY,EVENT_TYPE,max_length,2021-05-02 00:00:00,2021-05-03 00:00:00,3000,86400 +SAMPLE_WITH_ANOMALY,EVENT_TYPE,avg_length,2021-05-02 00:00:00,2021-05-03 00:00:00,3000,86400 +SAMPLE_WITH_ANOMALY,EVENT_TYPE,nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_WITH_ANOMALY,EVENT_TYPE,nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_WITH_ANOMALY,EVENT_TYPE,missing_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_WITH_ANOMALY,EVENT_TYPE,missing_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_WITH_ANOMALY,---,row_count,2021-05-02 00:00:00,2021-05-03 00:00:00,1000,86400 +SAMPLE_WITH_ANOMALY,---,freshness,2021-05-02 00:00:00,2021-05-03 00:00:00,41065000,86400 +SAMPLE_WITH_ANOMALY,---,my_distinct_table_rows,2021-05-02 00:00:00,2021-05-03 00:00:00,12000,86400 +BUY_EVENTS,VALUE1,min,2021-05-02 00:00:00,2021-05-03 00:00:00,98000,86400 +BUY_EVENTS,VALUE1,max,2021-05-02 00:00:00,2021-05-03 00:00:00,98000,86400 +BUY_EVENTS,VALUE1,avg,2021-05-02 00:00:00,2021-05-03 00:00:00,98000,86400 +BUY_EVENTS,VALUE1,stddev,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400 +BUY_EVENTS,VALUE1,variance,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400 +BUY_EVENTS,VALUE1,nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +BUY_EVENTS,VALUE1,nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +BUY_EVENTS,VALUE1,diff,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +BUY_EVENTS,VALUE2,min,2021-05-02 00:00:00,2021-05-03 00:00:00,205000,86400 +BUY_EVENTS,VALUE2,max,2021-05-02 00:00:00,2021-05-03 00:00:00,205000,86400 +BUY_EVENTS,VALUE2,avg,2021-05-02 00:00:00,2021-05-03 00:00:00,205000,86400 +BUY_EVENTS,VALUE2,stddev,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400 +BUY_EVENTS,VALUE2,variance,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400 +BUY_EVENTS,VALUE2,nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +BUY_EVENTS,VALUE2,nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +BUY_EVENTS,VALUE2,diff,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +BUY_EVENTS,EVENT_TYPE,min_length,2021-05-02 00:00:00,2021-05-03 00:00:00,3000,86400 +BUY_EVENTS,EVENT_TYPE,max_length,2021-05-02 00:00:00,2021-05-03 00:00:00,3000,86400 +BUY_EVENTS,EVENT_TYPE,avg_length,2021-05-02 00:00:00,2021-05-03 00:00:00,3000,86400 +BUY_EVENTS,EVENT_TYPE,nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +BUY_EVENTS,EVENT_TYPE,nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +BUY_EVENTS,EVENT_TYPE,missing_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +BUY_EVENTS,EVENT_TYPE,missing_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +BUY_EVENTS,---,row_count,2021-05-02 00:00:00,2021-05-03 00:00:00,1000,86400 +BUY_EVENTS,---,freshness,2021-05-02 00:00:00,2021-05-03 00:00:00,41065000,86400 +BUY_EVENTS,---,my_distinct_table_rows,2021-05-02 00:00:00,2021-05-03 00:00:00,10000,86400 +SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__min,2021-05-02 00:00:00,2021-05-03 00:00:00,990,86400 +SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__max,2021-05-02 00:00:00,2021-05-03 00:00:00,4990,86400 +SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__avg,2021-05-02 00:00:00,2021-05-03 00:00:00,3306,86400 +SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__stddev,2021-05-02 00:00:00,2021-05-03 00:00:00,1668,86400 +SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__variance,2021-05-02 00:00:00,2021-05-03 00:00:00,2784,86400 +SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__diff,2021-05-02 00:00:00,2021-05-03 00:00:00,4000,86400 +SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__min_length,2021-05-02 00:00:00,2021-05-03 00:00:00,10000,86400 +SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__max_length,2021-05-02 00:00:00,2021-05-03 00:00:00,17000,86400 +SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__avg_length,2021-05-02 00:00:00,2021-05-03 00:00:00,14105,86400 +SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__missing_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__missing_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,RATING,global__min_length,2021-05-02 00:00:00,2021-05-03 00:00:00,1000,86400 +SAMPLE_WITHOUT_TIME_FILTER,RATING,global__max_length,2021-05-02 00:00:00,2021-05-03 00:00:00,5000,86400 +SAMPLE_WITHOUT_TIME_FILTER,RATING,global__avg_length,2021-05-02 00:00:00,2021-05-03 00:00:00,3053,86400 +SAMPLE_WITHOUT_TIME_FILTER,RATING,global__nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,RATING,global__nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,RATING,global__missing_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,RATING,global__missing_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,---,global__row_count,2021-05-02 00:00:00,2021-05-03 00:00:00,19000,86400 +SAMPLE_WITHOUT_TIME_FILTER,---,global__freshness,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400 +SAMPLE_WITHOUT_TIME_FILTER,---,global__my_distinct_table_rows,2021-05-02 00:00:00,2021-05-03 00:00:00,19000,86400 +F2DQ_SOURCE_TEST_TABLE,NUMBER,global__min,2021-05-02 00:00:00,2021-05-03 00:00:00,1000,86400 +F2DQ_SOURCE_TEST_TABLE,NUMBER,global__max,2021-05-02 00:00:00,2021-05-03 00:00:00,10000,86400 +F2DQ_SOURCE_TEST_TABLE,NUMBER,global__avg,2021-05-02 00:00:00,2021-05-03 00:00:00,5500,86400 +F2DQ_SOURCE_TEST_TABLE,NUMBER,global__stddev,2021-05-02 00:00:00,2021-05-03 00:00:00,3028,86400 +F2DQ_SOURCE_TEST_TABLE,NUMBER,global__variance,2021-05-02 00:00:00,2021-05-03 00:00:00,9167,86400 +F2DQ_SOURCE_TEST_TABLE,NUMBER,global__nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +F2DQ_SOURCE_TEST_TABLE,NUMBER,global__nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +F2DQ_SOURCE_TEST_TABLE,NUMBER,global__diff,2021-05-02 00:00:00,2021-05-03 00:00:00,9000,86400 +F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__min_length,2021-05-02 00:00:00,2021-05-03 00:00:00,3000,86400 +F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__max_length,2021-05-02 00:00:00,2021-05-03 00:00:00,5000,86400 +F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__avg_length,2021-05-02 00:00:00,2021-05-03 00:00:00,3900,86400 +F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__missing_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__missing_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +F2DQ_SOURCE_TEST_TABLE,---,global__row_count,2021-05-02 00:00:00,2021-05-03 00:00:00,10000,86400 +F2DQ_SOURCE_TEST_TABLE,---,global__freshness,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400 +F2DQ_SOURCE_TEST_TABLE,---,global__my_distinct_table_rows,2021-05-02 00:00:00,2021-05-03 00:00:00,10000,86400 +SAMPLE_TABLE,VALUE1,min,2021-05-02 00:00:00,2021-05-03 00:00:00,100000,86400 +SAMPLE_TABLE,VALUE1,max,2021-05-02 00:00:00,2021-05-03 00:00:00,210000,86400 +SAMPLE_TABLE,VALUE1,avg,2021-05-02 00:00:00,2021-05-03 00:00:00,142500,86400 +SAMPLE_TABLE,VALUE1,stddev,2021-05-02 00:00:00,2021-05-03 00:00:00,49917,86400 +SAMPLE_TABLE,VALUE1,variance,2021-05-02 00:00:00,2021-05-03 00:00:00,2491667,86400 +SAMPLE_TABLE,VALUE1,nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_TABLE,VALUE1,nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_TABLE,VALUE1,diff,2021-05-02 00:00:00,2021-05-03 00:00:00,110000,86400 +SAMPLE_TABLE,VALUE2,min,2021-05-02 00:00:00,2021-05-03 00:00:00,109000,86400 +SAMPLE_TABLE,VALUE2,max,2021-05-02 00:00:00,2021-05-03 00:00:00,209000,86400 +SAMPLE_TABLE,VALUE2,avg,2021-05-02 00:00:00,2021-05-03 00:00:00,180750,86400 +SAMPLE_TABLE,VALUE2,stddev,2021-05-02 00:00:00,2021-05-03 00:00:00,47975,86400 +SAMPLE_TABLE,VALUE2,variance,2021-05-02 00:00:00,2021-05-03 00:00:00,2301583,86400 +SAMPLE_TABLE,VALUE2,nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_TABLE,VALUE2,nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_TABLE,VALUE2,diff,2021-05-02 00:00:00,2021-05-03 00:00:00,100000,86400 +SAMPLE_TABLE,NULL_VALUE,min,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400 +SAMPLE_TABLE,NULL_VALUE,max,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400 +SAMPLE_TABLE,NULL_VALUE,avg,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400 +SAMPLE_TABLE,NULL_VALUE,stddev,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400 +SAMPLE_TABLE,NULL_VALUE,variance,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400 +SAMPLE_TABLE,NULL_VALUE,nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,4000,86400 +SAMPLE_TABLE,NULL_VALUE,nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,100000,86400 +SAMPLE_TABLE,NULL_VALUE,diff,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400 +SAMPLE_TABLE,EVENT_TYPE,min_length,2021-05-02 00:00:00,2021-05-03 00:00:00,3000,86400 +SAMPLE_TABLE,EVENT_TYPE,max_length,2021-05-02 00:00:00,2021-05-03 00:00:00,3000,86400 +SAMPLE_TABLE,EVENT_TYPE,avg_length,2021-05-02 00:00:00,2021-05-03 00:00:00,3000,86400 +SAMPLE_TABLE,EVENT_TYPE,nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_TABLE,EVENT_TYPE,nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_TABLE,EVENT_TYPE,missing_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_TABLE,EVENT_TYPE,missing_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_TABLE,EVENT_TYPE,regex_test,2021-05-02 00:00:00,2021-05-03 00:00:00,4000,86400 +SAMPLE_TABLE,EVENT_TYPE,match_regex,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_TABLE,EVENT_TYPE,match_regex_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_TABLE,EVENT_TYPE,not_match_regex,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_TABLE,EVENT_TYPE,not_match_regex_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_TABLE,EVENT_TYPE,distinct_values,2021-05-02 00:00:00,2021-05-03 00:00:00,1000,86400 +SAMPLE_TABLE,EVENT_TYPE,duplicate_values,2021-05-02 00:00:00,2021-05-03 00:00:00,1000,86400 +SAMPLE_TABLE,EVENT_TYPE,duplicate_rows,2021-05-02 00:00:00,2021-05-03 00:00:00,4000,86400 +SAMPLE_TABLE,EVENT_TYPE,unique_rows,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400 +SAMPLE_TABLE,---,row_count,2021-05-02 00:00:00,2021-05-03 00:00:00,4000,86400 +SAMPLE_TABLE,---,freshness,2021-05-02 00:00:00,2021-05-03 00:00:00,40765000,86400 +SAMPLE_TABLE,---,my_distinct_table_rows,2021-05-02 00:00:00,2021-05-03 00:00:00,9000,86400 +SAMPLE_TABLE,---,my_custom_table_metric,2021-05-02 00:00:00,2021-05-03 00:00:00,1000000,86400 +SAMPLE_TABLE,---,distinct_table_rows,2021-05-02 00:00:00,2021-05-03 00:00:00,4000,86400 diff --git a/integration_tests/seeds/monitoring/expected_table_samples.csv b/integration_tests/seeds/monitoring/expected_table_samples.csv new file mode 100644 index 0000000..07c8f0b --- /dev/null +++ b/integration_tests/seeds/monitoring/expected_table_samples.csv @@ -0,0 +1,5 @@ +table_name,sample_data_length +BUY_EVENTS,506 +F2DQ_SOURCE_TEST_TABLE,361 +SAMPLE_TABLE,830 +SAMPLE_WITH_ANOMALY,507 diff --git a/integration_tests/seeds/monitoring/expected_test_history.csv b/integration_tests/seeds/monitoring/expected_test_history.csv new file mode 100644 index 0000000..28ed1cb --- /dev/null +++ b/integration_tests/seeds/monitoring/expected_test_history.csv @@ -0,0 +1,35 @@ +table_name,column_name,test_name,status,message,failures_count,severity +TEST_F2DQ_METRICS,---,pected_metrics_,Pass,---,0,ERROR +TEST_F2DQ_TABLE_SAMPLES,---,_table_samples_,Pass,---,0,ERROR +TEST_F2DQ_Z_SCORE,---,pected_z_score_,Pass,---,0,ERROR +TEST_F2DQ_ANOMALIES,---,cted_anomalies_,Pass,---,0,ERROR +BUY_EVENTS,---,_table_rows__10,Pass,---,0,ERROR +BUY_EVENTS,---,ts_row_count__1,Pass,---,0,ERROR +BUY_EVENTS,---,null__freshness,Pass,---,0,ERROR +BUY_EVENTS,VALUE2,5_02___min__200,Pass,---,0,ERROR +BUY_EVENTS,VALUE1,alue1__min__107,Pass,---,0,ERROR +BUY_EVENTS,VALUE1,ulls_percent__0,Pass,---,0,ERROR +BUY_EVENTS,VALUE1,5_02___min__100,Pass,---,0,ERROR +BUY_EVENTS,---,0__row_count__0,Pass,---,0,ERROR +F2DQ_METRICS,---,_buy_events___3,Pass,---,0,ERROR +F2DQ_METRICS,---,ref_buy_events_,Pass,---,0,ERROR +F2DQ_METRICS,---,ref_buy_events_,Pass,---,0,ERROR +BUY_EVENTS,VALUE1,__nulls_percent,Pass,---,0,ERROR +F2DQ_METRICS,---,ref_buy_events_,Pass,---,0,ERROR +TEST_F2DQ_TABLE_SAMPLES,---,_table_samples_,Pass,---,0,ERROR +TEST_F2DQ_ANOMALIES,---,cted_anomalies_,Pass,---,0,ERROR +BUY_EVENTS,---,ts_row_count__1,Pass,---,0,ERROR +BUY_EVENTS,---,null__freshness,Pass,---,0,ERROR +BUY_EVENTS,VALUE2,5_02___min__200,Pass,---,0,ERROR +BUY_EVENTS,---,_table_rows__10,Pass,---,0,ERROR +BUY_EVENTS,VALUE1,ulls_percent__0,Pass,---,0,ERROR +BUY_EVENTS,---,0__row_count__0,Pass,---,0,ERROR +BUY_EVENTS,VALUE1,5_02___min__100,Pass,---,0,ERROR +BUY_EVENTS,VALUE1,alue1__min__107,Pass,---,0,ERROR +BUY_EVENTS,VALUE1,__nulls_percent,Pass,---,0,ERROR +F2DQ_METRICS,---,_buy_events___3,Pass,---,0,ERROR +F2DQ_METRICS,---,ref_buy_events_,Pass,---,0,ERROR +F2DQ_METRICS,---,ref_buy_events_,Pass,---,0,ERROR +F2DQ_METRICS,---,ref_buy_events_,Pass,---,0,ERROR +TEST_F2DQ_METRICS,---,pected_metrics_,Pass,---,0,ERROR +TEST_F2DQ_Z_SCORE,---,pected_z_score_,Pass,---,0,ERROR diff --git a/integration_tests/seeds/monitoring/expected_z_score.csv b/integration_tests/seeds/monitoring/expected_z_score.csv new file mode 100644 index 0000000..49df41d --- /dev/null +++ b/integration_tests/seeds/monitoring/expected_z_score.csv @@ -0,0 +1,125 @@ +table_name,column_name,metric,time_window_end,z_score_value,modified_z_score_value,last_value,last_avg,last_stddev,last_median,last_iqr,last_median_absolute_deviation,last_mean_absolute_deviation,interval_length_sec +SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__avg_length,2021-05-03 00:00:00,0,0,14105,14105,0,14105,0,0,0,86400 +SAMPLE_TABLE,EVENT_TYPE,distinct_values,2021-05-03 00:00:00,-707,-674,1000,1500,707,1500,500,500,500,86400 +BUY_EVENTS,EVENT_TYPE,nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +SAMPLE_TABLE,EVENT_TYPE,max_length,2021-05-03 00:00:00,-707,-674,3000,3500,707,3500,500,500,500,86400 +SAMPLE_TABLE,EVENT_TYPE,nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +SAMPLE_TABLE,EVENT_TYPE,min_length,2021-05-03 00:00:00,0,0,3000,3000,0,3000,0,0,0,86400 +SAMPLE_TABLE,EVENT_TYPE,missing_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +SAMPLE_TABLE,NULL_VALUE,nulls_percent,2021-05-03 00:00:00,0,0,100000,100000,0,100000,0,0,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,RATING,global__avg_length,2021-05-03 00:00:00,0,0,3053,3053,0,3053,0,0,0,86400 +BUY_EVENTS,VALUE1,nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +SAMPLE_WITH_ANOMALY,EVENT_TYPE,nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +SAMPLE_TABLE,---,freshness,2021-05-03 00:00:00,0,0,40765000,40765000,0,40765000,0,0,0,86400 +F2DQ_SOURCE_TEST_TABLE,NUMBER,global__stddev,2021-05-03 00:00:00,0,0,3028,3028,0,3028,0,0,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,RATING,global__missing_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +BUY_EVENTS,EVENT_TYPE,min_length,2021-05-03 00:00:00,0,0,3000,3000,0,3000,0,0,0,86400 +SAMPLE_TABLE,VALUE1,nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +BUY_EVENTS,VALUE2,nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +SAMPLE_TABLE,VALUE1,variance,2021-05-03 00:00:00,707,674,2491667,2425000,94281,2425000,66667,66667,66667,86400 +F2DQ_SOURCE_TEST_TABLE,---,global__row_count,2021-05-03 00:00:00,0,0,10000,10000,0,10000,0,0,0,86400 +BUY_EVENTS,---,my_distinct_table_rows,2021-05-03 00:00:00,0,0,10000,10000,0,10000,0,0,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,---,global__row_count,2021-05-03 00:00:00,0,0,19000,19000,0,19000,0,0,0,86400 +SAMPLE_TABLE,EVENT_TYPE,match_regex,2021-05-03 00:00:00,-707,-674,0,500,707,500,500,500,500,86400 +SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__diff,2021-05-03 00:00:00,0,0,4000,4000,0,4000,0,0,0,86400 +BUY_EVENTS,VALUE2,diff,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +F2DQ_SOURCE_TEST_TABLE,NUMBER,global__avg,2021-05-03 00:00:00,0,0,5500,5500,0,5500,0,0,0,86400 +SAMPLE_TABLE,VALUE2,min,2021-05-03 00:00:00,0,0,109000,109000,0,109000,0,0,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__max_length,2021-05-03 00:00:00,0,0,17000,17000,0,17000,0,0,0,86400 +SAMPLE_WITH_ANOMALY,VALUE1,min,2021-05-03 00:00:00,-707,-674,98000,102500,6364,102500,4500,4500,4500,86400 +SAMPLE_WITH_ANOMALY,VALUE2,max,2021-05-03 00:00:00,707,674,205000,202500,3536,202500,2500,2500,2500,86400 +SAMPLE_WITH_ANOMALY,VALUE2,diff,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +SAMPLE_TABLE,EVENT_TYPE,not_match_regex_percent,2021-05-03 00:00:00,-707,-674,0,12500,17678,12500,12500,12500,12500,86400 +SAMPLE_TABLE,---,distinct_table_rows,2021-05-03 00:00:00,0,0,4000,4000,0,4000,0,0,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,RATING,global__missing_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +SAMPLE_WITH_ANOMALY,VALUE2,nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +SAMPLE_TABLE,VALUE2,stddev,2021-05-03 00:00:00,0,0,47975,47975,0,47975,0,0,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__min,2021-05-03 00:00:00,0,0,990,990,0,990,0,0,0,86400 +BUY_EVENTS,VALUE2,max,2021-05-03 00:00:00,707,674,205000,202500,3536,202500,2500,2500,2500,86400 +SAMPLE_WITHOUT_TIME_FILTER,---,global__my_distinct_table_rows,2021-05-03 00:00:00,0,0,19000,19000,0,19000,0,0,0,86400 +F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__avg_length,2021-05-03 00:00:00,0,0,3900,3900,0,3900,0,0,0,86400 +SAMPLE_WITH_ANOMALY,VALUE1,avg,2021-05-03 00:00:00,-707,-674,98000,102500,6364,102500,4500,4500,4500,86400 +SAMPLE_TABLE,VALUE2,variance,2021-05-03 00:00:00,0,0,2301583,2301583,0,2301583,0,0,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__missing_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +SAMPLE_WITH_ANOMALY,EVENT_TYPE,missing_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +BUY_EVENTS,VALUE1,nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__max,2021-05-03 00:00:00,0,0,4990,4990,0,4990,0,0,0,86400 +SAMPLE_WITH_ANOMALY,---,my_distinct_table_rows,2021-05-03 00:00:00,0,0,12000,12000,0,12000,0,0,0,86400 +BUY_EVENTS,VALUE1,diff,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__variance,2021-05-03 00:00:00,0,0,2784,2784,0,2784,0,0,0,86400 +SAMPLE_TABLE,EVENT_TYPE,duplicate_values,2021-05-03 00:00:00,0,0,1000,1000,0,1000,0,0,0,86400 +SAMPLE_TABLE,NULL_VALUE,nulls_count,2021-05-03 00:00:00,0,0,4000,4000,0,4000,0,0,0,86400 +F2DQ_SOURCE_TEST_TABLE,NUMBER,global__nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +F2DQ_SOURCE_TEST_TABLE,NUMBER,global__variance,2021-05-03 00:00:00,0,0,9167,9167,0,9167,0,0,0,86400 +SAMPLE_TABLE,EVENT_TYPE,avg_length,2021-05-03 00:00:00,-707,-674,3000,3125,177,3125,125,125,125,86400 +F2DQ_SOURCE_TEST_TABLE,NUMBER,global__min,2021-05-03 00:00:00,0,0,1000,1000,0,1000,0,0,0,86400 +F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__missing_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__avg,2021-05-03 00:00:00,0,0,3306,3306,0,3306,0,0,0,86400 +BUY_EVENTS,VALUE1,avg,2021-05-03 00:00:00,-707,-674,98000,102500,6364,102500,4500,4500,4500,86400 +SAMPLE_WITH_ANOMALY,VALUE2,min,2021-05-03 00:00:00,707,674,205000,202500,3536,202500,2500,2500,2500,86400 +F2DQ_SOURCE_TEST_TABLE,NUMBER,global__nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +SAMPLE_TABLE,EVENT_TYPE,not_match_regex,2021-05-03 00:00:00,-707,-674,0,500,707,500,500,500,500,86400 +BUY_EVENTS,VALUE1,max,2021-05-03 00:00:00,-707,-674,98000,102500,6364,102500,4500,4500,4500,86400 +SAMPLE_TABLE,VALUE1,stddev,2021-05-03 00:00:00,707,674,49917,49240,957,49240,677,677,677,86400 +BUY_EVENTS,VALUE1,min,2021-05-03 00:00:00,-707,-674,98000,102500,6364,102500,4500,4500,4500,86400 +SAMPLE_WITH_ANOMALY,VALUE2,nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +SAMPLE_TABLE,---,my_custom_table_metric,2021-05-03 00:00:00,0,0,1000000,1000000,0,1000000,0,0,0,86400 +BUY_EVENTS,EVENT_TYPE,max_length,2021-05-03 00:00:00,0,0,3000,3000,0,3000,0,0,0,86400 +SAMPLE_WITH_ANOMALY,EVENT_TYPE,avg_length,2021-05-03 00:00:00,0,0,3000,3000,0,3000,0,0,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +SAMPLE_WITH_ANOMALY,VALUE1,max,2021-05-03 00:00:00,-707,-674,98000,102500,6364,102500,4500,4500,4500,86400 +SAMPLE_TABLE,---,my_distinct_table_rows,2021-05-03 00:00:00,0,0,9000,9000,0,9000,0,0,0,86400 +SAMPLE_TABLE,---,row_count,2021-05-03 00:00:00,0,0,4000,4000,0,4000,0,0,0,86400 +SAMPLE_TABLE,VALUE2,max,2021-05-03 00:00:00,0,0,209000,209000,0,209000,0,0,0,86400 +SAMPLE_WITH_ANOMALY,VALUE1,diff,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +SAMPLE_WITH_ANOMALY,EVENT_TYPE,nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +BUY_EVENTS,EVENT_TYPE,avg_length,2021-05-03 00:00:00,0,0,3000,3000,0,3000,0,0,0,86400 +SAMPLE_TABLE,VALUE1,diff,2021-05-03 00:00:00,707,674,110000,105000,7071,105000,5000,5000,5000,86400 +F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__missing_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__missing_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +SAMPLE_TABLE,VALUE1,min,2021-05-03 00:00:00,0,0,100000,100000,0,100000,0,0,0,86400 +SAMPLE_TABLE,EVENT_TYPE,regex_test,2021-05-03 00:00:00,0,0,4000,4000,0,4000,0,0,0,86400 +SAMPLE_TABLE,EVENT_TYPE,nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,RATING,global__min_length,2021-05-03 00:00:00,0,0,1000,1000,0,1000,0,0,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__min_length,2021-05-03 00:00:00,0,0,10000,10000,0,10000,0,0,0,86400 +SAMPLE_WITH_ANOMALY,VALUE1,nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +F2DQ_SOURCE_TEST_TABLE,NUMBER,global__max,2021-05-03 00:00:00,0,0,10000,10000,0,10000,0,0,0,86400 +SAMPLE_WITH_ANOMALY,EVENT_TYPE,missing_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +SAMPLE_TABLE,VALUE2,nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +SAMPLE_WITH_ANOMALY,VALUE2,avg,2021-05-03 00:00:00,707,674,205000,202500,3536,202500,2500,2500,2500,86400 +BUY_EVENTS,---,freshness,2021-05-03 00:00:00,-707,-674,41065000,41186500,171827,41186500,121500,121500,121500,86400 +BUY_EVENTS,VALUE2,avg,2021-05-03 00:00:00,707,674,205000,202500,3536,202500,2500,2500,2500,86400 +BUY_EVENTS,---,row_count,2021-05-03 00:00:00,0,0,1000,1000,0,1000,0,0,0,86400 +SAMPLE_WITH_ANOMALY,---,freshness,2021-05-03 00:00:00,-707,-674,41065000,41186500,171827,41186500,121500,121500,121500,86400 +SAMPLE_WITHOUT_TIME_FILTER,RATING,global__max_length,2021-05-03 00:00:00,0,0,5000,5000,0,5000,0,0,0,86400 +SAMPLE_WITH_ANOMALY,EVENT_TYPE,min_length,2021-05-03 00:00:00,0,0,3000,3000,0,3000,0,0,0,86400 +SAMPLE_TABLE,EVENT_TYPE,match_regex_percent,2021-05-03 00:00:00,-707,-674,0,12500,17678,12500,12500,12500,12500,86400 +SAMPLE_WITH_ANOMALY,EVENT_TYPE,max_length,2021-05-03 00:00:00,0,0,3000,3000,0,3000,0,0,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__stddev,2021-05-03 00:00:00,0,0,1668,1668,0,1668,0,0,0,86400 +SAMPLE_TABLE,VALUE1,max,2021-05-03 00:00:00,707,674,210000,205000,7071,205000,5000,5000,5000,86400 +SAMPLE_TABLE,VALUE2,avg,2021-05-03 00:00:00,0,0,180750,180750,0,180750,0,0,0,86400 +SAMPLE_WITH_ANOMALY,VALUE1,nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__min_length,2021-05-03 00:00:00,0,0,3000,3000,0,3000,0,0,0,86400 +SAMPLE_TABLE,VALUE1,nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +BUY_EVENTS,EVENT_TYPE,nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +BUY_EVENTS,EVENT_TYPE,missing_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +BUY_EVENTS,EVENT_TYPE,missing_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,RATING,global__nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +BUY_EVENTS,VALUE2,nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +F2DQ_SOURCE_TEST_TABLE,NUMBER,global__diff,2021-05-03 00:00:00,0,0,9000,9000,0,9000,0,0,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,RATING,global__nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +BUY_EVENTS,VALUE2,min,2021-05-03 00:00:00,707,674,205000,202500,3536,202500,2500,2500,2500,86400 +F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__max_length,2021-05-03 00:00:00,0,0,5000,5000,0,5000,0,0,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +SAMPLE_TABLE,VALUE2,nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +SAMPLE_TABLE,EVENT_TYPE,duplicate_rows,2021-05-03 00:00:00,707,674,4000,3500,707,3500,500,500,500,86400 +F2DQ_SOURCE_TEST_TABLE,---,global__my_distinct_table_rows,2021-05-03 00:00:00,0,0,10000,10000,0,10000,0,0,0,86400 +SAMPLE_TABLE,VALUE2,diff,2021-05-03 00:00:00,0,0,100000,100000,0,100000,0,0,0,86400 +SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 +SAMPLE_WITH_ANOMALY,---,row_count,2021-05-03 00:00:00,0,0,1000,1000,0,1000,0,0,0,86400 +SAMPLE_TABLE,EVENT_TYPE,unique_rows,2021-05-03 00:00:00,-707,-674,0,500,707,500,500,500,500,86400 +SAMPLE_TABLE,VALUE1,avg,2021-05-03 00:00:00,707,674,142500,135000,10607,135000,7500,7500,7500,86400 +SAMPLE_TABLE,EVENT_TYPE,missing_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400 diff --git a/integration_tests/seeds/monitoring/sample_table.csv b/integration_tests/seeds/monitoring/sample_table.csv new file mode 100644 index 0000000..4d1021e --- /dev/null +++ b/integration_tests/seeds/monitoring/sample_table.csv @@ -0,0 +1,10 @@ +creation_time,update_time,event_type,value1,value2,null_value,not_used_colum +2021-04-30 12:40:35,,buy,100,109,,1 +2021-05-01 12:31:32,,buy,100,200,,1 +2021-05-01 12:35:35,,buy,110,205,,1 +2021-05-01 12:40:35,,sell,200,209,,1 +2021-05-01 12:40:35,2021-05-01 12:40:37,buy,100,109,,1 +2021-05-02 12:31:32,,buy,110,200,,1 +2021-05-02 12:35:35,,buy,150,205,,1 +2021-05-02 12:40:35,,buy,210,209,,1 +2021-05-02 12:40:35,,buy,100,109,,1 diff --git a/integration_tests/seeds/monitoring/sample_with_anomaly.csv b/integration_tests/seeds/monitoring/sample_with_anomaly.csv new file mode 100644 index 0000000..ff54aeb --- /dev/null +++ b/integration_tests/seeds/monitoring/sample_with_anomaly.csv @@ -0,0 +1,13 @@ +creation_time,event_type,value1,value2 +2021-04-30 12:40:35,buy,101,109 +2021-05-01 12:31:32,buy,107,200 +2021-05-02 12:35:35,buy,98,205 +2021-05-03 12:40:35,sell,108,209 +2021-05-04 12:40:35,buy,100,109 +2021-05-05 12:31:32,buy,110,200 +2021-05-06 12:35:35,buy,99,205 +2021-05-07 12:40:35,buy,94,209 +2021-05-08 12:40:35,buy,104,109 +2021-05-09 12:31:32,buy,10,200 +2021-05-10 12:35:35,buy,23,205 +2021-05-11 12:40:35,sell,10,209 diff --git a/integration_tests/seeds/monitoring/sample_without_time_filter.csv b/integration_tests/seeds/monitoring/sample_without_time_filter.csv new file mode 100644 index 0000000..e116410 --- /dev/null +++ b/integration_tests/seeds/monitoring/sample_without_time_filter.csv @@ -0,0 +1,20 @@ +title,rental_rate,rating +Academy Dinosaur,0.99,PG-13 +Alamo Videotape,0.99,G +Affair Prejudice,2.99,G +African Egg,2.99,G +Ace Goldfinger,4.99,G +Alice Fantasia,0.99,NC-17 +Adaptation Holes,2.99,NC-17 +Alien Center,2.99,NC-17 +Aladdin Calendar,4.99,NC-17 +Chamber Italian,4.99,NC-17 +Alaska Phantom,0.99,PG +Agent Truman,2.99,PG +Ali Forever,4.99,PG +Alabama Devil,2.99,PG-13 +Bright Encounters,4.99,PG-13 +Airplane Sierra,4.99,PG-13 +Date Speed,0.99,R +Grosse Wonderful,4.99,R +Airport Pollock,4.99,R \ No newline at end of file diff --git a/integration_tests/seeds/public_macros/cleaning/expected_sample_user_data.csv b/integration_tests/seeds/public_macros/cleaning/expected_sample_user_data.csv new file mode 100644 index 0000000..ca45723 --- /dev/null +++ b/integration_tests/seeds/public_macros/cleaning/expected_sample_user_data.csv @@ -0,0 +1,21 @@ +"full_name","email" +"Lizzie Effertz","*****@fakemail.com" +"Orlando Abbott","*****@fakemail.com" +"Kelley Harann","*****@fakemail.com" +"Ruth Langworth","*****@fakemail.com" +"Lane Swift","*****@fakemail.com" +"Bertha Corwin","*****@fakemail.com" +"Manuela Kling","*****@fakemail.com" +"Mose Balistreri","*****@fakemail.com" +"Robin Halvorson","*****@fakemail.com" +"Osbaldo Parker I","*****@fakemail.com" +"Javier Runolfsson","*****@fakemail.net" +"Amelia Batz","*****@fakemail.com" +"Abby Pouros","*****@fakemail.com" +"Markus Homenick","*****@fakemail.com" +"Braeden Turner","*****@fakemail.com" +"Horacio Parker","*****@fakemail.info" +"Ms. Stacy Padberg","*****@fakemail.com" +"Dr. Deshawn Stracke","*****@fakemail.com" +"Pascale Grady","*****@fakemail.com" +"Lacy Brekke","*****@fakemail.com" diff --git a/integration_tests/seeds/public_macros/cleaning/sample_user_data.csv b/integration_tests/seeds/public_macros/cleaning/sample_user_data.csv new file mode 100644 index 0000000..40aefd7 --- /dev/null +++ b/integration_tests/seeds/public_macros/cleaning/sample_user_data.csv @@ -0,0 +1,21 @@ +full_name,email + lizzie effertz,torp.trisha@fakemail.com + orlando abbott,dayton.hermiston@fakemail.com +kelley harann,borer.blake@fakemail.com +ruth langworth,garett66@fakemail.com +lane swift ,nienow.coralie@fakemail.com +bertha corwin ,tstroman@fakemail.com +manuela kling,shawn.langworth@fakemail.com +mose balistreri,dorris70@fakemail.com +robin halvorson,murazik.americo@fakemail.com + osbaldo parker i ,friesen.angeline@fakemail.com +javier runolfsson ,benjamin.bailey@fakemail.net +amelia batz,garrison60@fakemail.com +abby pouros,dominique.leannon@fakemail.com +markus homenick,piper73@fakemail.com +braeden turner,kozey.jace@fakemail.com +horacio parker,vtillman@fakemail.info +ms. stacy padberg,erdman.elaina@fakemail.com +dr. deshawn stracke,rosendo.beer@fakemail.com + pascale grady,princess60@fakemail.com +lacy brekke,romaguera.darrell@fakemail.com diff --git a/integration_tests/seeds/public_macros/filtering/duplicated.csv b/integration_tests/seeds/public_macros/filtering/duplicated.csv new file mode 100644 index 0000000..5e37a69 --- /dev/null +++ b/integration_tests/seeds/public_macros/filtering/duplicated.csv @@ -0,0 +1,9 @@ +transaction_id,creation_time,status,value +1,2021-05-01 12:31:32,pending,100 +2,2021-05-01 12:35:35,pending,200 +1,2021-05-01 12:40:35,completed,100 +3,2021-05-01 12:40:35,pending,300 +3,2021-05-02 12:31:32,completed,300 +4,2021-05-02 12:35:35,completed,10 +5,2021-05-02 12:40:35,pending,100 +4,2021-05-02 12:40:35,completed,40 diff --git a/integration_tests/seeds/public_macros/filtering/expected_deduplicated.csv b/integration_tests/seeds/public_macros/filtering/expected_deduplicated.csv new file mode 100644 index 0000000..884d28f --- /dev/null +++ b/integration_tests/seeds/public_macros/filtering/expected_deduplicated.csv @@ -0,0 +1,18 @@ +transaction_id,creation_time,status,value,use_case +1,2021-05-01 12:31:32,pending,100,take_first +2,2021-05-01 12:35:35,pending,200,take_first +3,2021-05-01 12:40:35,pending,300,take_first +4,2021-05-02 12:35:35,completed,10,take_first +5,2021-05-02 12:40:35,pending,100,take_first +1,2021-05-01 12:40:35,completed,100,take_last +2,2021-05-01 12:35:35,pending,200,take_last +3,2021-05-02 12:31:32,completed,300,take_last +4,2021-05-02 12:40:35,completed,40,take_last +5,2021-05-02 12:40:35,pending,100,take_last +1,2021-05-01 12:40:35,completed,100,take_all_statuses +1,2021-05-01 12:31:32,pending,100,take_all_statuses +2,2021-05-01 12:35:35,pending,200,take_all_statuses +3,2021-05-02 12:31:32,completed,300,take_all_statuses +3,2021-05-01 12:40:35,pending,300,take_all_statuses +4,2021-05-02 12:40:35,completed,40,take_all_statuses +5,2021-05-02 12:40:35,pending,100,take_all_statuses diff --git a/integration_tests/seeds/public_macros/filtering/expected_duplicates.csv b/integration_tests/seeds/public_macros/filtering/expected_duplicates.csv new file mode 100644 index 0000000..254f6b8 --- /dev/null +++ b/integration_tests/seeds/public_macros/filtering/expected_duplicates.csv @@ -0,0 +1,7 @@ +"transaction_id","creation_time","status","value","f2dq_duplicate_group_row_count","f2dq_duplicate_group_row_number" +1,"2021-05-01 12:31:32","pending",100,2,1 +1,"2021-05-01 12:40:35","completed",100,2,2 +3,"2021-05-01 12:40:35","pending",300,2,1 +3,"2021-05-02 12:31:32","completed",300,2,2 +4,"2021-05-02 12:35:35","completed",10,2,1 +4,"2021-05-02 12:40:35","completed",40,2,2 diff --git a/integration_tests/seeds/public_macros/normalizing/abbreviated_us_states.csv b/integration_tests/seeds/public_macros/normalizing/abbreviated_us_states.csv new file mode 100644 index 0000000..a2c5fea --- /dev/null +++ b/integration_tests/seeds/public_macros/normalizing/abbreviated_us_states.csv @@ -0,0 +1,52 @@ +state,code +Ala.,AL +Alaska,AK +Ariz.,AZ +Ark.,AR +Calif.,CA +Colo.,CO +Conn.,CT +Del.,DE +D.C.,DC +Fla.,FL +Ga.,GA +Hawaii,HI +Idaho,ID +Ill.,IL +Ind.,IN +Iowa,IA +Kans.,KS +Ky.,KY +La.,LA +Maine,ME +Md.,MD +Mass.,MA +Mich.,MI +Minn.,MN +Miss.,MS +Mo.,MO +Mont.,MT +Nebr.,NE +Nev.,NV +N.H.,NH +N.J.,NJ +N.M.,NM +N.Y.,NY +N.C.,NC +N.D.,ND +Ohio,OH +Okla.,OK +Ore.,OR +Pa.,PA +R.I.,RI +S.C.,SC +S.D.,SD +Tenn.,TN +Tex.,TX +Utah,UT +Vt.,VT +Va.,VA +Wash.,WA +W.Va.,WV +Wis.,WI +Wyo.,WY diff --git a/integration_tests/seeds/public_macros/normalizing/expected_us_states_normalized.csv b/integration_tests/seeds/public_macros/normalizing/expected_us_states_normalized.csv new file mode 100644 index 0000000..7068963 --- /dev/null +++ b/integration_tests/seeds/public_macros/normalizing/expected_us_states_normalized.csv @@ -0,0 +1,52 @@ +"state","code","state__normalized" +"Ariz.","AZ","Arizona" +"Ind.","IN","Indiana" +"La.","LA","Louisiana" +"W.Va.","WV","West Virginia" +"Nebr.","NE","Nebraska" +"Pa.","PA","Pennsylvania" +"Iowa","IA","Iowa" +"N.H.","NH","New Hampshire" +"S.C.","SC","South Carolina" +"Ore.","OR","Oregon" +"Conn.","CT","Connecticut" +"R.I.","RI","Rhode Island" +"Minn.","MN","Minnesota" +"D.C.","DC","District of Columbia" +"Wyo.","WY","Wyoming" +"Hawaii","HI","Hawaii" +"Wash.","WA","Washington" +"N.D.","ND","North Dakota" +"Mass.","MA","Massachusetts" +"N.Y.","NY","New York" +"N.M.","NM","New Mexico" +"Colo.","CO","Colorado" +"Ohio","OH","Ohio" +"Idaho","ID","Idaho" +"Ala.","AL","Alabama" +"Ark.","AR","Arkansas" +"S.D.","SD","South Dakota" +"Mo.","MO","Missouri" +"N.J.","NJ","New Jersey" +"Miss.","MS","Mississippi" +"Kans.","KS","Kansas" +"Vt.","VT","Vermont" +"Calif.","CA","California" +"Mich.","MI","Michigan" +"Alaska","AK","Alaska" +"Nev.","NV","Nevada" +"Okla.","OK","Oklahoma" +"Tenn.","TN","Tennessee" +"Ga.","GA","Georgia" +"Wis.","WI","Wisconsin" +"Ky.","KY","Kentucky" +"N.C.","NC","North Carolina" +"Mont.","MT","Montana" +"Fla.","FL","Florida" +"Va.","VA","Virginia" +"Tex.","TX","Texas" +"Md.","MD","Maryland" +"Utah","UT","Utah" +"Maine","ME","Maine" +"Del.","DE","Delaware" +"Ill.","IL","Illinois" diff --git a/integration_tests/seeds/public_macros/normalizing/us_states_normalization.csv b/integration_tests/seeds/public_macros/normalizing/us_states_normalization.csv new file mode 100644 index 0000000..cb74e40 --- /dev/null +++ b/integration_tests/seeds/public_macros/normalizing/us_states_normalization.csv @@ -0,0 +1,52 @@ +source,target +Ala.,Alabama +Alaska,Alaska +Ariz.,Arizona +Ark.,Arkansas +Calif.,California +Colo.,Colorado +Conn.,Connecticut +Del.,Delaware +D.C.,District of Columbia +Fla.,Florida +Ga.,Georgia +Hawaii,Hawaii +Idaho,Idaho +Ill.,Illinois +Ind.,Indiana +Iowa,Iowa +Kans.,Kansas +Ky.,Kentucky +La.,Louisiana +Maine,Maine +Md.,Maryland +Mass.,Massachusetts +Mich.,Michigan +Minn.,Minnesota +Miss.,Mississippi +Mo.,Missouri +Mont.,Montana +Nebr.,Nebraska +Nev.,Nevada +N.H.,New Hampshire +N.J.,New Jersey +N.M.,New Mexico +N.Y.,New York +N.C.,North Carolina +N.D.,North Dakota +Ohio,Ohio +Okla.,Oklahoma +Ore.,Oregon +Pa.,Pennsylvania +R.I.,Rhode Island +S.C.,South Carolina +S.D.,South Dakota +Tenn.,Tennessee +Tex.,Texas +Utah,Utah +Vt.,Vermont +Va.,Virginia +Wash.,Washington +W.Va.,West Virginia +Wis.,Wisconsin +Wyo.,Wyoming \ No newline at end of file diff --git a/integration_tests/seeds/public_macros/validating/expected_validated_credit_cards.csv b/integration_tests/seeds/public_macros/validating/expected_validated_credit_cards.csv new file mode 100644 index 0000000..14eca81 --- /dev/null +++ b/integration_tests/seeds/public_macros/validating/expected_validated_credit_cards.csv @@ -0,0 +1,11 @@ +credit_card_number,valid_credit_card +4941533405630082,1 +5476749195896614,1 +3568497486294461,1 +not_a_card_number,0 +3434-4351-4234-3234,0 +3434 4351 4234 3234,0 +34344 42344 43455 43456,0 +43423432,0 +234343443434,0 +2343423423423423423423423423,0 diff --git a/integration_tests/seeds/public_macros/validating/expected_validated_date_and_time.csv b/integration_tests/seeds/public_macros/validating/expected_validated_date_and_time.csv new file mode 100644 index 0000000..9b5d485 --- /dev/null +++ b/integration_tests/seeds/public_macros/validating/expected_validated_date_and_time.csv @@ -0,0 +1,13 @@ +date_time,valid_date_eu,valid_date_us,valid_date_inverse,valid_date_iso_8601,valid_time_24h,valid_time_12h,valid_time +31-01-2020,1,0,0,0,0,0,0 +01/31/2020,0,1,0,0,0,0,0 +05.05.2020,1,1,0,0,0,0,0 +2020-01-31,0,0,1,0,0,0,0 +23:59,0,0,0,0,1,0,1 +12:59,0,0,0,0,1,1,1 +13:59:01,0,0,0,0,0,0,1 +"12:59:01,55",0,0,0,0,0,0,1 +11:59:00,0,0,0,0,0,0,1 +midnight,0,0,0,0,0,0,0 +2020-01-31T12:59:00+02:00,0,0,0,1,0,0,0 +2020-01-31T12:59:00,0,0,0,1,0,0,0 diff --git a/integration_tests/seeds/public_macros/validating/expected_validated_emails.csv b/integration_tests/seeds/public_macros/validating/expected_validated_emails.csv new file mode 100644 index 0000000..640ac93 --- /dev/null +++ b/integration_tests/seeds/public_macros/validating/expected_validated_emails.csv @@ -0,0 +1,5 @@ +user_id,email,email_valid +1,test@fakemail.com,1 +2,novalidemail@,0 +3,novalidemail@com,0 +4,test+alovalidemail@fakemail.com,0 diff --git a/integration_tests/seeds/public_macros/validating/expected_validated_ips.csv b/integration_tests/seeds/public_macros/validating/expected_validated_ips.csv new file mode 100644 index 0000000..9982983 --- /dev/null +++ b/integration_tests/seeds/public_macros/validating/expected_validated_ips.csv @@ -0,0 +1,29 @@ +ip_address,valid_ip_v4,valid_ip_v6,valid_ip +1.2.3.4,1,0,1 +01.102.103.104,1,0,1 +124.171.228.4,1,0,1 +192.168.1.35,1,0,1 +192.168.1.198,1,0,1 +127.248.111.240,1,0,1 +01.1.1,0,0,0 +12325412,0,0,0 +notvalidatall,0,0,0 +232.232.33,0,0,0 +232.3232.232.232+2312,0,0,0 +::::erwerwe,0,0,0 +::3343:4343434343:34343:343434343:443,0,0,0 +,0,0,0 +2001:db8:3333:4444:5555:6666:7777:8888,0,1,1 +2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF,0,1,1 +2001:db8::,0,1,1 +::1234:5678,0,1,1 +2001:db8::1234:5678,0,1,1 +::11.22.33.44,0,1,1 +2001:db8::123.123.123.123,0,1,1 +2001:db8::1234:5678:5.6.7.8,0,1,1 +2001:db8:3333:4444:5555:6666:1.2.3.4,0,1,1 +::11.22.33.44,0,1,1 +2001:db8::123.123.123.123,0,1,1 +::1234:5678:91.123.4.56,0,1,1 +::1234:5678:1.2.3.4,0,1,1 +2001:db8::1234:5678:5.6.7.8,0,1,1 diff --git a/integration_tests/seeds/public_macros/validating/expected_validated_numbers.csv b/integration_tests/seeds/public_macros/validating/expected_validated_numbers.csv new file mode 100644 index 0000000..5ddcda2 --- /dev/null +++ b/integration_tests/seeds/public_macros/validating/expected_validated_numbers.csv @@ -0,0 +1,12 @@ +number,is_number,is_number_decimal_point,is_number_decimal_comma,is_percentage,is_percentage_decimal_point,is_percentage_decimal_comma +133,1,0,0,0,0,0 +1232.232,0,1,0,0,0,0 +"2332,123",0,0,1,0,0,0 +not a number,0,0,0,0,0,0 +"1,3%",0,0,0,1,0,1 +123%,0,0,0,1,1,1 +13 %,0,0,0,0,0,0 +76.234%,0,0,0,1,1,0 +not a number,0,0,0,0,0,0 +x,0,0,0,0,0,0 +123partly987,0,0,0,0,0,0 diff --git a/integration_tests/seeds/public_macros/validating/expected_validated_uuids.csv b/integration_tests/seeds/public_macros/validating/expected_validated_uuids.csv new file mode 100644 index 0000000..e23034f --- /dev/null +++ b/integration_tests/seeds/public_macros/validating/expected_validated_uuids.csv @@ -0,0 +1,8 @@ +uuid,valid_uuid +ace1245c-3af5-11ec-8d3d-0242ac130003,1 +a568464e-a05d-412c-8b30-517a46c57d88,1 +notanuid,0 +d0d61836-3af5-11ec-8d3d-0242ac130003,1 +d0d61c6e-3af5-11ec-8d3d-0242ac130003,1 +343422-234324-234234-4234234-23432,0 +343422-234324-234234-4234234-234xxx32,0 diff --git a/integration_tests/seeds/public_macros/validating/validate_date_and_time.csv b/integration_tests/seeds/public_macros/validating/validate_date_and_time.csv new file mode 100644 index 0000000..1a9f767 --- /dev/null +++ b/integration_tests/seeds/public_macros/validating/validate_date_and_time.csv @@ -0,0 +1,13 @@ +date_time +31-01-2020 +01/31/2020 +05.05.2020 +2020-01-31 +23:59 +12:59 +13:59:01 +"12:59:01,55" +11:59:00 +midnight +2020-01-31T12:59:00+02:00 +2020-01-31T12:59:00 \ No newline at end of file diff --git a/integration_tests/seeds/public_macros/validating/validate_emails.csv b/integration_tests/seeds/public_macros/validating/validate_emails.csv new file mode 100644 index 0000000..d7a1982 --- /dev/null +++ b/integration_tests/seeds/public_macros/validating/validate_emails.csv @@ -0,0 +1,5 @@ +user_id,email +1,test@fakemail.com +2,novalidemail@ +3,novalidemail@com +4,test+alovalidemail@fakemail.com diff --git a/integration_tests/seeds/public_macros/validating/validate_ip.csv b/integration_tests/seeds/public_macros/validating/validate_ip.csv new file mode 100644 index 0000000..cdc743b --- /dev/null +++ b/integration_tests/seeds/public_macros/validating/validate_ip.csv @@ -0,0 +1,29 @@ +ip_address +1.2.3.4 +01.102.103.104 +124.171.228.4 +192.168.1.35 +192.168.1.198 +127.248.111.240 +01.1.1 +12325412 +notvalidatall +232.232.33 +232.3232.232.232+2312 +::::erwerwe +::3343:4343434343:34343:343434343:443 + +2001:db8:3333:4444:5555:6666:7777:8888 +2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF +2001:db8:: +::1234:5678 +2001:db8::1234:5678 +::11.22.33.44 +2001:db8::123.123.123.123 +2001:db8::1234:5678:5.6.7.8 +2001:db8:3333:4444:5555:6666:1.2.3.4 +::11.22.33.44 +2001:db8::123.123.123.123 +::1234:5678:91.123.4.56 +::1234:5678:1.2.3.4 +2001:db8::1234:5678:5.6.7.8 \ No newline at end of file diff --git a/integration_tests/seeds/public_macros/validating/validate_numbers.csv b/integration_tests/seeds/public_macros/validating/validate_numbers.csv new file mode 100644 index 0000000..fad0aab --- /dev/null +++ b/integration_tests/seeds/public_macros/validating/validate_numbers.csv @@ -0,0 +1,12 @@ +number +"133" +"1232.232" +"2332,123" +"not a number" +"1,3%" +"123%" +"13 %" +"76.234%" +"not a number" +"x" +"123partly987" diff --git a/integration_tests/seeds/public_macros/validating/validate_uuid.csv b/integration_tests/seeds/public_macros/validating/validate_uuid.csv new file mode 100644 index 0000000..c4866ed --- /dev/null +++ b/integration_tests/seeds/public_macros/validating/validate_uuid.csv @@ -0,0 +1,8 @@ +uuid +ace1245c-3af5-11ec-8d3d-0242ac130003 +a568464e-a05d-412c-8b30-517a46c57d88 +notanuid +d0d61836-3af5-11ec-8d3d-0242ac130003 +d0d61c6e-3af5-11ec-8d3d-0242ac130003 +343422-234324-234234-4234234-23432 +343422-234324-234234-4234234-234xxx32 \ No newline at end of file diff --git a/integration_tests/snapshots/.gitkeep b/integration_tests/snapshots/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/integration_tests/tests/.gitkeep b/integration_tests/tests/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/macros/.gitkeep b/macros/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/macros/config/get_model_config.sql b/macros/config/get_model_config.sql new file mode 100644 index 0000000..2153636 --- /dev/null +++ b/macros/config/get_model_config.sql @@ -0,0 +1,41 @@ + +{% macro get_model_config(monitored) %} + {% set model = {} %} + {% do model.update({'name': f2dq.row_value(monitored, 'name')}) %} + {% do model.update({'schema': f2dq.row_value(monitored, 'schema')}) %} + {% do model.update({'database': f2dq.row_value(monitored, 'database')}) %} + {% do model.update({'time_filter': f2dq.row_value(monitored, 'time_filter')}) %} + {% do model.update({'metrics': fromjson(f2dq.row_value(monitored, 'metrics'))}) %} + {% do model.update({'model_name': model.get('database') + '.' + model.get('schema') + '.' + model.get('name')}) %} + {% do model.update({'table_name': full_table_name_values(model.get('name'), model.get('schema'), model.get('database'))}) %} + + {% set columns_db = f2dq.row_value(monitored, 'columns') %} + + {% set column_list = fromjson(columns_db) if columns_db is not none else none %} + {% set columns_dict = f2dq.dict_from_list(column_list) %} + + {% do model.update({'columns_dict': columns_dict}) %} + {% do model.update({'columns_compute_all': columns_dict is none}) %} + + {% set columns_query %} + select * from {{ ref('f2dq_columns') }} + where name = '{{ model.name }}' and schema = '{{ model.schema }}' and database = '{{ model.database }}' + {% endset %} + {% set columns = run_query(columns_query) %} + + {% set columns_info = {} %} + {% for col in columns %} + {% set column_name = f2dq.row_value(col, 'column_name') %} + {% set data_type = f2dq.get_column_type(col) %} + {% do columns_info.update({column_name: { 'data_type': data_type }}) %} + {% endfor %} + + {% do model.update({'columns_info': columns_info}) %} + {% do model.update({'columns': columns}) %} + + {{ return(model) }} +{% endmacro %} + +{% macro should_compute_metric(model, column_name) %} + {{ return(model.columns_compute_all or model.columns_dict.get(column_name)) }} +{% endmacro %} \ No newline at end of file diff --git a/macros/db/bigquery/quote_column_name.sql b/macros/db/bigquery/quote_column_name.sql new file mode 100644 index 0000000..f8124a7 --- /dev/null +++ b/macros/db/bigquery/quote_column_name.sql @@ -0,0 +1,4 @@ +{% macro bigquery__quote_column_name(column_name) %} + {% set quoted_col_name = '`' + column_name + '`' %} + {{ return(quoted_col_name) }} +{% endmacro %} \ No newline at end of file diff --git a/macros/db/bigquery/quote_string.sql b/macros/db/bigquery/quote_string.sql new file mode 100644 index 0000000..b472e12 --- /dev/null +++ b/macros/db/bigquery/quote_string.sql @@ -0,0 +1,4 @@ + +{%- macro bigquery__quote_string(str) %} + r"""{{ str }}""" +{% endmacro %} \ No newline at end of file diff --git a/macros/db/bigquery/split_and_return_nth_value.sql b/macros/db/bigquery/split_and_return_nth_value.sql new file mode 100644 index 0000000..28ff01e --- /dev/null +++ b/macros/db/bigquery/split_and_return_nth_value.sql @@ -0,0 +1,3 @@ +{% macro bigquery__split_and_return_nth_value(column_name, delimiter, ordinal) %} + split({{ f2dq.clean_blacklist(column_name, ['"', '`'], '') }}, '{{ delimiter }}')[ORDINAL( {{ ordinal }} )] +{% endmacro %} \ No newline at end of file diff --git a/macros/db/core/escape.sql b/macros/db/core/escape.sql new file mode 100644 index 0000000..7d23ee3 --- /dev/null +++ b/macros/db/core/escape.sql @@ -0,0 +1,4 @@ + +{% macro escape_seq_for_json(chr) %}{{adapter.dispatch('escape_seq_for_json', 'f2dq')(chr)}}{% endmacro %} + +{% macro default__escape_seq_for_json(chr) %}'\\\{{chr}}'{% endmacro %} \ No newline at end of file diff --git a/macros/db/core/identifier_mapping.sql b/macros/db/core/identifier_mapping.sql new file mode 100644 index 0000000..270bd89 --- /dev/null +++ b/macros/db/core/identifier_mapping.sql @@ -0,0 +1,10 @@ + +{% macro name_in_db(name) %} + {% set translated = adapter.dispatch('name_in_db', 'f2dq')(name) %} + {{ return(translated) }} + +{% endmacro %} + +{% macro default__name_in_db(name) %} + {{ return(name) }} +{% endmacro %} \ No newline at end of file diff --git a/macros/db/core/quote_column_name.sql b/macros/db/core/quote_column_name.sql new file mode 100644 index 0000000..39c3361 --- /dev/null +++ b/macros/db/core/quote_column_name.sql @@ -0,0 +1,10 @@ +{% macro quote_column_name(column_name) %} + {% set col_name = adapter.dispatch('quote_column_name', 'f2dq')(column_name) %} + {{ return(col_name) }} +{% endmacro %} + + +{% macro default__quote_column_name(column_name) %} + {% set quoted_col_name = '"' + column_name + '"' %} + {{ return(quoted_col_name) }} +{% endmacro %} \ No newline at end of file diff --git a/macros/db/core/quote_string.sql b/macros/db/core/quote_string.sql new file mode 100644 index 0000000..2183752 --- /dev/null +++ b/macros/db/core/quote_string.sql @@ -0,0 +1,13 @@ + +{%- macro quote_string(str) %} + {{ adapter.dispatch('quote_string', 'f2dq')(str) }} +{% endmacro %} + +{%- macro default__quote_string(str) %} + $${{ str }}$$ +{% endmacro %} + + +{%- macro quote_new_line() %}{{ adapter.dispatch('quote_new_line', 'f2dq')() }}{% endmacro %} + +{%- macro default__quote_new_line() %}'\134\134n'{% endmacro %} \ No newline at end of file diff --git a/macros/db/core/split_and_return_nth_value.sql b/macros/db/core/split_and_return_nth_value.sql new file mode 100644 index 0000000..e9c4d0b --- /dev/null +++ b/macros/db/core/split_and_return_nth_value.sql @@ -0,0 +1,7 @@ +{% macro split_and_return_nth_value(column_name, delimiter, ordinal) -%} + {{ adapter.dispatch('split_and_return_nth_value', 'f2dq')(column_name, delimiter, ordinal) }} +{%- endmacro %} + +{% macro default__split_and_return_nth_value(column_name, delimiter, ordinal) -%} + split_part({{ f2dq.clean_blacklist(column_name, ['"', '`'], '') }}, '{{ delimiter }}', {{ ordinal }}) +{%- endmacro %} \ No newline at end of file diff --git a/macros/db/postgres/escape.sql b/macros/db/postgres/escape.sql new file mode 100644 index 0000000..105c2ad --- /dev/null +++ b/macros/db/postgres/escape.sql @@ -0,0 +1 @@ +{% macro postgres__escape_seq_for_json(chr) %}'\{{chr}}'{% endmacro %} \ No newline at end of file diff --git a/macros/db/postgres/postgres_type_db.sql b/macros/db/postgres/postgres_type_db.sql new file mode 100644 index 0000000..725e728 --- /dev/null +++ b/macros/db/postgres/postgres_type_db.sql @@ -0,0 +1,3 @@ +{% macro postgres_type_db() %} + {{ ('postgres', 'greenplum') }} +{% endmacro %} \ No newline at end of file diff --git a/macros/db/postgres/quote_string.sql b/macros/db/postgres/quote_string.sql new file mode 100644 index 0000000..d8e6f87 --- /dev/null +++ b/macros/db/postgres/quote_string.sql @@ -0,0 +1,3 @@ + + +{%- macro postgres__quote_new_line() %}'\\n'{% endmacro %} \ No newline at end of file diff --git a/macros/db/redshift/escape.sql b/macros/db/redshift/escape.sql new file mode 100644 index 0000000..74f4447 --- /dev/null +++ b/macros/db/redshift/escape.sql @@ -0,0 +1,2 @@ + +{% macro redshift__escape_seq_for_json(chr) %}'\\\{{chr}}'{% endmacro %} \ No newline at end of file diff --git a/macros/db/redshift/quote_string.sql b/macros/db/redshift/quote_string.sql new file mode 100644 index 0000000..450f917 --- /dev/null +++ b/macros/db/redshift/quote_string.sql @@ -0,0 +1,2 @@ + +{%- macro redshift__quote_new_line() %}'\134\134n'{% endmacro %} \ No newline at end of file diff --git a/macros/db/snowflake/identifier_mapping.sql b/macros/db/snowflake/identifier_mapping.sql new file mode 100644 index 0000000..7a1c36f --- /dev/null +++ b/macros/db/snowflake/identifier_mapping.sql @@ -0,0 +1,8 @@ + +{% macro snowflake__name_in_db(name) %} + {% if name %} + {{ return (name.upper()) }} + {% else %} + {{ return (name) }} + {% endif %} +{% endmacro %} \ No newline at end of file diff --git a/macros/meta/get_monitored.sql b/macros/meta/get_monitored.sql new file mode 100644 index 0000000..a76cf1f --- /dev/null +++ b/macros/meta/get_monitored.sql @@ -0,0 +1,97 @@ +{% macro pub_monitored_from_graph() %} + {% set monitored = [] %} + {% set both = []%} + {% do both.extend(graph.nodes.values()) %} + {% do both.extend(graph.sources.values()) %} + {% set owners_config = f2dq.get_owners_config() %} + + {% set select_var = f2dq.f2dq_var('select', none) %} + {% set select_all = true %} + + {% set selected_nodes = none %} + {% set selected_tags = none %} + + {% if select_var is not none %} + {% set select_all = false %} + {% set selected_nodes = dict() %} + {% set selected_tags = dict() %} + + {% for el in select_var %} + {% if el.startswith('tag:') %} + {% do selected_tags.update({el[4:]: True}) %} + {% else %} + {% do selected_nodes.update({el: True}) %} + {% endif %} + {% endfor %} + {% endif %} + + {% for el in both %} + {% if el.resource_type in ['model', 'seed', 'source'] %} + {% if f2dq.f2dq_node_config(el, 'monitored') %} + {% set target_name = el.identifier or el.alias or el.name %} + + {% if select_all %} + {% set selected = true %} + {% else %} + {% set selected_name = selected_nodes.get(target_name, false) %} + {% set selected_tag = [] %} + + {% for tag in el.tags %} + {% if selected_tags.get(tag, false) %} + {% do selected_tag.append(true) %} + {% endif %} + {% endfor %} + + {% set selected = selected_name or (selected_tag | length > 0) %} + {% endif %} + + {% set metrics_groups = f2dq.f2dq_node_config(el, 'metrics_groups', f2dq.f2dq_var('default_metrics')) %} + {% set additional_metrics = f2dq.f2dq_node_config(el, 'metrics', {}) %} + + {% do monitored.append({ + 'name': f2dq.name_in_db(target_name), + 'schema': f2dq.name_in_db(el.schema), + 'database': f2dq.name_in_db(el.database), + 'time_filter': f2dq.f2dq_node_config(el, 'time_filter', none), + 'metrics_groups': metrics_groups, + 'additional_metrics': f2dq.metrics_in_db(additional_metrics), + 'metrics': f2dq.metrics_in_db(f2dq.final_metrics(metrics_groups, additional_metrics)), + 'columns': f2dq.columns_in_db(f2dq.f2dq_node_config(el, 'columns', none)), + 'anomaly_detector': f2dq.f2dq_node_config(el, 'anomaly_detector', f2dq.f2dq_var('anomaly_detector', {})), + 'owners': f2dq.prepare_model_owners(f2dq.f2dq_node_config(el, 'owners', []), owners_config), + 'selected': selected + }) + %} + {% endif %} + {% endif %} + {% endfor %} + + {{ return(monitored) }} +{% endmacro %} + +{% macro get_owners_config() %} + {% set owners_config = f2dq.f2dq_var('owners_config', {}) %} + {{ return (owners_config) }} +{% endmacro %} + +{% macro prepare_model_owners(f2dq_owners, owners_config) %} + {% set owners = {} %} + {% set seen_identifiers = {} %} + {% for owner in f2dq_owners if owners_config.get(owner) %} + {% set members = owners_config.get(owner) %} + {% for member in members %} + {% set identifier = member.get('identifier') %} + {% if identifier not in seen_identifiers %} + {% do seen_identifiers.update({identifier: true }) %} + {% do owners.update({ + identifier: { + 'notify_channel': member.get('type'), + 'owner': owner, + 'name': member.get('name') + } + }) %} + {% endif %} + {% endfor %} + {% endfor %} + {{ return (owners) }} +{% endmacro %} diff --git a/macros/meta/information_schema.sql b/macros/meta/information_schema.sql new file mode 100644 index 0000000..c9ed690 --- /dev/null +++ b/macros/meta/information_schema.sql @@ -0,0 +1,32 @@ +{% macro get_monitored_columns(schema, database) %} + {{ adapter.dispatch('get_monitored_columns', 'f2dq')(schema, database) }} +{% endmacro %} + +{% macro default__get_monitored_columns(table_schema, db_name) %} + {% set relation = api.Relation.create(database=db_name, schema=table_schema) %} + select + table_name, + table_schema, + table_catalog, + column_name, + data_type, + is_nullable + from + {{ relation.information_schema('COLUMNS') }} + where + table_schema = '{{ table_schema }}' +{% endmacro %} + +{% macro redshift__get_monitored_columns(table_schema, db_name) %} + select + table_name, + table_schema, + table_catalog, + column_name, + data_type, + is_nullable + from + svv_columns + where + table_schema = '{{ table_schema }}' +{% endmacro %} diff --git a/macros/meta/monitored_model_queries.sql b/macros/meta/monitored_model_queries.sql new file mode 100644 index 0000000..f701415 --- /dev/null +++ b/macros/meta/monitored_model_queries.sql @@ -0,0 +1,10 @@ +{% macro get_tables() %} + select * + from {{ ref('f2dq_selected') }} + order by name, schema, database, time_filter +{% endmacro %} + +{% macro get_schemas() %} + select distinct schema, database + from {{ ref('f2dq_selected') }} +{% endmacro %} \ No newline at end of file diff --git a/macros/meta/save_monitored.sql b/macros/meta/save_monitored.sql new file mode 100644 index 0000000..9760433 --- /dev/null +++ b/macros/meta/save_monitored.sql @@ -0,0 +1,17 @@ +{% macro save_monitored(monitored_path) %} + + {% set monitored_query %} + select + {{ full_table_name('name', 'schema', 'database') }} as {{ f2dq.quote_column('model') }}, + time_filter as {{ f2dq.quote_column('time_filter') }}, + metrics as {{ f2dq.quote_column('metrics') }}, + columns as {{ f2dq.quote_column('columns') }}, + anomaly_detector as {{ f2dq.quote_column('anomaly_detector') }}, + owners as {{ f2dq.quote_column('owners') }} + from {{ ref('f2dq_selected') }} + {% endset %} + {% set query_result = run_query(monitored_query) %} + {% set monitored_file_path = monitored_path or 'target/f2dq/monitored.json' %} + {% do query_result.to_json(monitored_file_path) %} + +{% endmacro %} \ No newline at end of file diff --git a/macros/meta/table_name.sql b/macros/meta/table_name.sql new file mode 100644 index 0000000..a468cec --- /dev/null +++ b/macros/meta/table_name.sql @@ -0,0 +1,33 @@ +{% macro full_table_name(table_name, table_schema, table_catalog) %} + {{ adapter.dispatch('full_table_name', 'f2dq')(table_name, table_schema, table_catalog) }} +{% endmacro %} + + +{% macro default__full_table_name(table_name, table_schema, table_catalog) %} + '"' || {{table_catalog}} || '"' || '.' || '"' || {{table_schema}} || '"' || '.' || '"' || {{table_name}} || '"' +{% endmacro %} + + +{% macro bigquery__full_table_name(table_name, table_schema, table_catalog) %} + '`' || {{table_catalog}} || '`' || '.' || '`' || {{table_schema}} || '`' || '.' || '`' || {{table_name}} || '`' +{% endmacro %} + + +{% macro full_table_name_values(table_name, table_schema, table_catalog) %} + {% set result = adapter.dispatch('full_table_name_values', 'f2dq')(table_name, table_schema, table_catalog) %} + {{ return (result.strip()) }} +{% endmacro %} + +{% macro default__full_table_name_values(table_name, table_schema, table_catalog) %} + "{{table_catalog}}"."{{table_schema}}"."{{table_name}}" +{% endmacro %} + + +{% macro bigquery__full_table_name_values(table_name, table_schema, table_catalog) %} + `{{table_catalog}}`.`{{table_schema}}`.`{{table_name}}` +{% endmacro %} + + +{% macro snowflake__full_table_name_values(table_name, table_schema, table_catalog) %} + "{{table_catalog|upper}}"."{{table_schema|upper}}"."{{table_name|upper}}" +{% endmacro %} \ No newline at end of file diff --git a/macros/metrics/base/build_in/column_default.sql b/macros/metrics/base/build_in/column_default.sql new file mode 100644 index 0000000..e4352b2 --- /dev/null +++ b/macros/metrics/base/build_in/column_default.sql @@ -0,0 +1,91 @@ + +{% macro f2dq_metric_max(context) %} + max({{context.column_name}}) +{% endmacro %} + +{% macro f2dq_metric_min(context) %} + min({{context.column_name}}) +{% endmacro %} + +{% macro f2dq_metric_avg(context) %} + avg(cast ({{context.column_name}} as {{ numeric_type() }})) +{% endmacro %} + +{% macro f2dq_metric_stddev(context) %} + stddev(cast ( {{context.column_name}} as {{ numeric_type() }})) +{% endmacro %} + +{% macro f2dq_metric_variance(context) %} + variance(cast ( {{context.column_name}} as {{ numeric_type() }})) +{% endmacro %} + +{% macro f2dq_metric_max_length(context) %} + max(length({{context.column_name}})) +{% endmacro %} + +{% macro f2dq_metric_min_length(context) %} + min(length({{context.column_name}})) +{% endmacro %} + +{% macro f2dq_metric_avg_length(context) %} + avg(cast (length( {{context.column_name}} ) as {{ numeric_type() }})) +{% endmacro %} + +{% macro f2dq_metric_nulls_count(context) %} + coalesce( + sum( + case when {{context.column_name}} is null + then 1 + else 0 + end + ), 0 + ) +{% endmacro %} + +{% macro f2dq_metric_missing_count(context) %} + coalesce( + sum( + case + when {{context.column_name}} is null + then 1 + when {{context.column_name}} = '' + then 1 + else 0 + end + ), 0 + ) +{% endmacro %} + +{% macro f2dq_metric_nulls_percent(context) %} + {{ percentage_formula(f2dq_metric_nulls_count(context), f2dq_metric_row_count()) }} +{% endmacro %} + +{% macro f2dq_metric_missing_percent(context) %} + {{ percentage_formula(f2dq_metric_missing_count(context), f2dq_metric_row_count()) }} +{% endmacro %} + +{% macro f2dq_metric_count_true(context) %} + COALESCE( + SUM( + CASE + WHEN {{ context.column_name }} IS TRUE THEN 1 + ELSE 0 + END + ), + 0 + ) +{% endmacro %} + +{% macro f2dq_metric_count_false(context) %} + COALESCE( + SUM( + CASE + WHEN {{ context.column_name }} IS FALSE THEN 1 + ELSE 0 + END + ), + 0 + ) +{% endmacro %} + + diff --git a/macros/metrics/base/build_in/optional_column_metrics.sql b/macros/metrics/base/build_in/optional_column_metrics.sql new file mode 100644 index 0000000..1d3f7e4 --- /dev/null +++ b/macros/metrics/base/build_in/optional_column_metrics.sql @@ -0,0 +1,104 @@ +{% macro f2dq_metric_regex_count(column_name, pattern) %} + coalesce( + sum( + case when {{ regex_match_expression(column_name, pattern) }} + then 1 + else 0 + end + ), 0 + ) +{% endmacro %} + +{% macro f2dq_metric_match_regex(context) %} + {{ f2dq_metric_regex_count(context.column_name, context.config.regex) }} +{% endmacro %} + +{% macro f2dq_metric_match_regex_percent(context) %} + {{ percentage_formula(f2dq_metric_match_regex(context), f2dq_metric_row_count()) }} +{% endmacro %} + +{% macro f2dq_metric_not_match_regex(context) %} + {{ f2dq_metric_row_count() }} - {{ f2dq_metric_regex_count(context.column_name, context.config.regex) }} +{% endmacro %} + +{% macro f2dq_metric_not_match_regex_percent(context) %} + {{ percentage_formula(f2dq_metric_not_match_regex(context), f2dq_metric_row_count()) }} +{% endmacro %} + +{% macro f2dq_metric_distinct_values(context) %} + {{ distinct_values(context) }} +{% endmacro %} + +{% macro distinct_values(context) %} + {{ adapter.dispatch('distinct_values', 'f2dq')(context) }} +{% endmacro %} + +{% macro default__distinct_values(context) %} + coalesce( + count(distinct {{ context.column_name }} ) + , 0) +{% endmacro %} + +{% macro postgres__distinct_values(context) %} + {# /* In postgres, its faster to count distinct values in a column by selecting then counting in separate steps */ #} + with temp_table as ( + select distinct {{ context.column_name }} from {{ context.table_name }} + where {{ in_time_window(context.time_filter) }} + ) + select coalesce(count(*), 0) from temp_table +{% endmacro %} + +{% macro f2dq_metric_approx_distinct_values(context) %} + {{ approx_distinct_values(context) }} +{% endmacro %} + +{% macro approx_distinct_values(context) %} + {{ adapter.dispatch('approx_distinct_values', 'f2dq')(context) }} +{% endmacro %} + +{% macro default__approx_distinct_values(context) %} + {# /* No approximate distinct count in postgres so we default to using a distinct count */ #} + {{ f2dq_metric_distinct_values(context) }} +{% endmacro %} + +{% macro redshift__approx_distinct_values(context) %} + approximate {{ f2dq_metric_distinct_values(context.column_name) }} +{% endmacro %} + +{% macro bigquery__approx_distinct_values(context) %} + approx_count_distinct({{ context.column_name }}) +{% endmacro %} + +{% macro snowflake__approx_distinct_values(context) %} + approx_count_distinct({{ context.column_name }}) +{% endmacro %} + +{% macro f2dq_metric_duplicate_values(context) %} + with temp_table as ( + select {{ context.column_name }} from {{ context.table_name }} + where {{ in_time_window(context.time_filter) }} + group by {{ context.column_name }} + having count(1) > 1 + ) + select coalesce(count(*), 0) from temp_table +{% endmacro %} + +{% macro f2dq_metric_duplicate_rows(context) %} + with temp_table as ( + select {{ context.column_name }}, count(1) as row_count from {{ context.table_name }} + where {{ in_time_window(context.time_filter) }} + group by {{ context.column_name }} + having count(1) > 1 + ) + select coalesce(sum(row_count), 0) from temp_table +{% endmacro %} + +{% macro f2dq_metric_unique_rows(context) %} + with temp_table as ( + select {{ context.column_name }}, count(1) as row_count from {{ context.table_name }} + where {{ in_time_window(context.time_filter) }} + group by {{ context.column_name }} + having count(1) = 1 + ) + select coalesce(sum(row_count), 0) from temp_table +{% endmacro %} \ No newline at end of file diff --git a/macros/metrics/base/build_in/optional_table_metrics.sql b/macros/metrics/base/build_in/optional_table_metrics.sql new file mode 100644 index 0000000..51df797 --- /dev/null +++ b/macros/metrics/base/build_in/optional_table_metrics.sql @@ -0,0 +1,7 @@ +{% macro f2dq_metric_distinct_table_rows(context) %} + with temp_table AS ( + select distinct * from {{ context.table_name }} + where {{ in_time_window(context.time_filter) }} + ) + select coalesce(count(*), 0) FROM temp_table +{% endmacro %} diff --git a/macros/metrics/base/build_in/table_default.sql b/macros/metrics/base/build_in/table_default.sql new file mode 100644 index 0000000..17bdef7 --- /dev/null +++ b/macros/metrics/base/build_in/table_default.sql @@ -0,0 +1,33 @@ + +{% macro f2dq_metric_row_count(context) %} + count(1) +{% endmacro %} + +{% macro f2dq_metric_freshness(context) %} + {{ freshness_expression(context.time_filter) }} +{% endmacro %} + +{% macro freshness_expression(time_filter) %} + {# /* If time_filter is none, we are computing the metric globally. we set the value as null since a table without use of a time filter doesn't really have a freshness metric */ #} + {% if time_filter is none %} + cast(null as {{ numeric_type() }}) + {% else %} + {{ adapter.dispatch('freshness_expression', 'f2dq')(time_filter) }} + {% endif %} +{% endmacro %} + +{% macro default__freshness_expression(time_filter) %} + EXTRACT(EPOCH FROM ({{time_window_end()}} - max({{time_filter}}))) +{% endmacro %} + +{% macro bigquery__freshness_expression(time_filter) %} + TIMESTAMP_DIFF ( timestamp({{ time_window_end()}}), timestamp(max({{time_filter}})), SECOND) +{% endmacro %} + +{% macro snowflake__freshness_expression(time_filter) %} + timediff(second, max({{time_filter}}), {{- time_window_end() -}}) +{% endmacro %} + +{% macro redshift__freshness_expression(time_filter) %} + DATEDIFF(second, max({{time_filter}}), {{- time_window_end() -}}) +{% endmacro %} diff --git a/macros/metrics/base/expression.sql b/macros/metrics/base/expression.sql new file mode 100644 index 0000000..bf876d5 --- /dev/null +++ b/macros/metrics/base/expression.sql @@ -0,0 +1,102 @@ +{% macro metrics_base_expressions(model, columns, table_level=False) %} + + {% set col_expr = [] %} + + {% for col in columns %} + {% set column_name = f2dq.row_value(col, 'column_name') %} + {% do col_expr.extend(f2dq.metrics_base_expression_column_all(model, col)) %} + {% endfor %} + + {% if table_level %} + {% do col_expr.extend(f2dq.metrics_base_expresion_table_all(model)) %} + {% endif %} + + {{ return (col_expr) }} + +{% endmacro %} + +{% macro metrics_base_expression_column_all(model, column) %} + + {%- set col_expr = [] %} + {%- set metrics_to_compute = [] %} + {% set column_name = f2dq.row_value(column, 'column_name') %} + {% set data_type = model.columns_info[column_name].data_type %} + {% do metrics_to_compute.extend(model.metrics.get('group').get('column', {}).get(data_type, [])) %} + {% do metrics_to_compute.extend(model.metrics.get('additional').get('column', {}).get(column_name, [])) %} + + {% for metric_value in metrics_to_compute %} + {% set metric_obj = f2dq.extract_metric_config(metric_value) %} + {% set expression = f2dq.metrics_base_expression_column(model, column_name, metric_obj['metric'], metric_obj['config']) %} + {% do col_expr.append({ 'expr': expression, 'col_name': column_name, 'metric': metric_obj['metric']}) %} + {% endfor %} + + {{ return (col_expr) }} + +{% endmacro %} + + +{% macro metrics_base_expresion_table_all(model) %} + {%- set table_expr = [] %} + {%- set metrics_to_compute = [] %} + {% do metrics_to_compute.extend(model.metrics.get('group').get('table', [])) %} + {% do metrics_to_compute.extend(model.metrics.get('additional').get('table', [])) %} + + {% for metric_value in metrics_to_compute %} + {% set metric_obj = f2dq.extract_metric_config(metric_value) %} + {% set expression = f2dq.metrics_base_expression_table(model, metric_obj['metric'], metric_obj['config']) %} + {% do table_expr.append({ 'expr': expression, 'col_name': '', 'metric': metric_obj['metric']}) %} + {% endfor %} + + {{ return (table_expr) }} + +{% endmacro %} + +{% macro metrics_base_expression_table(model, metric_name, config) %} + {% set metric_macro = f2dq.get_metric_macro(metric_name) %} + {% set context = {'time_filter': model.time_filter, 'metric_name': metric_name, 'config': config, 'table_name': model.table_name, 'column_name': none} %} + + {{ metric_macro(context) }} + +{% endmacro %} + + +{%- macro metrics_base_expression_column(model, column_name, metric_name, config) %} + {% set metric_macro = f2dq.get_metric_macro(metric_name) %} + {% set context = {'time_filter': model.time_filter, 'metric_name': metric_name, 'config': config, 'table_name': model.table_name, 'column_name': f2dq.quote_column_name(column_name)} %} + + {{ metric_macro(context) }} + +{% endmacro %} + +{% macro extract_metric_config(metric_value) %} + + {% set config = none %} + + {% if metric_value is mapping %} + {% set metric = metric_value.keys() | first %} + {% if metric_value[metric] is none %} + {{ exceptions.raise_compiler_error("Empty configuration passed for metric: " ~ metric ~ ". If the metric doesn't use a config, please use the column name as a string.") }} + {% endif %} + + {% set config = metric_value[metric] %} + {%- else %} + {% set metric = metric_value %} + {% endif %} + + {{ return ({'metric': metric, 'config': config}) }} + +{% endmacro %} + +{%- macro get_metric_macro(metric_name) %} + {% set macro_name = 'f2dq_metric' + '_' + metric_name %} + + {% if context['f2dq'].get(macro_name) %} + {% set metric_macro = context['f2dq'][macro_name] %} + {%- else %} + {% set metric_macro = context[project_name][macro_name] %} + {% endif %} + + {{ return (metric_macro) }} + +{% endmacro %} + diff --git a/macros/metrics/base/internal_model_template.sql b/macros/metrics/base/internal_model_template.sql new file mode 100644 index 0000000..c61f61f --- /dev/null +++ b/macros/metrics/base/internal_model_template.sql @@ -0,0 +1,31 @@ +{% macro f2dq_last_base_metrics_part() %} + +-- depends_on: {{ ref('f2dq_columns') }} + +{{ + config( + materialized='table', + ) +}} + +{{ f2dq.empty_last_base_metrics() }} + +{% endmacro %} + +{% macro f2dq_last_base_metrics_thread(num) %} + {% set part_name = 'f2dq_last_base_metrics_part' ~ num %} + {{ f2dq.generate_depends(['f2dq_selected', 'f2dq_monitored', 'f2dq_columns', 'f2dq_run_started_at', part_name]) }} + + {{ + config( + materialized='table', + ) + }} + + {% if not f2dq.in_compile() %} + {{ f2dq.metrics_base_compute_for_thread(num, part_name) }} + {% endif %} + + {{ f2dq.empty_last_base_metrics() }} + +{% endmacro %} \ No newline at end of file diff --git a/macros/metrics/base/queries.sql b/macros/metrics/base/queries.sql new file mode 100644 index 0000000..6bd1c8d --- /dev/null +++ b/macros/metrics/base/queries.sql @@ -0,0 +1,75 @@ +{% macro metrics_base_compute_for_thread(thread_value, ref_model) %} + {%- set tables = run_query(f2dq.get_tables()) %} + {%- for mtable in tables %} + -- we are splitting computing metrics to 4 different threads + {% set for_loop_mod = (loop.index % 4) %} + {% if for_loop_mod == thread_value %} + {% set model = get_model_config(mtable) %} + + {% set columns_to_query = [] %} + {% set size = 0 %} + + {% for column in model.columns %} + {% set column_name = f2dq.row_value(column, 'column_name') %} + + {% if should_compute_metric(model, column_name) %} + {% do columns_to_query.append(column) %} + {% endif %} + + {% set columns_size = columns_to_query| length %} + + {% if columns_size == var('f2dq:max_columns_in_query') %} + {%- set insert_stats_query = f2dq.metrics_base_insert(model, ref_model, columns_to_query) -%} + + {% if insert_stats_query %} + {% do run_query(insert_stats_query) %} + {% endif %} + {% do columns_to_query.clear() %} + {% endif %} + {% endfor %} + + {%- set insert_stats_query = f2dq.metrics_base_insert(model, ref_model, columns_to_query, table_level=True) -%} + {% do run_query(insert_stats_query) %} + + {{ dbt_utils.log_info('[f2dq_log] - finished computing metrics for:' ~ model.model_name) }} + {% endif %} + {% endfor %} +{% endmacro %} + +{% macro metrics_base_insert(model, ref_model, columns, table_level=False) %} + + {% set col_exprs = f2dq.metrics_base_expressions(model, columns, table_level) %} + {% if col_exprs == [] %} + {{ return ('') }} + {% endif %} + + insert into {{ ref(ref_model) }} + with temp_table_metrics as ( + select + {%- for col_expr in col_exprs %} + ( {{ col_expr.expr }} ) as {{ f2dq.quote_column_name(col_expr.col_name + '___' + col_expr.metric) }} + {%- if not loop.last %},{%- endif %} + {% endfor %} + from + {{ model.table_name }} + where + {{ in_time_window(model.time_filter) }} + ) + + {%- for col_expr in col_exprs %} + {% set final_metric_name = get_final_metric_name(col_expr.metric, model.time_filter) %} + + select '{{model.table_name}}' as table_name, '{{ col_expr.col_name }}' as column_name, '{{ final_metric_name }}' as metric, {{ f2dq.quote_column_name(col_expr.col_name + '___' + col_expr.metric) }} as value + from temp_table_metrics + {% if not loop.last %}union all{% endif %} + {% endfor %} + +{% endmacro %} + +{% macro get_final_metric_name(metric_name, time_filter) %} + {% if time_filter is none %} + {{ return ('global__' + metric_name) }} + {% else %} + {{ return (metric_name) }} + {% endif %} +{% endmacro %} \ No newline at end of file diff --git a/macros/post_hook/f2dq_monitored.sql b/macros/post_hook/f2dq_monitored.sql new file mode 100644 index 0000000..e3dbb3e --- /dev/null +++ b/macros/post_hook/f2dq_monitored.sql @@ -0,0 +1,11 @@ +{% macro pub_insert_into_f2dq_monitored() %} + {% set monitored = f2dq.pub_monitored_from_graph() %} + {% do f2dq.insert_list_to_table( + this, + monitored, + ['name', 'schema', 'database', 'time_filter', 'metrics_groups', 'additional_metrics', 'metrics', 'columns', 'anomaly_detector', 'owners', 'selected'] + ) %} + + {{ return('') }} + +{% endmacro %} \ No newline at end of file diff --git a/macros/public/cleaning/clean_additional_whitespace.sql b/macros/public/cleaning/clean_additional_whitespace.sql new file mode 100644 index 0000000..24f3ed8 --- /dev/null +++ b/macros/public/cleaning/clean_additional_whitespace.sql @@ -0,0 +1,23 @@ +{% macro clean_additional_whitespaces(column_name) %} + {{ adapter.dispatch('clean_additional_whitespaces', 'f2dq')(column_name) }} +{% endmacro %} + +{% macro default__clean_additional_whitespaces(column_name) %} + trim(regexp_replace( {{ column_name }}, '\s\s+', ' ')) +{% endmacro %} + +{% macro postgres__clean_additional_whitespaces(column_name) %} + trim(regexp_replace( {{ column_name }}, '\s\s+', ' ', 'g')) +{% endmacro %} + +{% macro redshift__clean_additional_whitespaces(column_name) %} + trim(regexp_replace( {{ column_name }}, '\\s\\s+', ' ')) +{% endmacro %} + +{% macro bigquery__clean_additional_whitespaces(column_name) %} + trim(regexp_replace( {{ column_name }}, r'\s\s+', ' ')) +{% endmacro %} + +{% macro snowflake__clean_additional_whitespaces(column_name) %} + trim(regexp_replace( {{ column_name }}, '\\s\\s+', ' ')) +{% endmacro %} \ No newline at end of file diff --git a/macros/public/cleaning/clean_blacklist.sql b/macros/public/cleaning/clean_blacklist.sql new file mode 100644 index 0000000..ce47fcf --- /dev/null +++ b/macros/public/cleaning/clean_blacklist.sql @@ -0,0 +1,31 @@ +{% macro generate_blacklist_pattern(chars_to_blacklist) %} + {% set pattern = [] %} + {% for char in chars_to_blacklist %} + {% set expr = '(' + char + ')' %} + {% do pattern.append(expr) %} + {% endfor %} + + {{ return(pattern | join('|')) }} +{% endmacro %} + +{%- macro clean_blacklist(column_name, chars_to_blacklist, replacement) -%} + {% set pattern_string = f2dq.generate_blacklist_pattern(chars_to_blacklist) %} + + {{ adapter.dispatch('clean_blacklist', 'f2dq')(column_name, pattern_string, replacement) }} +{%- endmacro -%} + +{%- macro default__clean_blacklist(column_name, pattern_string, replacement) -%} + regexp_replace( {{ column_name }}, '{{ pattern_string }}', '{{ replacement }}') +{%- endmacro -%} + +{%- macro postgres__clean_blacklist(column_name, pattern_string, replacement) -%} + regexp_replace( {{ column_name }}, '{{ pattern_string }}', '{{ replacement }}', 'g') +{%- endmacro -%} + +{%- macro redshift__clean_blacklist(column_name, pattern_string, replacement) -%} + regexp_replace( {{ column_name }}, '{{ pattern_string }}', '{{ replacement }}') +{%- endmacro -%} + +{%- macro bigquery__clean_blacklist(column_name, pattern_string, replacement) -%} + regexp_replace( {{ column_name }}, """{{ pattern_string }}""", '{{ replacement }}') +{%- endmacro -%} \ No newline at end of file diff --git a/macros/public/cleaning/clean_capitalize_words.sql b/macros/public/cleaning/clean_capitalize_words.sql new file mode 100644 index 0000000..93bda09 --- /dev/null +++ b/macros/public/cleaning/clean_capitalize_words.sql @@ -0,0 +1,3 @@ +{% macro clean_capitalize_words(column_name) %} + initcap( {{column_name}} ) +{% endmacro %} \ No newline at end of file diff --git a/macros/public/filtering/get_duplicates.sql b/macros/public/filtering/get_duplicates.sql new file mode 100644 index 0000000..55b6653 --- /dev/null +++ b/macros/public/filtering/get_duplicates.sql @@ -0,0 +1,23 @@ +{# https://github.com/fast2dev/f2dq/issues/143 #} + +{# +macro returns rows with she same key set (unique_cols) + +along with the fields of the base model duplicates information added: +f2dq_duplicate_count - total number of duplicates with the same current key set +f2dq_duplicate_row_number - number of a duplicate row inside the group of duplicates with the same current key set +#} + +{% macro filter_get_duplicates(relation, unique_cols, sort_columns) %} + ( + with duplication_context as ( + {{f2dq.add_duplication_context(relation, unique_cols, sort_columns)}} + ), + duplicate_rows as ( + select * from duplication_context where f2dq_duplicate_group_row_count > 1 + ) + {# return surrogate key as well? #} + select * + from duplicate_rows + ) +{% endmacro %} \ No newline at end of file diff --git a/macros/public/filtering/remove_duplicates.sql b/macros/public/filtering/remove_duplicates.sql new file mode 100644 index 0000000..22a71e3 --- /dev/null +++ b/macros/public/filtering/remove_duplicates.sql @@ -0,0 +1,15 @@ + + + +{% macro filter_remove_duplicates(relation, unique_cols, sort_columns) %} + ( + with with_row_num as ( + {{f2dq.add_duplication_context(relation, unique_cols, sort_columns)}} + ), + one_row_num as ( + select * from with_row_num where f2dq_duplicate_group_row_number = 1 + ) + select {{ dbt_utils.star(from=relation) }} + from one_row_num + ) +{% endmacro %} \ No newline at end of file diff --git a/macros/public/normalizing/normalize_values.sql b/macros/public/normalizing/normalize_values.sql new file mode 100644 index 0000000..9186742 --- /dev/null +++ b/macros/public/normalizing/normalize_values.sql @@ -0,0 +1,32 @@ +{% macro is_dbt_relation(obj) %} + {{ return (obj is mapping and obj.get('metadata', {}).get('type', '').endswith('Relation') )}} +{% endmacro %} + +{% macro normalize_expression_cte(reference_table) %} + with target_table as ( + {% if f2dq.is_dbt_relation(reference_table) or reference_table is string %} + select * from {{ reference_table }} + {% elif reference_table is mapping %} + {% for key, value in reference_table.items() %} + select '{{key}}' as source, '{{value}}' as target + {% if not loop.last %}union all{% endif %} + {% endfor %} + {% endif %} + ) +{% endmacro %} + +{%- macro normalize_values(source_relation, column_name, reference_table) -%} + ( + {{ f2dq.normalize_expression_cte(reference_table) }} + + select s.*, + case when t.source is null + then s.{{column_name}} + else t.target + end as {{ column_name + '__normalized'}} + from {{ source_relation }} s + left join target_table t + on t.source = s.{{column_name}} + ) +{%- endmacro -%} + diff --git a/macros/public/store/export_alerts.sql b/macros/public/store/export_alerts.sql new file mode 100644 index 0000000..0922231 --- /dev/null +++ b/macros/public/store/export_alerts.sql @@ -0,0 +1,22 @@ +{% macro export_alerts(start_date, end_date, alerts_path=None, monitored_path=None) %} + {% set alerts_query %} + select + type as {{ f2dq.quote_column('type') }}, + model as {{ f2dq.quote_column('model') }}, + message as {{ f2dq.quote_column('message') }}, + value as {{ f2dq.quote_column('value') }}, + {{ format_timestamp('time_window_end')}} as {{ f2dq.quote_column('time_window_end') }} + from {{ ref('f2dq_alerts') }} + where + case + when type = 'anomaly' then {{ in_date_window('time_window_end', start_date, end_date) }} + else {{ in_date_window('time_window_end', start_date, none) }} + end + order by time_window_end desc + {% endset %} + + {% set query_result = run_query(alerts_query) %} + {% set alerts_file_path = alerts_path or 'target/f2dq/alerts.json' %} + {% do query_result.to_json(alerts_file_path) %} + {{ save_monitored(monitored_path) }} +{% endmacro %} diff --git a/macros/public/store/export_table_samples.sql b/macros/public/store/export_table_samples.sql new file mode 100644 index 0000000..e093c38 --- /dev/null +++ b/macros/public/store/export_table_samples.sql @@ -0,0 +1,15 @@ +{% macro export_table_samples(start_date, end_date, table_samples_path=None) %} + {% set table_samples_query %} + select + lower(table_name) as table_name, + sample_data, + sampled_on + from + {{ ref('f2dq_table_samples') }} + {% endset %} + + {% set query_result = run_query(table_samples_query) %} + {% set table_samples_file_path = table_samples_path or 'target/f2dq/table_samples.json' %} + {% do query_result.to_json(table_samples_file_path) %} + +{% endmacro %} \ No newline at end of file diff --git a/macros/public/store/export_tests_history.sql b/macros/public/store/export_tests_history.sql new file mode 100644 index 0000000..e21a15a --- /dev/null +++ b/macros/public/store/export_tests_history.sql @@ -0,0 +1,25 @@ +{% macro export_tests_history(start_date, end_date, tests_history_path=None) %} + {% set tests_history_query %} + select + table_name, + column_name, + test_name, + run_at, + status, + execution_time, + message, + failures_count, + failures_json, + failures_table, + severity, + compiled_sql + from + {{ ref('f2dq_test_history') }} + where {{ in_date_window('run_at', start_date, end_date) }} + {% endset %} + + {% set query_result = run_query(tests_history_query) %} + {% set tests_history_file_path = tests_history_path or 'target/f2dq/tests_history.json' %} + {% do query_result.to_json(tests_history_file_path) %} + +{% endmacro %} diff --git a/macros/public/store/generate_overview.sql b/macros/public/store/generate_overview.sql new file mode 100644 index 0000000..10eb6c9 --- /dev/null +++ b/macros/public/store/generate_overview.sql @@ -0,0 +1,93 @@ + +{% macro overview_select_base(type, timestamp_col) %} + '{{ type }}' as {{ f2dq.quote_column('type') }}, + table_name as {{ f2dq.quote_column('table_name') }}, + column_name as {{ f2dq.quote_column('column_name') }}, + {{ timestamp_col }} as {{ f2dq.quote_column('computed_on') }}, +{% endmacro %} + +{% macro generate_overview(start_date, end_date, interval, overview_path=None, monitored_path=None) %} +-- depends_on: {{ ref('f2dq_anomalies') }} +-- depends_on: {{ ref('f2dq_base_metrics') }} +-- depends_on: {{ ref('f2dq_schema_changes') }} +-- depends_on: {{ ref('f2dq_columns') }} + + {# time grain is either days or hour #} + {% set time_grain, num_str = interval.split(':') %} + {% set num = num_str | int %} + {% if time_grain == 'hours' %} + {% set interval_length_sec = num * 3600 %} + {% elif time_grain == 'days'%} + {% set interval_length_sec = num * 3600 * 24 %} + {% else %} + {{ exceptions.raise_compiler_error("Invalid interval. Got: " ~ interval) }} + {% endif %} + {{ dbt_utils.log_info('[f2dq] interval length in seconds is ' ~ interval_length_sec) }} + {% set overview_query %} + with schema_changes_casted as ( + select id, table_name, operation, column_name, data_type, {{ bool_to_string('is_nullable') }}, prev_column_name, prev_data_type, {{ bool_to_string('prev_is_nullable') }}, detected_time + from {{ ref('f2dq_schema_changes') }} + ), + columns_casted as ( + select {{ full_table_name('name', 'schema', 'database') }} as table_name, column_name, data_type, {{ bool_to_string('is_nullable') }}, computed_on + from {{ ref('f2dq_columns') }} + ) + + ( + select + {{ overview_select_base('metric', 'computed_on')}} + {{ to_single_json(['metric', 'value', 'time_window_end', 'interval_length_sec']) }} as {{ f2dq.quote_column('data') }} + from + {{ ref('f2dq_base_metrics') }} + where {{ in_date_window('time_window_end', start_date, end_date) }} + and interval_length_sec = {{interval_length_sec}} + ) union all + ( + select + {{ overview_select_base('anomaly', 'computed_on')}} + {{ to_single_json(['id', 'metric', 'z_score_value', 'last_value', 'last_avg', 'last_stddev', 'time_window_end', 'interval_length_sec']) }} as {{ f2dq.quote_column('data') }} + from + {{ ref('f2dq_anomalies') }} + where {{ in_date_window('time_window_end', start_date, end_date) }} + and interval_length_sec = {{interval_length_sec}} + ) union all + ( + select + {{ overview_select_base('schema_change', 'detected_time')}} + {{ to_single_json(['id', 'operation', 'data_type', 'is_nullable', 'prev_column_name', 'prev_data_type', 'prev_is_nullable', 'detected_time']) }} as {{ f2dq.quote_column('data') }} + from + schema_changes_casted + where {{ in_date_window('detected_time', start_date, none) }} + ) union all + ( + select + {{ overview_select_base('schema', 'computed_on')}} + {{ to_single_json(['data_type', 'is_nullable']) }} as {{ f2dq.quote_column('data') }} + from + columns_casted + ) + union all + ( + select + 'alert' as {{ f2dq.quote_column('type') }}, + model as {{ f2dq.quote_column('table_name') }}, + null as {{ f2dq.quote_column('column_name') }}, + time_window_end as {{ f2dq.quote_column('computed_on') }}, + {{ to_single_json(['type', 'model', 'message', 'value', 'time_window_end']) }} as {{ f2dq.quote_column('data') }} + from + {{ ref('f2dq_alerts') }} + where + case + when type = 'anomaly' then {{ in_date_window('time_window_end', start_date, end_date) }} + else {{ in_date_window('time_window_end', start_date, none) }} + end + ) + order by {{ f2dq.quote_column('computed_on')}} desc + {% endset %} + + {% set overview_result = run_query(overview_query) %} + {% set overview_file_path = overview_path or '../target/f2dq/overview.json' %} + {% do overview_result.to_json(overview_file_path) %} + {{ save_monitored(monitored_path) }} + +{% endmacro %} diff --git a/macros/public/validating/regex_dict.sql b/macros/public/validating/regex_dict.sql new file mode 100644 index 0000000..29fc42a --- /dev/null +++ b/macros/public/validating/regex_dict.sql @@ -0,0 +1,65 @@ +{# +# This file contains significant part of code licensed under: +# Copyright 2020 Soda +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +{% macro get_regex_for(to_validate) %} + + {% set regexp_dict = { + 'number_whole': '^\-?[0-9]+$', + 'number_decimal_point': '^\-?[0-9]+\.[0-9]+$', + 'number_decimal_comma': '^\-?[0-9]+,[0-9]+$', + 'number_percentage': '^\-?[0-9]+([\.,][0-9]+)? ?%$', + 'number_percentage_point': '^\-?[0-9]+([\.][0-9]+)? ?%$', + 'number_percentage_comma': '^\-?[0-9]+([,][0-9]+)? ?%$', + 'date_eu': '^([1-9]|0[1-9]|[12][0-9]|3[01])[-\./]([1-9]|0[1-9]|1[012])[-\./](19|20)?[0-9][0-9]$', + 'date_us': '^([1-9]|0[1-9]|1[012])[-\./]([1-9]|0[1-9]|[12][0-9]|3[01])[-\./](19|20)?[0-9][0-9]$', + 'date_inverse': '^(19|20)[0-9][0-9][-\./]?([1-9]|0[1-9]|1[012])[-\./]?([1-9]|0[1-9]|[12][0-9]|3[01])$', + 'time_24h': '^([01][0-9]|2[0-3]):([0-5][0-9])$', + 'time_12h': '^(1[0-2]|0?[1-9]):[0-5][0-9]$', + 'time': '^([0-9]|1[0-9]|2[0-4])[:-]([0-9]|[0-5][0-9])([:-]([0-9]|[0-5][0-9])(,[0-9]+)?)?$', + 'date_iso_8601': + '^' + '([1-9][0-9]{3}-((0[1-9]|1[0-2])-(0[1-9]|1[0-9]|2[0-8])|(0[13-9]|1[0-2])-(29|30)|(0[13578]|1[02])-31)|' + '([1-9][0-9](0[48]|[2468][048]|[13579][26])|([2468][048]|[13579][26])00)-02-29)' + + 'T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\.[0-9]+)?' + + '(Z|[+-][01][0-9]:[0-5][0-9])?' + '$', + 'uuid': '^[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}$', + 'ipv4_address': '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$', + 'ipv6_address': '^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])){3}))|:)))(%.+)?$', + 'email': '^[A-Za-z0-9.-_%]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,4}$', + } %} + + {% set base_regex = regexp_dict[to_validate] %} + {% set qualifed_regex = adapter.dispatch('get_regex_for', 'f2dq')(base_regex) %} + {{ return(qualifed_regex) }} + +{% endmacro %} + +{% macro default__get_regex_for(pattern) %} + {{ return (pattern) }} +{% endmacro %} + +{% macro redshift__get_regex_for(pattern) %} + {% set changed = modules.re.sub('\.', '\\.', pattern) %} + {% set changed = modules.re.sub('\-', '\\-', changed) %} + {{ return (changed) }} +{% endmacro %} + +{% macro snowflake__get_regex_for(pattern) %} + {% set changed = modules.re.sub('\.', '\\.', pattern) %} + {% set changed = modules.re.sub('\-', '\\-', changed) %} + {{ return (changed) }} +{% endmacro %} \ No newline at end of file diff --git a/macros/public/validating/valid_with_regex.sql b/macros/public/validating/valid_with_regex.sql new file mode 100644 index 0000000..acd3148 --- /dev/null +++ b/macros/public/validating/valid_with_regex.sql @@ -0,0 +1,93 @@ + + +{% macro valid_regex(column_name, to_validate) %} +{% set pattern = f2dq.get_regex_for(to_validate) %} + case when + {{ column_name }} is null then false + else {{ f2dq.regex_match_expression(column_name, pattern) }} + end +{% endmacro %} + +{% macro valid_email(column_name) %} + {{ f2dq.valid_regex(column_name, 'email')}} +{% endmacro %} + +{% macro valid_date_eu(column_name) %} + {{ f2dq.valid_regex(column_name, 'date_eu')}} +{% endmacro %} + +{% macro valid_date_us(column_name) %} + {{ f2dq.valid_regex(column_name, 'date_us')}} +{% endmacro %} + +{% macro valid_date_inverse(column_name) %} + {{ f2dq.valid_regex(column_name, 'date_inverse')}} +{% endmacro %} + +{% macro valid_date_iso_8601(column_name) %} + {{ f2dq.valid_regex(column_name, 'date_iso_8601')}} +{% endmacro %} + +{% macro valid_time_24h(column_name) %} + {{ f2dq.valid_regex(column_name, 'time_24h')}} +{% endmacro %} + +{% macro valid_time_12h(column_name) %} + {{ f2dq.valid_regex(column_name, 'time_12h')}} +{% endmacro %} + +{% macro valid_time(column_name) %} + {{ f2dq.valid_regex(column_name, 'time')}} +{% endmacro %} + +{% macro valid_ip_v4(column_name) %} + {{ f2dq.valid_regex(column_name, 'ipv4_address')}} +{% endmacro %} + +{% macro valid_ip_v6(column_name) %} + {{ f2dq.valid_regex(column_name, 'ipv6_address')}} +{% endmacro %} + +{% macro valid_ip(column_name) %} + ( + {{ f2dq.valid_regex(column_name, 'ipv4_address')}} + or + {{ f2dq.valid_regex(column_name, 'ipv6_address')}} + ) +{% endmacro %} + +{% macro valid_number(column_name) %} + {{ f2dq.valid_regex(column_name, 'number_whole')}} +{% endmacro %} + +{% macro valid_number_decimal_point(column_name) %} + {{ f2dq.valid_regex(column_name, 'number_decimal_point')}} +{% endmacro %} + +{% macro valid_number_decimal_comma(column_name) %} + {{ f2dq.valid_regex(column_name, 'number_decimal_comma')}} +{% endmacro %} + +{% macro valid_number_percentage(column_name) %} + {{ f2dq.valid_regex(column_name, 'number_percentage')}} +{% endmacro %} + +{% macro valid_number_percentage_point(column_name) %} + {{ f2dq.valid_regex(column_name, 'number_percentage_point')}} +{% endmacro %} + +{% macro valid_number_percentage_comma(column_name) %} + {{ f2dq.valid_regex(column_name, 'number_percentage_comma')}} +{% endmacro %} + +{% macro valid_phone(column_name) %} + {{ f2dq.valid_regex(column_name, 'phone')}} +{% endmacro %} + +{% macro valid_uuid(column_name) %} + {{ f2dq.valid_regex(column_name, 'uuid')}} +{% endmacro %} + +{% macro valid_credit_card(column_name) %} + {{ f2dq.valid_regex(column_name, 'credit_card_number')}} +{% endmacro %} \ No newline at end of file diff --git a/macros/run_end/save_results_history.sql b/macros/run_end/save_results_history.sql new file mode 100644 index 0000000..d0de079 --- /dev/null +++ b/macros/run_end/save_results_history.sql @@ -0,0 +1,112 @@ + +{% macro save_test_history(results) -%} + +{{ adapter.dispatch('save_test_history_impl', 'f2dq') (results) }} + +{%- endmacro %} + +{% macro default__save_test_history_impl(results) %} + -- depends_on: {{ ref('f2dq_test_history') }} + {% set command = flags.WHICH %} + {% if execute and results and command in ('test', 'build') %} + {% set tests = [] %} + {% for el in results %} + {% if el.node.resource_type.value == 'test' %} + {% do tests.append(f2dq.test_data_dict(el)) %} + {% endif %} + {% endfor %} + + {% if tests %} + {% do f2dq.insert_list_to_table( + ref('f2dq_test_history'), + tests, + ['table_name', 'column_name', 'test_name', 'status', 'execution_time', 'message', 'failures_count', 'failures_json', 'failures_table', 'severity', 'compiled_sql', 'run_at'], + { 'run_at': timestamp_type() } + ) %} + {% endif %} + + {% endif %} + {{ return ('') }} + +{% endmacro %} + +{% macro test_data_dict(el) %} + + {% set run_started_at_str = run_started_at.strftime('%Y-%m-%d %H:%M:%S') %} + + {% if el.node.to_dict().get('test_metadata') %} + {% set any_refs = modules.re.findall("ref\(\'(?P.*)\'\)", el.node.test_metadata.kwargs['model']) %} + {% set any_source = modules.re.findall("source\(\'(?P.*)\'\,\s+\'(?P.*)\'\)", el.node.test_metadata.kwargs['model']) %} + + {% if any_refs %} + {% set name = any_refs[0] %} + {% set node_name = f2dq.priv_full_name_from_depends(el.node, name) %} + {% set schema = graph.nodes.get(node_name)['schema'] %} + {% set database = graph.nodes.get(node_name)['database'] %} + {% set table_name = (database + '.' + schema + '.' + name) | lower %} + + {% elif any_source %} + {% set package_name = any_source[0][0] %} + {% set name = any_source[0][1] %} + {% set node_name = f2dq.priv_full_name_from_depends(el.node, name) %} + {% set schema = graph.sources.get(node_name)['schema'] %} + {% set database = graph.sources.get(node_name)['database'] %} + {% set table_name = (database + '.' + schema + '.' + name) | lower %} + {% else %} + {% set table_name = none %} + {% endif %} + {% else %} + {% set table_name = none %} + {% endif %} + + {% if var.has_var('f2dq:query_test_failures') %} + {% set query_failures = var('f2dq:query_test_failures') %} + {% else %} + {% set query_failures = true %} + {% endif %} + + {% if el.failures and el.failures > 0 and el.node.relation_name and query_failures %} + {% if var.has_var('f2dq:test_history_failures_limit') %} + {% set limit_count = var('f2dq:test_history_failures_limit')%} + {% else %} + {% set limit_count = 10 %} + {% endif %} + + {% set failures_query %} + select * from {{ el.node.relation_name}} limit {{ limit_count }} + {% endset %} + {% set failures_list = f2dq.agate_to_list(run_query(failures_query)) %} + {% endif %} + + {% set failures_json = none %} + + {{ return ({ + 'table_name': table_name, + 'column_name': el.node.column_name or none, + 'test_name': el.node.name, + 'status': el.status.name, + 'execution_time': el.execution_time, + 'message': el.message, + 'failures_count': el.failures, + 'failures_json': '' ~ failures_list, + 'failures_table': el.node.relation_name or none, + 'severity': el.node.config.severity, + 'compiled_sql': el.node.compiled_sql or el.node.compiled_code or none, + 'run_at': run_started_at_str, + }) + }} + +{% endmacro %} + +{% macro priv_full_name_from_depends(node, name) %} + + {% for full_name in node.depends_on.nodes %} + {% set node_name = full_name.split('.')[-1] %} + {% if node_name == name %} + {{ return(full_name) }} + {% endif %} + {% endfor %} + + {{ return(none) }} + +{% endmacro %} diff --git a/macros/samples/internal_model_template.sql b/macros/samples/internal_model_template.sql new file mode 100644 index 0000000..bfa32e6 --- /dev/null +++ b/macros/samples/internal_model_template.sql @@ -0,0 +1,54 @@ +{%- macro order_by_if_time_filter(time_filter) -%} + {%- if time_filter is not none -%} + order by {{ time_filter }} desc + {%- endif -%} +{%- endmacro -%} + + +{% macro f2dq_last_table_samples() %} + {{ f2dq.generate_depends(['f2dq_selected', 'f2dq_monitored', 'f2dq_columns', 'f2dq_run_started_at', 'f2dq_last_table_samples_part']) }} + + {{ + config( + materialized='table', + ) + }} + + {% set store_samples = f2dq.f2dq_var('store_table_samples', false) %} + {% if not f2dq.in_compile() and store_samples is sameas true %} + {%- set tables = run_query(f2dq.get_tables()) %} + + {% set samples_list = [] %} + {%- for sample_table in tables %} + + {% set model = get_model_config(sample_table) %} + {% set columns_to_sample = [] %} + {% for key, value in model.columns_info.items() | sort %} + {% if value.data_type in ['numeric', 'text'] %} + {% do columns_to_sample.append(key) %} + {% endif %} + {% endfor %} + + {% set samples_query %} + select {{ print_list(columns_to_sample)}} from {{ model.table_name }} + {{ order_by_if_time_filter(model.time_filter) }} + limit 10 + {% endset %} + + {% set samples = f2dq.agate_to_list(run_query(samples_query)) %} + {% do samples_list.append({ + 'table_name': model.model_name, + 'sample_data': samples, + }) %} + + {% endfor %} + {% do f2dq.insert_list_to_table( + ref('f2dq_last_table_samples_part'), + samples_list, + ['table_name', 'sample_data'] + ) %} + {% endif %} + + {{ f2dq.empty_last_table_samples() }} + +{% endmacro %} \ No newline at end of file diff --git a/macros/store/insert_list_to_table.sql b/macros/store/insert_list_to_table.sql new file mode 100644 index 0000000..59ceb6b --- /dev/null +++ b/macros/store/insert_list_to_table.sql @@ -0,0 +1,42 @@ +{% macro insert_list_to_table(table, list, params, dtype=None,insert_size=100) %} + + {% set single_insert_list = [] %} + {% for el in list %} + {% do single_insert_list.append(el) %} + {% set single_insert_list_size = single_insert_list | length %} + {% if single_insert_list_size == insert_size or loop.last %} + + {% set insert_query %} + insert into {{ table }} ({%- for p in params %}{{p}}{% if not loop.last %}, {% endif %}{% endfor %}) values + {%- for row in single_insert_list -%} + ( + {%- for p in params -%} + {%- if row[p] is none -%} + NULL + {%- else -%} + {%- if row[p] is string -%} + {%- if dtype and p in dtype -%} + {% set cast_type = dtype[p] %} + cast ({{ f2dq.quote_string(row[p]) }} as {{ cast_type }}) + {%- else %} + {{- f2dq.quote_string(row[p]) -}} + {%- endif -%} + {%- elif row[p] is number -%} + {{-row[p]-}} + {%- else -%} + {{- f2dq.quote_string(tojson(row[p])) -}} + {%- endif -%} + {%- endif -%} + {%- if not loop.last -%},{%- endif -%} + {%- endfor -%} + ) + {%- if not loop.last -%},{%- endif %} + {% endfor -%} + {% endset %} + + {% do run_query(insert_query) %} + {% do single_insert_list.clear() %} + {% endif %} + {% endfor %} + +{% endmacro %} diff --git a/macros/tests/test_metrics.sql b/macros/tests/test_metrics.sql new file mode 100644 index 0000000..45cf3d2 --- /dev/null +++ b/macros/tests/test_metrics.sql @@ -0,0 +1,106 @@ + +{% macro metric_expression(table, metric, expression, column_name=None, condition=None) %} + select * from {{ref('f2dq_base_metrics')}} + where + table_name = '{{ f2dq.full_table_name_values(table.identifier, table.schema, table.database)}}' and + metric = '{{ metric }}' and + {% if condition is not none %} + {{ condition }} and + {% endif %} + {% if column_name is none %} + not ( {{ expression }} ) + {% else %} + column_name = '{{ column_name }}' and + not ( {{ expression }} ) + {% endif %} + +{% endmacro %} + +{# old test macros, will be removed after some time #} +{% test metric_expression_is_true(model, table, metric, expression, column_name=None, condition=None) %} + {{ f2dq.metric_expression(table, metric, expression, column_name=None, condition=None) }} +{% endtest %} + + +{% test metric_equal_to(model, table, metric, value, column_name=None, condition=None) %} + {{ f2dq.metric_expression(table, metric, 'value = ' ~ value, column_name, condition) }} +{% endtest %} + + +{% test metric_in_range(model, table, metric, min_value, max_value, column_name=None, condition=None) %} + {{ f2dq.metric_expression(table, metric, 'value >= ' ~ min_value ~ ' and value <= ' ~ max_value, column_name, condition) }} +{% endtest %} + +{# new test macros #} + +{% test assert_true(model, column_name=None, metric=None, expression=expression, condition=None) %} + -- depends_on: {{ ref('f2dq_base_metrics') }} + {% if execute %} + {{ f2dq.metric_expression(model, metric, expression, column_name, condition) }} + {% else %} + {{ f2dq.empty_table() }} + {% endif %} +{% endtest %} + +{% test assert_false(model, column_name=None, metric=None, expression=expression, condition=None) %} + -- depends_on: {{ ref('f2dq_base_metrics') }} + {% if execute %} + {{ f2dq.metric_expression(model, metric, 'not (' ~ expression ~ ')', column_name, condition) }} + {% else %} + {{ f2dq.empty_table() }} + {% endif %} +{% endtest %} + +{% test assert_in_range(model, column_name=None, metric=None, min_value=None, max_value=None, condition=None) %} + -- depends_on: {{ ref('f2dq_base_metrics') }} + {% if execute %} + {{ f2dq.metric_expression(model, metric, 'value >= ' ~ min_value ~ ' and value <= ' ~ max_value, column_name, condition) }} + {% else %} + {{ f2dq.empty_table() }} + {% endif %} +{% endtest %} + +{% test assert_equal(model, column_name=None, metric=None, value=value, condition=None) %} + -- depends_on: {{ ref('f2dq_base_metrics') }} + {% if execute %} + {{ f2dq.metric_expression(model, metric, 'value = ' ~ value, column_name, condition) }} + {% else %} + {{ f2dq.empty_table() }} + {% endif %} +{% endtest %} + +{% test assert_greater(model, column_name=None, metric=None, value=None, condition=None) %} + -- depends_on: {{ ref('f2dq_base_metrics') }} + {% if execute %} + {{ f2dq.metric_expression(model, metric, 'value > ' ~ value, column_name, condition) }} + {% else %} + {{ f2dq.empty_table() }} + {% endif %} +{% endtest %} + +{% test assert_greater_equal(model, column_name=None, metric=None, value=None, condition=None) %} + -- depends_on: {{ ref('f2dq_base_metrics') }} + {% if execute %} + {{ f2dq.metric_expression(model, metric, 'value >= ' ~ value, column_name, condition) }} + {% else %} + {{ f2dq.empty_table() }} + {% endif %} +{% endtest %} + +{% test assert_less(model, column_name=None, metric=None, value=None, condition=None) %} + -- depends_on: {{ ref('f2dq_base_metrics') }} + {% if execute %} + {{ f2dq.metric_expression(model, metric, 'value < ' ~ value, column_name, condition) }} + {% else %} + {{ f2dq.empty_table() }} + {% endif %} +{% endtest %} + +{% test assert_less_equal(model, column_name=None, metric=None, value=None, condition=None) %} + -- depends_on: {{ ref('f2dq_base_metrics') }} + {% if execute %} + {{ f2dq.metric_expression(model, metric, 'value <= ' ~ value, column_name, condition) }} + {% else %} + {{ f2dq.empty_table() }} + {% endif %} +{% endtest %} \ No newline at end of file diff --git a/macros/utils/agate/row_value.sql b/macros/utils/agate/row_value.sql new file mode 100644 index 0000000..b8a7797 --- /dev/null +++ b/macros/utils/agate/row_value.sql @@ -0,0 +1,20 @@ + +{% macro row_value(agate_row, column) %} + {{ return (agate_row[f2dq.name_in_db(column)]) }} +{% endmacro %} + +{% macro agate_to_list(table) %} + {% set col_names = table.column_names %} + {% set query_result = [] %} + {% for row in table.rows %} + {% set pairs = [] %} + {% for col_name in col_names %} + {% set value = row.get(col_name) | string %} + {% do pairs.append('"' ~ (col_name | lower) ~ '":' ~ '"' ~ (value | replace('"', '\\\"') | replace('\n', '\\n') ) ~ '"') %} + {% endfor %} + {% set joined_pairs = '{' ~ (pairs | join(',')) ~ '}' %} + {% do query_result.append(joined_pairs) %} + {% endfor %} + {% set query_result = '[' ~ (query_result | join(',')) ~ ']' %} + {{ return (query_result) }} +{% endmacro %} \ No newline at end of file diff --git a/macros/utils/bool_to_string.sql b/macros/utils/bool_to_string.sql new file mode 100644 index 0000000..897bf4a --- /dev/null +++ b/macros/utils/bool_to_string.sql @@ -0,0 +1,8 @@ + +{% macro bool_to_string(column) %} + ( + case when {{ column }} = true then 'true' + when {{ column }} = false then 'false' + end + ) as {{ column }} +{% endmacro %} \ No newline at end of file diff --git a/macros/utils/column_types.sql b/macros/utils/column_types.sql new file mode 100644 index 0000000..d7cba96 --- /dev/null +++ b/macros/utils/column_types.sql @@ -0,0 +1,102 @@ + +{% macro get_column_type(column) %} + {% set result = adapter.dispatch('get_column_type', 'f2dq')(column) %} + {{ return(result) }} +{% endmacro %} + + +{% macro default__get_column_type(column) %} + + {% if column.data_type in [ + 'character varying', + 'varchar', + 'character', + 'char', + 'text' + ] %} + {{ return('text') }} + + {% elif column.data_type in [ + 'smallint', + 'integer', + 'bigint', + 'decimal', + 'numeric', + 'real', + 'double precision', + 'enum', + ] %} + {{ return('numeric') }} + + {% elif column.data_type in [ 'boolean', 'bool' ] %} + + {{ return('boolean') }} + + {% else %} + {{ return('unknown') }} + + {% endif %} + +{% endmacro %} + + +{% macro snowflake__get_column_type(column) %} + + {% if column.DATA_TYPE in [ + 'VARCHAR', + 'CHAR', + 'CHARACTER', + 'STRING', + 'TEXT' + ] %} + + {{ return('text') }} + + {% elif column.DATA_TYPE in [ + 'NUMBER', + 'DECIMAL', + 'NUMERIC', + 'INT', + 'INTEGER', + 'BIGINT', + 'SMALLINT', + 'TINYINT', + 'BYTEINT', + 'FLOAT', + 'FLOAT4', + 'FLOAT8', + 'DOUBLE', + 'DOUBLE PRECISION', + 'REAL', + ] %} + + {{ return('numeric') }} + + {% elif column.data_type in [ 'BOOLEAN' ] %} + + {{ return('boolean') }} + + {% else %} + + {{ return('unknown') }} + + {% endif %} + +{% endmacro %} + + +{% macro bigquery__get_column_type(column) %} + + {% if column.data_type in [ 'STRING' ] %} + {{ return('text') }} + + {% elif column.data_type in [ "INT64", "NUMERIC", "BIGNUMERIC", "FLOAT64", "INTEGER"] %} + {{ return('numeric') }} + + {% elif column.data_type in [ "BOOLEAN", "BOOL"] %} + {{ return('boolean') }} + {% else %} + {{ return('unknown') }} + + {% endif %} +{% endmacro %} diff --git a/macros/utils/comma_delimited_list.sql b/macros/utils/comma_delimited_list.sql new file mode 100644 index 0000000..fcabb3a --- /dev/null +++ b/macros/utils/comma_delimited_list.sql @@ -0,0 +1,5 @@ +{% macro comma_delimited_list(args) %} + {%- for arg in args %} + {{- arg -}} {{- ", " if not loop.last else "" -}} + {% endfor %} +{% endmacro %} diff --git a/macros/utils/comparison_text.sql b/macros/utils/comparison_text.sql new file mode 100644 index 0000000..eeb767e --- /dev/null +++ b/macros/utils/comparison_text.sql @@ -0,0 +1,5 @@ +{% macro comparison_text(a, b) %} + case when {{a}} > {{b}} then 'greater than' + when {{a}} = {{b}} then 'equal to' + else 'less than' end +{% endmacro %} \ No newline at end of file diff --git a/macros/utils/compat_dual_read.sql b/macros/utils/compat_dual_read.sql new file mode 100644 index 0000000..ecf1f9b --- /dev/null +++ b/macros/utils/compat_dual_read.sql @@ -0,0 +1,58 @@ +{# Dual-read helpers for migrating from re_data → f2dq without breaking + projects that still use re_data_* config keys or re_data:* vars. #} + +{% macro f2dq_var(name, default=none) %} + {# Prefer f2dq:; fall back to re_data:; then default. + For booleans, either namespace set to true wins (so a project that + still sets re_data:store_table_samples: true keeps working even + though the package default for f2dq:store_table_samples is false). #} + {% set fkey = 'f2dq:' ~ name %} + {% set rkey = 're_data:' ~ name %} + {% set fval = var(fkey) if var.has_var(fkey) else none %} + {% set rval = var(rkey) if var.has_var(rkey) else none %} + {% if fval is sameas true or rval is sameas true %} + {{ return(true) }} + {% elif fval is sameas false and rval is sameas false %} + {{ return(false) }} + {% elif fval is not none and fval is not sameas false %} + {{ return(fval) }} + {% elif rval is not none and rval is not sameas false %} + {{ return(rval) }} + {% elif fval is not none %} + {{ return(fval) }} + {% elif rval is not none %} + {{ return(rval) }} + {% else %} + {{ return(default) }} + {% endif %} +{% endmacro %} + +{% macro f2dq_node_config(el, key, default=none) %} + {# Read f2dq_ / re_data_ from node config, then config.meta / + node.meta (dbt 1.12 CustomKeyInConfigDeprecation migration path). + `key` is the suffix without package prefix, e.g. 'monitored', + 'time_filter', 'metrics_groups'. #} + {% set keys = ['f2dq_' ~ key, 're_data_' ~ key] %} + {% for k in keys %} + {% if el.config is mapping and el.config.get(k) is not none %} + {{ return(el.config.get(k)) }} + {% endif %} + {% endfor %} + {% set conf_meta = el.config.get('meta', {}) if el.config is mapping else {} %} + {% if conf_meta is mapping %} + {% for k in keys %} + {% if conf_meta.get(k) is not none %} + {{ return(conf_meta.get(k)) }} + {% endif %} + {% endfor %} + {% endif %} + {% set node_meta = el.get('meta', {}) if el is mapping else {} %} + {% if node_meta is mapping %} + {% for k in keys %} + {% if node_meta.get(k) is not none %} + {{ return(node_meta.get(k)) }} + {% endif %} + {% endfor %} + {% endif %} + {{ return(default) }} +{% endmacro %} diff --git a/macros/utils/deduplication/add_duplication_context.sql b/macros/utils/deduplication/add_duplication_context.sql new file mode 100644 index 0000000..3f1482d --- /dev/null +++ b/macros/utils/deduplication/add_duplication_context.sql @@ -0,0 +1,13 @@ +{% macro add_duplication_context(relation, unique_cols, sort_columns) %} + + select {{ dbt_utils.star(from=relation) }} + , count(*) over ( + partition by {{ f2dq.comma_delimited_list(unique_cols) }} + ) as f2dq_duplicate_group_row_count + , row_number() over ( + partition by {{ f2dq.comma_delimited_list(unique_cols) }} {% if sort_columns %} order by {{ f2dq.comma_delimited_list(sort_columns) }} {% endif %} + ) as f2dq_duplicate_group_row_number + + from {{ relation }} + +{% endmacro %} \ No newline at end of file diff --git a/macros/utils/depends_macro.sql b/macros/utils/depends_macro.sql new file mode 100644 index 0000000..95cd988 --- /dev/null +++ b/macros/utils/depends_macro.sql @@ -0,0 +1,6 @@ +{% macro generate_depends(used_tables) %} + {% for t in used_tables %} + -- depends_on: {{ ref(t) }} + {% endfor %} + +{% endmacro %} \ No newline at end of file diff --git a/macros/utils/dict_from_list.sql b/macros/utils/dict_from_list.sql new file mode 100644 index 0000000..a52e7b9 --- /dev/null +++ b/macros/utils/dict_from_list.sql @@ -0,0 +1,14 @@ + +{% macro dict_from_list(el_list) %} + + {% if el_list is none %} + {{ return (none) }} + {% endif %} + + {% set for_cols_dict = {} %} + {% for col in el_list %} + {% do for_cols_dict.update({col: True})%} + {% endfor %} + {% do return(for_cols_dict) %} + +{% endmacro %} \ No newline at end of file diff --git a/macros/utils/fivetran_utils/json_extract.sql b/macros/utils/fivetran_utils/json_extract.sql new file mode 100644 index 0000000..8fc6002 --- /dev/null +++ b/macros/utils/fivetran_utils/json_extract.sql @@ -0,0 +1,40 @@ +{# +# This file contains significant part of code derived from +# https://github.com/fivetran/dbt_fivetran_utils/tree/v0.4.0 which is licensed under Apache License 2.0. +#} + +{% macro json_extract(string, string_path) -%} + +{{ adapter.dispatch('json_extract','f2dq') (string, string_path) }} + +{%- endmacro %} + +{% macro default__json_extract(string, string_path) %} + + json_extract_path_text({{string}}, {{ "'" ~ string_path ~ "'" }} ) + +{% endmacro %} + +{% macro snowflake__json_extract(string, string_path) %} + + json_extract_path_text(try_parse_json( {{string}} ), {{ "'" ~ string_path ~ "'" }} ) + +{% endmacro %} + +{% macro redshift__json_extract(string, string_path) %} + + case when is_valid_json( {{string}} ) then json_extract_path_text({{string}}, {{ "'" ~ string_path ~ "'" }} ) else null end + +{% endmacro %} + +{% macro bigquery__json_extract(string, string_path) %} + + json_extract_scalar({{string}}, {{ "'$." ~ string_path ~ "'" }} ) + +{% endmacro %} + +{% macro postgres__json_extract(string, string_path) %} + + {{string}}::json->>{{"'" ~ string_path ~ "'" }} + +{% endmacro %} diff --git a/macros/utils/fivetran_utils/percentile.sql b/macros/utils/fivetran_utils/percentile.sql new file mode 100644 index 0000000..7fca13f --- /dev/null +++ b/macros/utils/fivetran_utils/percentile.sql @@ -0,0 +1,60 @@ +{# +# This file contains significant part of code derived from +# https://github.com/fivetran/dbt_fivetran_utils/tree/v0.4.0 which is licensed under Apache License 2.0. +#} + +{% macro percentile(percentile_field, partition_field, percent) -%} + +{{ adapter.dispatch('percentile','f2dq') (percentile_field, partition_field, percent) }} + +{%- endmacro %} + +--percentile calculation specific to Redshift +{% macro default__percentile(percentile_field, partition_field, percent) %} + + percentile_cont( + {{ percent }} ) + within group ( order by {{ percentile_field }} ) + over ( partition by {{ partition_field }} ) + +{% endmacro %} + +--percentile calculation specific to Redshift +{% macro redshift__percentile(percentile_field, partition_field, percent) %} + + percentile_cont( + {{ percent }} ) + within group ( order by {{ percentile_field }} ) + over ( partition by {{ partition_field }} ) + +{% endmacro %} + +--percentile calculation specific to BigQuery +{% macro bigquery__percentile(percentile_field, partition_field, percent) %} + + percentile_cont( + {{ percentile_field }}, + {{ percent }}) + over (partition by {{ partition_field }} + ) + +{% endmacro %} + +{% macro postgres__percentile(percentile_field, partition_field, percent) %} + + percentile_cont( + {{ percent }} ) + within group ( order by {{ percentile_field }} ) + /* have to group by partition field */ + +{% endmacro %} + +{% macro spark__percentile(percentile_field, partition_field, percent) %} + + percentile( + {{ percentile_field }}, + {{ percent }}) + over (partition by {{ partition_field }} + ) + +{% endmacro %} diff --git a/macros/utils/for_loops.sql b/macros/utils/for_loops.sql new file mode 100644 index 0000000..719271c --- /dev/null +++ b/macros/utils/for_loops.sql @@ -0,0 +1,4 @@ + +{% macro print_list(l) %} + {% for el in l %}{{el}}{% if not loop.last %},{% endif %}{% endfor %} +{% endmacro %} \ No newline at end of file diff --git a/macros/utils/formulas.sql b/macros/utils/formulas.sql new file mode 100644 index 0000000..20a14bd --- /dev/null +++ b/macros/utils/formulas.sql @@ -0,0 +1,10 @@ +{% macro percentage_formula(summation, total) %} + abs( + ( + cast({{ summation }} as {{ numeric_type() }}) + ) / + nullif( + cast( {{ total }} as {{ numeric_type() }} ) + , 0) * 100.0 + ) +{% endmacro %} \ No newline at end of file diff --git a/macros/utils/generate_alert_message.sql b/macros/utils/generate_alert_message.sql new file mode 100644 index 0000000..5f1cce4 --- /dev/null +++ b/macros/utils/generate_alert_message.sql @@ -0,0 +1,65 @@ +{% macro generate_anomaly_message(column_name, metric, last_value, last_avg) -%} + {{ return(adapter.dispatch('generate_anomaly_message', 'f2dq')(column_name, metric, last_value, last_avg)) }} +{%- endmacro %} + +{% macro default__generate_anomaly_message(column_name, metric, last_value, last_avg) %} + + case when {{ column_name }} != '' then metric || '(' || column_name || ')' + else metric + end + || ' is ' || + {{ to_2dp( percentage_formula('last_value - last_avg', last_avg) ) }} + || '% ' || + {{ comparison_text(last_value, last_avg) }} + || ' average.' +{% endmacro %} + +{% macro to_2dp(val) %} + {{ adapter.dispatch('to_2dp', 'f2dq')(val) }} +{% endmacro %} + +{% macro default__to_2dp(val) %} + trim(to_char({{ val }}, '9999999999999999990D00')) +{% endmacro %} + +{% macro bigquery__to_2dp(val) %} + format('%.2f', {{ val }}) +{% endmacro %} + +{% macro seconds_to_hours(val) %} + cast({{ val }} as {{ numeric_type() }}) / 3600 +{% endmacro %} + +{% macro generate_metric_value_text(metric, value) %} + case + when {{ metric }} = 'freshness' + then cast({{ to_2dp(seconds_to_hours(value)) }} as {{ string_type() }}) || ' hours' + when {{ regex_match_expression(metric, 'percent') }} + then cast({{ to_2dp(value) }} as {{ string_type() }}) || '%' + when {{ regex_match_expression(metric, 'count') }} + then cast({{ value }} as {{ string_type() }}) + else cast({{ to_2dp(value) }} as {{ string_type() }}) + end + +{% endmacro %} + +{% macro generate_schema_change_message(operation, column_name, prev_column_name, prev_data_type, data_type, detected_time) %} + case + when {{ operation }} = 'column_added' + then 'column ' || {{ column_name }} || ' of type ' || {{ data_type }} || ' was added.' + when {{ operation }} = 'column_removed' + then 'column ' || {{ prev_column_name }} || ' of type ' || {{ prev_data_type }} || ' was removed.' + when {{ operation }} = 'type_change' + then {{ column_name }} || ' column data type was changed from ' || {{ prev_data_type }} || ' to ' || {{ data_type }} || '.' + else '' + end +{% endmacro %} + +{% macro generate_failed_test_message(test_name, column_name) %} + case + when {{ column_name }} is null + then 'Test ' || {{ test_name }} || ' failed.' + else + 'Test ' || {{ test_name }} || ' failed for column ' || {{ column_name }} || '.' + end +{% endmacro %} diff --git a/macros/utils/get_database.sql b/macros/utils/get_database.sql new file mode 100644 index 0000000..e4a89a6 --- /dev/null +++ b/macros/utils/get_database.sql @@ -0,0 +1,16 @@ + +{% macro get_target_database() %} + {{- adapter.dispatch('get_target_database', 'f2dq')() -}} +{% endmacro %} + +{% macro default__get_target_database() %} + {{- return (target.dbname) -}} +{% endmacro %} + +{% macro bigquery__get_target_database() %} + {{- return (target.project) -}} +{% endmacro %} + +{% macro snowflake__get_target_database() %} + {{- return (target.database) -}} +{% endmacro %} \ No newline at end of file diff --git a/macros/utils/in_compile.sql b/macros/utils/in_compile.sql new file mode 100644 index 0000000..7caf6b6 --- /dev/null +++ b/macros/utils/in_compile.sql @@ -0,0 +1,16 @@ +{% macro in_compile() %} + + {%- call statement('in_compile', fetch_result=True) -%} + select * from {{ ref('f2dq_run_started_at') }} + {%- endcall -%} + + {% if execute %} + {%- set result = load_result('in_compile')['data'][0][0] -%} + {% if result == run_started_at.timestamp() * 1000000 %} + {{ return(False) }} + {% else %} + {{ return(True) }} + {% endif %} + {% endif %} + +{% endmacro %} \ No newline at end of file diff --git a/macros/utils/is_list.sql b/macros/utils/is_list.sql new file mode 100644 index 0000000..c1d0732 --- /dev/null +++ b/macros/utils/is_list.sql @@ -0,0 +1,7 @@ +{% macro is_list(obj) %} + {% if not obj %} + {{ return (False) }} + {% endif %} + {% set check = obj is iterable and (obj is not string and obj is not mapping) %} + {{ return (check) }} +{% endmacro %} \ No newline at end of file diff --git a/macros/utils/json/to_single_json.sql b/macros/utils/json/to_single_json.sql new file mode 100644 index 0000000..3a0a079 --- /dev/null +++ b/macros/utils/json/to_single_json.sql @@ -0,0 +1,22 @@ +{% macro to_json_string_value_or_null(column) %} + ( + case + when {{ column }} is null then 'null' + else '"' || + regexp_replace( + replace(cast({{ column }} as {{ string_type() }}), '"', {{ escape_seq_for_json('"') }}), + '\n', {{ quote_new_line() }} {% if target.type in postgres_type_db() %}, 'g' {% endif %} + ) || '"' + end + ) +{% endmacro %} + +{% macro to_single_json(columns) %} + '{' || + {%- for column in columns %} + '"{{ column }}": ' || + {{ to_json_string_value_or_null(column) }} + {%- if not loop.last %} || ',' || {%- endif %} + {%- endfor %} + || '}' +{% endmacro %} diff --git a/macros/utils/mock/empty_tables.sql b/macros/utils/mock/empty_tables.sql new file mode 100644 index 0000000..4c35a50 --- /dev/null +++ b/macros/utils/mock/empty_tables.sql @@ -0,0 +1,67 @@ + +{% macro all_types_select() %} + with types_table as ( + select + cast (null as {{ string_type() }}) as string_type, + cast (null as {{ long_string_type() }}) as long_string_type, + cast (1 as {{ numeric_type() }}) as numeric_type, + cast ('2000-01-10' as {{ timestamp_type() }}) as timestamp_type, + cast (true as {{ boolean_type() }}) as boolean_type + ) +{% endmacro %} + +{% macro empty_table_generic(list) %} + {{ f2dq.all_types_select() }} + select + {% for name, type in list %} + {{ type }}_type as {{ name }} + {%- if not loop.last %}, {%- endif %} + {% endfor %} + from types_table + where string_type is not null +{% endmacro %} + +{% macro empty_last_base_metrics() %} + {{ + f2dq.empty_table_generic([ + ('table_name', 'string'), + ('column_name', 'string'), + ('metric', 'string'), + ('value', 'numeric') + ]) + }} +{% endmacro %} + +{% macro empty_last_table_samples() %} + {{ + f2dq.empty_table_generic([ + ('table_name', 'string'), + ('sample_data', 'string') + ]) + }} +{% endmacro %} + +{% macro empty_columns_table() %} + {{ + f2dq.empty_table_generic([ + ('name', 'string'), + ('schema', 'string'), + ('database', 'string'), + ('column_name', 'string'), + ('data_type', 'string'), + ('is_nullable', 'boolean'), + ('time_filter', 'string'), + ('computed_on', 'timestamp') + ]) + }} +{% endmacro %} + + +{% macro empty_table() %} + {{ + f2dq.empty_table_generic([ + ('name', 'string') + ]) + }} +{% endmacro %} + diff --git a/macros/utils/monitored_config.sql b/macros/utils/monitored_config.sql new file mode 100644 index 0000000..645b1d9 --- /dev/null +++ b/macros/utils/monitored_config.sql @@ -0,0 +1,38 @@ + + +{% macro columns_in_db(columns) %} + {% set translated = [] %} + {% if columns is none %} + {{ return (none) }} + {% endif %} + {% for col in columns %} + {% do translated.append(f2dq.name_in_db(col))%} + {% endfor %} + {{ return (translated) }} +{% endmacro %} + +{% macro metrics_in_db(metrics) %} + {% set translated = metrics %} + {% set column_metrics = {} %} + {% for col in metrics.column %} + {% do column_metrics.update({f2dq.name_in_db(col): metrics.column[col]}) %} + {% endfor %} + {% if column_metrics %} + {% do metrics.update({'column': column_metrics}) %} + {% endif %} + {{ return (metrics) }} +{% endmacro %} + +{% macro final_metrics(metrics_groups, additional_metrics) %} + {% set final_metrics_dict = dict([('group', {}), ('additional', {})]) %} + {% set all_metrics_groups = f2dq.f2dq_var('metrics_groups') %} + + {% for group in metrics_groups %} + {% set value = all_metrics_groups.get(group) %} + {% do final_metrics_dict['group'].update(value) %} + {% endfor %} + + {% do final_metrics_dict['additional'].update(additional_metrics) %} + {{ return (final_metrics_dict) }} + +{% endmacro %} \ No newline at end of file diff --git a/macros/utils/quote.sql b/macros/utils/quote.sql new file mode 100644 index 0000000..2ad8f37 --- /dev/null +++ b/macros/utils/quote.sql @@ -0,0 +1,13 @@ + + +{% macro quote_column(col_name) %} + {{ adapter.dispatch('quote_column', 'f2dq')(col_name) }} +{% endmacro %} + +{% macro default__quote_column(col_name) %} + "{{ col_name }}" +{% endmacro %} + +{% macro bigquery__quote_column(col_name) %} + `{{ col_name }}` +{% endmacro %} \ No newline at end of file diff --git a/macros/utils/regular_expression.sql b/macros/utils/regular_expression.sql new file mode 100644 index 0000000..24be854 --- /dev/null +++ b/macros/utils/regular_expression.sql @@ -0,0 +1,15 @@ +{% macro regex_match_expression(column_name, pattern) %} + {{ adapter.dispatch('regex_match_expression', 'f2dq')(column_name, pattern) }} +{% endmacro %} + +{% macro default__regex_match_expression(column_name, pattern) %} + ({{column_name}} ~ '{{pattern}}') +{% endmacro %} + +{% macro bigquery__regex_match_expression(column_name, pattern) %} + regexp_contains({{column_name}}, r'{{pattern}}') +{% endmacro %} + +{% macro snowflake__regex_match_expression(column_name, pattern) %} + regexp_like({{column_name | upper}}, '{{pattern}}') +{% endmacro %} diff --git a/macros/utils/time_macros.sql b/macros/utils/time_macros.sql new file mode 100644 index 0000000..359fe8f --- /dev/null +++ b/macros/utils/time_macros.sql @@ -0,0 +1,100 @@ + +{% macro time_window_start() %} + cast('{{- f2dq.f2dq_var('time_window_start') -}}' as timestamp) +{% endmacro %} + + +{% macro time_window_end() %} + cast('{{- f2dq.f2dq_var('time_window_end') -}}' as timestamp) +{% endmacro %} + + +{% macro anamaly_detection_time_window_start() %} + {{ adapter.dispatch('anamaly_detection_time_window_start', 'f2dq')() }} +{% endmacro %} + +{% macro default__anamaly_detection_time_window_start() %} + {{ time_window_start() }} - interval '{{ f2dq.f2dq_var('anomaly_detection_look_back_days') }} days' +{% endmacro %} + +{% macro bigquery__anamaly_detection_time_window_start() %} + DATE_ADD({{ time_window_start() }}, INTERVAL -{{ f2dq.f2dq_var('anomaly_detection_look_back_days') }} DAY) +{% endmacro %} + +{% macro snowflake__anamaly_detection_time_window_start() %} + DATEADD('DAY', -{{- f2dq.f2dq_var('anomaly_detection_look_back_days') -}}, {{ time_window_start() }}) +{% endmacro %} + + +{% macro interval_length_sec(start_timestamp, end_timestamp) %} + {{ adapter.dispatch('interval_length_sec', 'f2dq')(start_timestamp, end_timestamp) }} +{% endmacro %} + +{% macro default__interval_length_sec(start_timestamp, end_timestamp) %} + EXTRACT(EPOCH FROM ({{ end_timestamp }} - {{ start_timestamp }} )) +{% endmacro %} + +{% macro bigquery__interval_length_sec(start_timestamp, end_timestamp) %} + TIMESTAMP_DIFF ({{ end_timestamp }}, {{ start_timestamp }}, SECOND) +{% endmacro %} + +{% macro snowflake__interval_length_sec(start_timestamp, end_timestamp) %} + timediff(second, {{ start_timestamp }}, {{ end_timestamp }}) +{% endmacro %} + +{% macro redshift__interval_length_sec(start_timestamp, end_timestamp) %} + DATEDIFF(second, {{ start_timestamp }}, {{ end_timestamp }}) +{% endmacro %} + +{%- macro in_time_window(time_column) %} + {# /* If not time_filter is specified, we compute the metric over the entire table else we filter for the time frame */ #} + {% if time_column is none %} + true + {% else %} + {{ adapter.dispatch('in_time_window', 'f2dq')(time_column) }} + {% endif %} +{% endmacro -%} + +{% macro default__in_time_window(time_column) %} + {{time_column}} >= {{ time_window_start() }} and + {{time_column}} < {{ time_window_end() }} +{% endmacro %} + +{% macro bigquery__in_time_window(time_column) %} + cast({{time_column}} as timestamp) >= {{ time_window_start() }} and + cast({{time_column}} as timestamp) < {{ time_window_end() }} +{% endmacro %} + + +{% macro format_timestamp(column_name) %} + {{ adapter.dispatch('format_timestamp', 'f2dq')(column_name) }} +{% endmacro %} + +{% macro default__format_timestamp(column_name) %} + to_char({{column_name}}, 'YYYY-MM-DD HH24:MI:SS') +{% endmacro %} + +{% macro bigquery__format_timestamp(column_name) %} + FORMAT_TIMESTAMP('%Y-%m-%d %H:%I:%S', {{column_name}}) +{% endmacro %} + +/* +provide a common way to compare time vs a range: start_date <= target <= end_date +if start_date is none: target <= end_date +if end_date is none: target >= start_date +think none as infinity +*/ +{%- macro in_date_window(target, start_date, end_date) %} + {{ adapter.dispatch('in_date_window','f2dq')(target, start_date, end_date) }} +{% endmacro -%} + +{% macro default__in_date_window(target, start_date, end_date) %} + {% if start_date is not none and end_date is not none %} + date({{target}}) between '{{start_date}}' and '{{end_date}}' + {% elif start_date is none %} + date({{target}}) <= '{{end_date}}' + {% elif end_date is none %} + date({{target}}) >= '{{start_date}}' + {% endif %} +{% endmacro %} + diff --git a/macros/utils/used_types.sql b/macros/utils/used_types.sql new file mode 100644 index 0000000..a5a6fa3 --- /dev/null +++ b/macros/utils/used_types.sql @@ -0,0 +1,100 @@ +{% macro timestamp_type() %} + {{ adapter.dispatch('timestamp_type', 'f2dq')() }} +{% endmacro %} + +{% macro default__timestamp_type() %} + timestamp without time zone +{% endmacro %} + +{% macro redshift__timestamp_type() %} + TIMESTAMP +{% endmacro %} + +{% macro bigquery__timestamp_type() %} + TIMESTAMP +{% endmacro %} + +{% macro snowflake__timestamp_type() %} + TIMESTAMP_NTZ +{% endmacro %} + +{% macro string_type() %} + {{ adapter.dispatch('string_type', 'f2dq')() }} +{% endmacro %} + +{% macro default__string_type() %} + text +{% endmacro %} + +{% macro redshift__string_type() %} + varchar(2047) +{% endmacro %} + +{% macro bigquery__string_type() %} + STRING +{% endmacro %} + +{% macro snowflake__string_type() %} + STRING +{% endmacro %} + +{% macro long_string_type() %} + {{ adapter.dispatch('long_string_type', 'f2dq')() }} +{% endmacro %} + +{% macro default__long_string_type() %} + {{ f2dq.string_type() }} +{% endmacro %} + +{% macro redshift__long_string_type() %} + varchar(65535) +{% endmacro %} + +{% macro integer_type() %} + INTEGER +{% endmacro %} + + +{% macro boolean_type() %} + {{ adapter.dispatch('boolean_type', 'f2dq')() }} +{% endmacro %} + +{% macro default__boolean_type() %} + BOOLEAN +{% endmacro %} + +{% macro redshift__boolean_type() %} + boolean +{% endmacro %} + +{% macro bigquery__boolean_type() %} + BOOLEAN +{% endmacro %} + +{% macro snowflake__boolean_type() %} + BOOLEAN +{% endmacro %} + + +{% macro numeric_type() %} + {{ adapter.dispatch('numeric_type', 'f2dq')() }} +{% endmacro %} + +{% macro default__numeric_type() %} + double precision +{% endmacro %} + +{% macro redshift__numeric_type() %} + DOUBLE PRECISION +{% endmacro %} + +{% macro bigquery__numeric_type() %} + FLOAT64 +{% endmacro %} + +{% macro snowflake__numeric_type() %} + FLOAT +{% endmacro %} + + + diff --git a/models/alerts/f2dq_alerts.sql b/models/alerts/f2dq_alerts.sql new file mode 100644 index 0000000..62e99a5 --- /dev/null +++ b/models/alerts/f2dq_alerts.sql @@ -0,0 +1,34 @@ +select + 'anomaly' as type, + {{ f2dq.clean_blacklist('table_name', ['"', '`'], '') }} as model, + message, + last_value_text as value, + time_window_end +from + {{ ref(var('f2dq:anomalies_filtered')) }} +union all + +select + 'schema_change' as type, + {{ f2dq.clean_blacklist('table_name', ['"', '`'], '') }} as model, + {{ generate_schema_change_message('operation', 'column_name', 'prev_column_name', 'prev_data_type', 'data_type', 'detected_time') }} as message, + '' as value, + detected_time as time_window_end +from {{ ref('f2dq_schema_changes') }} + +union all + +select + 'test' as type, + table_name as model, + {{ generate_failed_test_message('test_name', 'column_name') }}, + status as value, + run_at as time_window_end + +from {{ ref('f2dq_test_history') }} +where + status = 'Fail' + or status = 'Error' + {% if f2dq.f2dq_var('show_warns_as_alerts', false) %} + or status = 'Warn' + {% endif %} diff --git a/models/alerts/f2dq_anomalies.sql b/models/alerts/f2dq_anomalies.sql new file mode 100644 index 0000000..1286fa7 --- /dev/null +++ b/models/alerts/f2dq_anomalies.sql @@ -0,0 +1,55 @@ +{{ + config( + materialized='view' + ) +}} +select + z.id, + z.table_name, + z.column_name, + z.metric, + z.z_score_value, + z.modified_z_score_value, + m.anomaly_detector, + z.last_value, + z.last_avg, + z.last_median, + z.last_stddev, + z.last_median_absolute_deviation, + z.last_mean_absolute_deviation, + z.last_iqr, + z.last_first_quartile - (cast( {{ json_extract('m.anomaly_detector', 'whisker_boundary_multiplier') }} as {{numeric_type()}} ) * z.last_iqr) lower_bound, + z.last_third_quartile + (cast( {{ json_extract('m.anomaly_detector', 'whisker_boundary_multiplier') }} as {{numeric_type()}} ) * z.last_iqr) upper_bound, + z.last_first_quartile, + z.last_third_quartile, + z.time_window_end, + z.interval_length_sec, + z.computed_on, + {{ f2dq.generate_anomaly_message('z.column_name', 'z.metric', 'z.last_value', 'z.last_avg') }} as message, + {{ f2dq.generate_metric_value_text('z.metric', 'z.last_value') }} as last_value_text +from + {{ ref('f2dq_z_score')}} z +left join {{ ref('f2dq_selected') }} m +on {{ split_and_return_nth_value('table_name', '.', 1) }} = m.database +and {{ split_and_return_nth_value('table_name', '.', 2) }} = m.schema +and {{ split_and_return_nth_value('table_name', '.', 3) }} = m.name +where + case when (lower(coalesce({{ json_extract('m.anomaly_detector', 'direction') }}, 'both')) = 'up' and z.last_value > z.last_avg) + or (lower(coalesce({{ json_extract('m.anomaly_detector', 'direction') }}, 'both')) = 'down' and z.last_value < z.last_avg) + or (lower(coalesce({{ json_extract('m.anomaly_detector', 'direction') }}, 'both')) != 'up' and lower(coalesce({{ json_extract('m.anomaly_detector', 'direction') }}, 'both')) != 'down') + then + case + when {{ json_extract('m.anomaly_detector', 'name') }} = 'z_score' + then abs(z_score_value) > cast({{ json_extract('m.anomaly_detector', 'threshold') }} as {{ numeric_type() }}) + when {{ json_extract('m.anomaly_detector', 'name') }} = 'modified_z_score' + then abs(modified_z_score_value) > cast( {{ json_extract('m.anomaly_detector', 'threshold') }} as {{numeric_type()}} ) + when {{ json_extract('m.anomaly_detector', 'name') }} = 'boxplot' + then ( + z.last_value < z.last_first_quartile - (cast( {{ json_extract('m.anomaly_detector', 'whisker_boundary_multiplier') }} as {{numeric_type()}} ) * z.last_iqr) + or + z.last_value > z.last_third_quartile + (cast( {{ json_extract('m.anomaly_detector', 'whisker_boundary_multiplier') }} as {{numeric_type()}} ) * z.last_iqr) + ) + else false + end + else false + end diff --git a/models/alerts/f2dq_schema_changes.sql b/models/alerts/f2dq_schema_changes.sql new file mode 100644 index 0000000..d1af2a3 --- /dev/null +++ b/models/alerts/f2dq_schema_changes.sql @@ -0,0 +1,167 @@ +{{ + config( + materialized='incremental', + unique_key = 'id', + on_schema_change='sync_all_columns', + ) +}} + +-- depends_on: {{ ref('f2dq_run_started_at') }} +-- depends_on: {{ ref('f2dq_columns_over_time') }} +-- depends_on: {{ ref('f2dq_monitored') }} +-- depends_on: {{ ref('f2dq_selected') }} + +{% if execute and not f2dq.in_compile() %} + {% set last_data_points %} + select + distinct detected_time + from {{ ref('f2dq_columns_over_time') }} + order by + detected_time desc limit 2; + {% endset %} + + {% set detected_times = run_query(last_data_points) %} + + {% set times_list = detected_times.columns[0].values() %} + {% set most_recent_time = times_list[0] %} + + {% if times_list | length > 1 %} + {% set prev_most_recent = times_list[1] %} + {% else %} + {% set prev_most_recent = times_list[0] %} + {% endif %} +{% else %} + {% set times_list = () %} +{% endif %} + +{% if times_list == () %} + {{ + f2dq.empty_table_generic([ + ('id', 'string'), + ('table_name', 'string'), + ('operation', 'string'), + ('column_name', 'string'), + ('data_type', 'string'), + ('is_nullable', 'boolean'), + ('prev_column_name', 'string'), + ('prev_data_type', 'string'), + ('prev_is_nullable', 'boolean'), + ('detected_time', 'timestamp') + ]) + }} +{% else %} + + with curr_monitored_schema as ( + select * from {{ ref('f2dq_columns_over_time')}} + where detected_time = cast('{{ most_recent_time }}' as {{ timestamp_type() }}) + and table_name in ( + select {{ full_table_name('name', 'schema', 'database') }} from {{ ref('f2dq_selected')}} + ) + ), + + + prev_monitored_schema as ( + select * from {{ ref('f2dq_columns_over_time')}} + where detected_time = cast('{{ prev_most_recent}}' as {{ timestamp_type() }}) + and table_name in ( + select {{ full_table_name('name', 'schema', 'database') }} from {{ ref('f2dq_selected')}} + ) + ), + + all_changes as ( + ( + select + curr.table_name as table_name, + 'type_change' as operation, + curr.column_name as column_name, + curr.data_type as data_type, + curr.is_nullable as is_nullable, + + prev.column_name as prev_column_name, + prev.data_type as prev_data_type, + prev.is_nullable as prev_is_nullable + + from curr_monitored_schema curr inner join prev_monitored_schema prev on (curr.table_name = prev.table_name and curr.column_name = prev.column_name) + where + curr.data_type != prev.data_type or + curr.is_nullable != prev.is_nullable + ) + + union all + + ( + + select + curr.table_name as table_name, + 'column_added' as operation, + curr.column_name as column_name, + curr.data_type as data_type, + curr.is_nullable as is_nullable, + + null as prev_column_name, + null as prev_data_type, + null as prev_is_nullable + + from curr_monitored_schema curr left join prev_monitored_schema prev on (curr.table_name = prev.table_name and curr.column_name = prev.column_name) + where prev.table_name is null and prev.column_name is null + {# note: when a column is added, make sure we only detect for models that were previously monitored, + this avoids a situation where a newly monitored model has all its columns detected with 'column_added' operation#} + and curr.table_name in ( + select table_name from prev_monitored_schema + ) + + ) + + union all + + ( + + select + prev.table_name as table_name, + 'column_removed' as operation, + null as column_name, + null as data_type, + null as is_nullable, + + prev.column_name as prev_column_name, + prev.data_type as prev_data_type, + prev.is_nullable as prev_is_nullable + + from prev_monitored_schema prev left join curr_monitored_schema curr on (curr.table_name = prev.table_name and curr.column_name = prev.column_name) + where curr.table_name is null and curr.column_name is null + + ) + ), + + all_with_time as ( + select + all_changes.table_name, + all_changes.operation, + all_changes.column_name, + all_changes.data_type, + all_changes.is_nullable, + all_changes.prev_column_name, + all_changes.prev_data_type, + all_changes.prev_is_nullable, + cast({{dbt.current_timestamp_backcompat()}} as {{ timestamp_type() }}) as detected_time + from all_changes + ) + + select + cast ({{ dbt_utils.generate_surrogate_key([ + 'table_name', + 'column_name', + 'detected_time' + ]) }} as {{ string_type() }} ) as id, + table_name, + cast (operation as {{ string_type() }}) as operation, + column_name, + data_type, + is_nullable, + prev_column_name, + prev_data_type, + prev_is_nullable, + detected_time + from all_with_time + +{% endif %} diff --git a/models/alerts/f2dq_test_runs.sql b/models/alerts/f2dq_test_runs.sql new file mode 100644 index 0000000..f716d9c --- /dev/null +++ b/models/alerts/f2dq_test_runs.sql @@ -0,0 +1,13 @@ +{{ + config( + materialized='view' + ) +}} + +select + sum(case when status = 'Fail' then 1 else 0 end) as failed, + sum(case when status = 'Pass' then 1 else 0 end) as passed, + run_at +from {{ ref ('f2dq_test_history') }} +group by run_at +order by run_at desc \ No newline at end of file diff --git a/models/alerts/f2dq_z_score.sql b/models/alerts/f2dq_z_score.sql new file mode 100644 index 0000000..138e5aa --- /dev/null +++ b/models/alerts/f2dq_z_score.sql @@ -0,0 +1,76 @@ +{{ + config( + materialized='incremental', + unique_key = 'id', + on_schema_change='sync_all_columns', + ) +}} + +with z_score_without_id as ( + + select + stats.table_name as table_name, + stats.column_name as column_name, + stats.metric as metric, + stats.interval_length_sec, + (last_metric.last_value - stats.last_avg) / (stats.last_stddev + 0.0000000001) as z_score_value, + case + when stats.last_median_absolute_deviation = 0 then + (last_metric.last_value - stats.last_median) / (1.253314 * (stats.last_mean_absolute_deviation + 0.0000000001)) + else + (0.6745 * (last_metric.last_value - stats.last_median)) / (stats.last_median_absolute_deviation + 0.0000000001) + end as modified_z_score_value, + last_metric.last_value as last_value, + stats.last_avg as last_avg, + stats.last_median as last_median, + stats.last_stddev as last_stddev, + stats.last_median_absolute_deviation, + stats.last_mean_absolute_deviation, + stats.last_third_quartile - stats.last_first_quartile as last_iqr, + stats.last_first_quartile, + stats.last_third_quartile, + {{ time_window_end() }} as time_window_end, + cast( {{dbt.current_timestamp_backcompat()}} as {{ timestamp_type() }} ) as computed_on + from + {{ ref('f2dq_last_stats') }} as stats, + {{ ref('f2dq_last_metrics') }} as last_metric + where + stats.table_name = last_metric.table_name and + stats.column_name = last_metric.column_name and + stats.metric = last_metric.metric and + ( + stats.interval_length_sec = last_metric.interval_length_sec or + (stats.interval_length_sec is null and last_metric.interval_length_sec is null) + ) and + last_metric.last_value is not null and + stats.last_avg is not null and + stats.last_stddev is not null + ) + +select + cast ({{ dbt_utils.generate_surrogate_key([ + 'table_name', + 'column_name', + 'metric', + 'interval_length_sec', + 'time_window_end' + ]) }} as {{ string_type() }} ) as id, + table_name, + column_name, + metric, + z_score_value, + modified_z_score_value, + last_value, + last_avg, + last_median, + last_stddev, + last_median_absolute_deviation, + last_mean_absolute_deviation, + last_iqr, + last_first_quartile, + last_third_quartile, + time_window_end, + interval_length_sec, + computed_on + +from z_score_without_id diff --git a/models/internal/f2dq_run_started_at.sql b/models/internal/f2dq_run_started_at.sql new file mode 100644 index 0000000..8ce8035 --- /dev/null +++ b/models/internal/f2dq_run_started_at.sql @@ -0,0 +1,7 @@ +{{ + config( + materialized='table', + ) +}} + +select {{ run_started_at.timestamp() * 1000000 }} as run_started_at \ No newline at end of file diff --git a/models/internal/metrics/base/f2dq_last_base_metrics_part0.sql b/models/internal/metrics/base/f2dq_last_base_metrics_part0.sql new file mode 100644 index 0000000..f6d3ec1 --- /dev/null +++ b/models/internal/metrics/base/f2dq_last_base_metrics_part0.sql @@ -0,0 +1 @@ +{{ f2dq_last_base_metrics_part() }} \ No newline at end of file diff --git a/models/internal/metrics/base/f2dq_last_base_metrics_part1.sql b/models/internal/metrics/base/f2dq_last_base_metrics_part1.sql new file mode 100644 index 0000000..f6d3ec1 --- /dev/null +++ b/models/internal/metrics/base/f2dq_last_base_metrics_part1.sql @@ -0,0 +1 @@ +{{ f2dq_last_base_metrics_part() }} \ No newline at end of file diff --git a/models/internal/metrics/base/f2dq_last_base_metrics_part2.sql b/models/internal/metrics/base/f2dq_last_base_metrics_part2.sql new file mode 100644 index 0000000..f6d3ec1 --- /dev/null +++ b/models/internal/metrics/base/f2dq_last_base_metrics_part2.sql @@ -0,0 +1 @@ +{{ f2dq_last_base_metrics_part() }} \ No newline at end of file diff --git a/models/internal/metrics/base/f2dq_last_base_metrics_part3.sql b/models/internal/metrics/base/f2dq_last_base_metrics_part3.sql new file mode 100644 index 0000000..f6d3ec1 --- /dev/null +++ b/models/internal/metrics/base/f2dq_last_base_metrics_part3.sql @@ -0,0 +1 @@ +{{ f2dq_last_base_metrics_part() }} \ No newline at end of file diff --git a/models/internal/metrics/base/f2dq_last_base_metrics_thread0.sql b/models/internal/metrics/base/f2dq_last_base_metrics_thread0.sql new file mode 100644 index 0000000..c559bc6 --- /dev/null +++ b/models/internal/metrics/base/f2dq_last_base_metrics_thread0.sql @@ -0,0 +1 @@ +{{ f2dq_last_base_metrics_thread(0)}} \ No newline at end of file diff --git a/models/internal/metrics/base/f2dq_last_base_metrics_thread1.sql b/models/internal/metrics/base/f2dq_last_base_metrics_thread1.sql new file mode 100644 index 0000000..f5f4227 --- /dev/null +++ b/models/internal/metrics/base/f2dq_last_base_metrics_thread1.sql @@ -0,0 +1 @@ +{{ f2dq_last_base_metrics_thread(1)}} \ No newline at end of file diff --git a/models/internal/metrics/base/f2dq_last_base_metrics_thread2.sql b/models/internal/metrics/base/f2dq_last_base_metrics_thread2.sql new file mode 100644 index 0000000..91c16d3 --- /dev/null +++ b/models/internal/metrics/base/f2dq_last_base_metrics_thread2.sql @@ -0,0 +1 @@ +{{ f2dq_last_base_metrics_thread(2)}} \ No newline at end of file diff --git a/models/internal/metrics/base/f2dq_last_base_metrics_thread3.sql b/models/internal/metrics/base/f2dq_last_base_metrics_thread3.sql new file mode 100644 index 0000000..bde4984 --- /dev/null +++ b/models/internal/metrics/base/f2dq_last_base_metrics_thread3.sql @@ -0,0 +1 @@ +{{ f2dq_last_base_metrics_thread(3)}} \ No newline at end of file diff --git a/models/internal/samples/f2dq_last_table_samples.sql b/models/internal/samples/f2dq_last_table_samples.sql new file mode 100644 index 0000000..0a63945 --- /dev/null +++ b/models/internal/samples/f2dq_last_table_samples.sql @@ -0,0 +1 @@ +{{ f2dq_last_table_samples() }} \ No newline at end of file diff --git a/models/internal/samples/f2dq_last_table_samples_part.sql b/models/internal/samples/f2dq_last_table_samples_part.sql new file mode 100644 index 0000000..1522c36 --- /dev/null +++ b/models/internal/samples/f2dq_last_table_samples_part.sql @@ -0,0 +1,7 @@ +{{ + config( + materialized='table', + ) +}} + +{{ f2dq.empty_last_table_samples() }} \ No newline at end of file diff --git a/models/meta/f2dq_columns.sql b/models/meta/f2dq_columns.sql new file mode 100644 index 0000000..df8c63f --- /dev/null +++ b/models/meta/f2dq_columns.sql @@ -0,0 +1,39 @@ +{{ + config( + materialized='table', + ) +}} + +-- depends_on: {{ ref('f2dq_run_started_at') }} +-- depends_on: {{ ref('f2dq_monitored') }} +-- depends_on: {{ ref('f2dq_selected') }} + +{% if execute %} + {% set schemas = run_query(f2dq.get_schemas()) %} + {% if schemas %} + + with columns_from_select as ( + {% for row in schemas %} + {% set schema_name = f2dq.name_in_db(f2dq.row_value(row, 'schema')) %} + {{ get_monitored_columns(schema_name, f2dq.row_value(row, 'database')) }} + {%- if not loop.last %} union all {%- endif %} + {% endfor %} + ) + + select + cast (table_name as {{ string_type() }} ) as name, + cast (table_schema as {{ string_type() }} ) as schema, + cast (table_catalog as {{ string_type() }} ) as database, + cast (column_name as {{ string_type() }} ) as column_name, + cast (data_type as {{ string_type() }} ) as data_type, + cast (case is_nullable when 'YES' then 1 else 0 end as {{ boolean_type() }} ) as is_nullable, + cast ({{dbt.current_timestamp_backcompat()}} as {{ timestamp_type() }}) as computed_on + from columns_from_select + + {% else %} + {{ empty_columns_table() }} + {% endif %} + +{% else %} + {{ empty_columns_table() }} +{% endif %} diff --git a/models/meta/f2dq_monitored.sql b/models/meta/f2dq_monitored.sql new file mode 100644 index 0000000..fee012d --- /dev/null +++ b/models/meta/f2dq_monitored.sql @@ -0,0 +1,23 @@ +{{ + config( + materialized='table', + unique_key = 'table_name', + post_hook="{% if execute %}{{ pub_insert_into_f2dq_monitored() }}{% endif %}" + ) +}} + +{{ + f2dq.empty_table_generic([ + ('name', 'string'), + ('schema', 'string'), + ('database', 'string'), + ('time_filter', 'string'), + ('metrics_groups', 'string'), + ('additional_metrics', 'string'), + ('metrics', 'string'), + ('columns', 'string'), + ('anomaly_detector', 'string'), + ('owners', 'string'), + ('selected', 'boolean') + ]) +}} \ No newline at end of file diff --git a/models/meta/f2dq_selected.sql b/models/meta/f2dq_selected.sql new file mode 100644 index 0000000..e293550 --- /dev/null +++ b/models/meta/f2dq_selected.sql @@ -0,0 +1,6 @@ + +select + name, schema, database, time_filter, metrics, columns, anomaly_detector, owners +from {{ ref('f2dq_monitored')}} +where + selected = true \ No newline at end of file diff --git a/models/metrics/final/f2dq_metrics.sql b/models/metrics/final/f2dq_metrics.sql new file mode 100644 index 0000000..9686a50 --- /dev/null +++ b/models/metrics/final/f2dq_metrics.sql @@ -0,0 +1 @@ +select * from {{ ref('f2dq_base_metrics') }} \ No newline at end of file diff --git a/models/metrics/for_anomalies/f2dq_last_metrics.sql b/models/metrics/for_anomalies/f2dq_last_metrics.sql new file mode 100644 index 0000000..fffac13 --- /dev/null +++ b/models/metrics/for_anomalies/f2dq_last_metrics.sql @@ -0,0 +1,12 @@ +select + table_name, + column_name, + metric, + value as last_value, + interval_length_sec, + computed_on +from + {{ ref('f2dq_base_metrics') }} +where + time_window_end = {{- time_window_end() -}} + diff --git a/models/metrics/for_anomalies/f2dq_last_stats.sql b/models/metrics/for_anomalies/f2dq_last_stats.sql new file mode 100644 index 0000000..bb7fd7a --- /dev/null +++ b/models/metrics/for_anomalies/f2dq_last_stats.sql @@ -0,0 +1,104 @@ +{% set columns_to_group_by = 'table_name, column_name, metric, interval_length_sec' %} + +with median_value as ( + select distinct + table_name, + column_name, + metric, + interval_length_sec, + avg(value) {% if target.type not in postgres_type_db() %} over(partition by {{ columns_to_group_by }}) {% endif %} as last_avg, + {{ percentile(percentile_field='value', partition_field=columns_to_group_by, percent='0.25') }} as last_first_quartile, + {{ percentile(percentile_field='value', partition_field=columns_to_group_by, percent='0.5') }} as last_median, + {{ percentile(percentile_field='value', partition_field=columns_to_group_by, percent='0.75') }} as last_third_quartile + from + {{ ref('f2dq_base_metrics') }} + where + time_window_end > {{- anamaly_detection_time_window_start() -}} and + time_window_end <= {{- time_window_end() -}} + {% if target.type in postgres_type_db() %} + group by + {{ columns_to_group_by }} + {% endif %} + +), abs_deviation as ( + select + s.table_name, + s.column_name, + s.metric, + s.interval_length_sec, + abs( s.value - mv.last_avg ) as absolute_deviation_from_mean, + abs( s.value - mv.last_median ) as absolute_deviation_from_median + from + {{ ref('f2dq_base_metrics') }} s + left join + median_value mv + on + s.table_name = mv.table_name and + s.column_name = mv.column_name and + s.metric = mv.metric and + s.interval_length_sec = mv.interval_length_sec + where + s.time_window_end > {{- anamaly_detection_time_window_start() -}} and + s.time_window_end <= {{- time_window_end() -}} +), median_abs_deviation as ( + select distinct + table_name, + column_name, + metric, + interval_length_sec, + avg(absolute_deviation_from_mean) {% if target.type not in postgres_type_db() %} over(partition by {{ columns_to_group_by }}) {% endif %} as mean_absolute_deviation, + {{ percentile(percentile_field='absolute_deviation_from_median', partition_field=columns_to_group_by, percent='0.5') }} as median_absolute_deviation + from + abs_deviation + {% if target.type in postgres_type_db() %} + group by + {{ columns_to_group_by }} + {% endif %} +), stats as ( + select + table_name, + column_name, + metric, + stddev(value) as last_stddev, + max(time_window_end) as last_metric_time, + interval_length_sec, + max(computed_on) as computed_on + from + {{ ref('f2dq_base_metrics') }} + where + time_window_end > {{- anamaly_detection_time_window_start() -}} and + time_window_end <= {{- time_window_end() -}} + group by + {{ columns_to_group_by }} +) + +select + s.table_name, + s.column_name, + s.metric, + mv.last_avg, + s.last_stddev, + s.last_metric_time, + s.interval_length_sec, + s.computed_on, + mv.last_median, + mv.last_first_quartile, + mv.last_third_quartile, + md.median_absolute_deviation last_median_absolute_deviation, + md.mean_absolute_deviation last_mean_absolute_deviation +from + stats s +left join + median_value mv + on + s.table_name = mv.table_name and + s.column_name = mv.column_name and + s.metric = mv.metric and + s.interval_length_sec = mv.interval_length_sec +left join + median_abs_deviation md + on + s.table_name = md.table_name and + s.column_name = md.column_name and + s.metric = md.metric and + s.interval_length_sec = md.interval_length_sec diff --git a/models/metrics/types/base/f2dq_base_metrics.sql b/models/metrics/types/base/f2dq_base_metrics.sql new file mode 100644 index 0000000..8b78030 --- /dev/null +++ b/models/metrics/types/base/f2dq_base_metrics.sql @@ -0,0 +1,54 @@ +{{ + config( + materialized='incremental', + unique_key = 'id', + on_schema_change='sync_all_columns', + ) +}} + +-- depends_on: {{ ref('f2dq_columns') }} +-- depends_on: {{ ref('f2dq_last_base_metrics_thread0') }} +-- depends_on: {{ ref('f2dq_last_base_metrics_thread1') }} +-- depends_on: {{ ref('f2dq_last_base_metrics_thread2') }} +-- depends_on: {{ ref('f2dq_last_base_metrics_thread3') }} +-- depends_on: {{ ref('f2dq_last_base_metrics_part0') }} +-- depends_on: {{ ref('f2dq_last_base_metrics_part1') }} +-- depends_on: {{ ref('f2dq_last_base_metrics_part2') }} +-- depends_on: {{ ref('f2dq_last_base_metrics_part3') }} +-- depends_on: {{ ref('f2dq_run_started_at') }} +-- depends_on: {{ ref('f2dq_monitored') }} +-- depends_on: {{ ref('f2dq_selected') }} + +with + +with_time_window as ( + {% set parts = ['0','1','2','3'] %} + {% for part in parts %} + {% set ref_name = 'f2dq_last_base_metrics_part' + part %} + select + *, + {{ time_window_start() }} as time_window_start, + {{ time_window_end() }} as time_window_end + from {{ ref(ref_name) }} + {%- if not loop.last %} union all {%- endif %} + {% endfor %} +) +select + cast ({{ dbt_utils.generate_surrogate_key([ + 'table_name', + 'column_name', + 'metric', + 'time_window_start', + 'time_window_end' + ]) }} as {{ string_type() }} ) as id, + cast (table_name as {{ string_type() }} ) as table_name, + cast (column_name as {{ string_type() }} ) as column_name, + cast (metric as {{ string_type() }} ) as metric, + cast (value as {{ numeric_type() }} ) as value, + cast (time_window_start as {{ timestamp_type() }} ) as time_window_start, + cast (time_window_end as {{ timestamp_type() }} ) as time_window_end, + cast ( + {{ interval_length_sec('time_window_start', 'time_window_end') }} as {{ integer_type() }} + ) as interval_length_sec, + cast ({{dbt.current_timestamp_backcompat()}} as {{ timestamp_type() }}) as computed_on +from with_time_window diff --git a/models/metrics/types/samples/f2dq_table_samples.sql b/models/metrics/types/samples/f2dq_table_samples.sql new file mode 100644 index 0000000..3dd442d --- /dev/null +++ b/models/metrics/types/samples/f2dq_table_samples.sql @@ -0,0 +1,17 @@ +{{ + config( + materialized='table', + unique_key = 'table_name', + on_schema_change='sync_all_columns', + ) +}} + +-- depends_on: {{ ref('f2dq_last_table_samples') }} +-- depends_on: {{ ref('f2dq_last_table_samples_part') }} + +select + table_name, + sample_data, + cast ({{dbt.current_timestamp_backcompat()}} as {{ timestamp_type() }}) as sampled_on + +from {{ ref('f2dq_last_table_samples_part') }} diff --git a/models/metrics/types/schema/f2dq_columns_over_time.sql b/models/metrics/types/schema/f2dq_columns_over_time.sql new file mode 100644 index 0000000..2f093bd --- /dev/null +++ b/models/metrics/types/schema/f2dq_columns_over_time.sql @@ -0,0 +1,35 @@ +{{ + config( + materialized='incremental', + unique_key = 'id', + on_schema_change='sync_all_columns', + ) +}} + + +with columns as ( + +select + {{ full_table_name('cols.name', 'cols.schema', 'cols.database') }} as table_name, + cols.column_name, + cols.data_type, + cols.is_nullable, + cast ({{dbt.current_timestamp_backcompat()}} as {{ timestamp_type() }} ) as detected_time +from + {{ ref('f2dq_columns')}} cols, {{ ref('f2dq_selected')}} tables +where + cols.name = tables.name and cols.schema = tables.schema and cols.database = tables.database +) + +select + cast ({{ dbt_utils.generate_surrogate_key([ + 'table_name', + 'column_name', + 'detected_time' + ]) }} as {{ string_type() }} ) as id, + table_name, + column_name, + data_type, + is_nullable, + detected_time +from columns \ No newline at end of file diff --git a/packages.yml b/packages.yml new file mode 100644 index 0000000..60c4d13 --- /dev/null +++ b/packages.yml @@ -0,0 +1,3 @@ +packages: + - package: dbt-labs/dbt_utils + version: 1.4.1 diff --git a/profiles.yml b/profiles.yml new file mode 100644 index 0000000..8555b9e --- /dev/null +++ b/profiles.yml @@ -0,0 +1,60 @@ +f2dq_postgres: + target: dev + outputs: + dev: + type: postgres + host: localhost + user: postgres + password: postgres + port: 5432 + dbname: postgres + schema: dq + threads: 4 +f2dq_snowflake: + target: dev + outputs: + dev: + type: snowflake + account: "{{ env_var('SNOWFLAKE_F2DQ_TESTING_ACCOUNT') }}" + user: "{{ env_var('F2DQ_TESTING_USER') }}" + password: "{{ env_var('F2DQ_TESTING_PASSWORD') }}" + database: F2DQ_TESTING + warehouse: F2DQ_TESTING_DWH + schema: "{{ env_var('DQ_SCHEMA') }}" + threads: 4 +f2dq_redshift: + target: dev + outputs: + dev: + type: redshift + host: "{{ env_var('REDSHIFT_F2DQ_TESTING_HOST') }}" + user: "{{ env_var('F2DQ_TESTING_USER') }}" + password: "{{ env_var('F2DQ_TESTING_PASSWORD') }}" + port: 5439 + dbname: f2dq_testing + schema: "{{ env_var('DQ_SCHEMA') }}" + threads: 4 +f2dq_bigquery: + target: dev + outputs: + dev: + type: bigquery + method: service-account-json + project: "{{ env_var('BIGQUERY_TESTING_PROJECT_ID') }}" + dataset: "{{ env_var('DQ_SCHEMA') }}" + threads: 4 + keyfile_json: + type: "{{ env_var('BIGQUERY_TESTING_TYPE') }}" + project_id: "{{ env_var('BIGQUERY_TESTING_PROJECT_ID') }}" + private_key_id: "{{ env_var('BIGQUERY_TESTING_PRIVATE_KEY_ID') }}" + private_key: "{{ env_var('BIGQUERY_TESTING_PRIVATE_KEY') }}" + client_email: "{{ env_var('BIGQUERY_TESTING_CLIENT_EMAIL') }}" + client_id: "{{ env_var('BIGQUERY_TESTING_CLIENT_ID') }}" + auth_uri: "{{ env_var('BIGQUERY_TESTING_AUTH_URI') }}" + token_uri: "{{ env_var('BIGQUERY_TESTING_TOKEN_URI') }}" + auth_provider_x509_cert_url: "{{ env_var('BIGQUERY_TESTING_AUTH_PROVIDER_X509_CERT_URL') }}" + client_x509_cert_url: "{{ env_var('BIGQUERY_TESTING_CLIENT_X509_CERT_URL') }}" + location: US + timeout_seconds: 300 + priority: interactive + retries: 1 \ No newline at end of file diff --git a/snapshots/.gitkeep b/snapshots/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/static/lineage_graph.png b/static/lineage_graph.png new file mode 100644 index 0000000..ee2b794 Binary files /dev/null and b/static/lineage_graph.png differ diff --git a/tests/.gitkeep b/tests/.gitkeep new file mode 100644 index 0000000..e69de29