Initial publish of dbt_f2dq (f2dq) package v0.12.1.

Clean package repo for Gitea installs via packages.yml git URL.

Co-authored-by: Cursor <cursoragent@cursor.com>

integration_tests/python_tests/conftest.py

@@ -0,0 +1,14 @@
+import pytest
+
+def pytest_addoption(parser):
+    parser.addoption("--db", action="store")
+    parser.addoption("--source_schema", action="store")
+    
+
+@pytest.fixture()
+def db(pytestconfig):
+    return pytestconfig.getoption("db")
+
+@pytest.fixture()
+def source_schema(pytestconfig):
+    return pytestconfig.getoption("source_schema")

integration_tests/python_tests/run_all_dbs.sh

@@ -0,0 +1,5 @@
+pytest --db postgres $@ & 
+pytest --db snowflake $@ &
+pytest --db bigquery $@ &
+pytest --db redshift $@ &
+wait 

integration_tests/python_tests/test_cleaners.py

@@ -0,0 +1,14 @@
+from .utils.run import dbt_seed, dbt_run, dbt_test
+
+def test_cleaners(db, source_schema, debug=True):
+    dbt_vars = {
+        'source_schema': source_schema
+    }
+
+    print (f"Running setup and tests for {db}")
+
+    dbt_seed(f'--select public_macros.cleaning', db, dbt_vars)
+    dbt_run(f'--select sanitized_user_data+', db, dbt_vars)
+    dbt_test(f'--select sanitized_user_data', db, dbt_vars)
+
+    print (f"Running tests completed for {db}")

integration_tests/python_tests/test_filters.py

@@ -0,0 +1,25 @@
+from .utils.run import dbt_seed, dbt_run, dbt_test
+
+def test_deduplication(db, source_schema, debug=True):
+    dbt_vars = {
+        'source_schema': source_schema
+    }
+
+    print (f"Running setup and tests for {db}")
+
+    dbt_seed(f'--select public_macros.filtering', db, dbt_vars)
+    dbt_run(f'--select deduplicated', db, dbt_vars)
+    dbt_test(f'--select deduplicated', db, dbt_vars)
+
+def test_get_duplicates(db, source_schema, debug=True):
+    dbt_vars = {
+        'source_schema': source_schema
+    }
+
+    print (f"Running setup and tests for {db}")
+
+    dbt_seed(f'--select public_macros.filtering', db, dbt_vars)
+    dbt_run(f'--select duplicates', db, dbt_vars)
+    dbt_test(f'--select duplicates', db, dbt_vars)
+
+    print (f"Running tests completed for {db}")

integration_tests/python_tests/test_monitoring.py

@@ -0,0 +1,88 @@
+import os
+import copy
+import yaml
+import json
+from datetime import datetime, timedelta
+from .utils.run import dbt_seed, dbt_run, dbt_test, dbt_command, dbt_build
+
+RUN_TIME = datetime(2021, 5, 2, 0, 0, 0)
+
+DBT_VARS = {
+    'f2dq:time_window_start': (RUN_TIME - timedelta(days=1)).strftime("%Y-%m-%d %H:%M:%S"),
+    'f2dq:time_window_end': RUN_TIME.strftime("%Y-%m-%d %H:%M:%S"),
+    'f2dq:save_test_history': True
+}
+
+def test_monitoring(db, source_schema):
+    DBT_VARS.update({'source_schema': source_schema})
+
+    load_deps = 'dbt deps'
+    assert os.system(load_deps) == 0
+
+    dbt_vars = copy.deepcopy(DBT_VARS)
+    
+    print (f"Running setup and tests for {db}")
+
+    dbt_seed('--select monitoring', db, dbt_vars)
+    dbt_run('--models transformed', db, dbt_vars)
+    dbt_command(
+        f'dbt run-operation create_test_source_tables',
+        db, dbt_vars
+    )
+
+    print (f"Computing f2dq metrics for {db}") 
+    dbt_run('--select package:f2dq', db, dbt_vars)
+
+    dbt_command(
+        f'dbt run-operation schema_change_buy_events_add_column',
+        db, dbt_vars
+    )
+
+    # update dbts_vars to run dbt for next day of data
+    dbt_vars['f2dq:time_window_start'] = dbt_vars['f2dq:time_window_end']
+    dbt_vars['f2dq:time_window_end'] = (RUN_TIME + timedelta(days=1)).strftime("%Y-%m-%d %H:%M:%S")
+
+    dbt_command(
+        'dbt run --select package:f2dq --fail-fast',
+        db, dbt_vars
+    )
+
+    dbt_command(
+        'dbt run --select monitoring.*', db, dbt_vars
+    )
+
+    dbt_test('--select test_f2dq_anomalies test_f2dq_metrics test_f2dq_z_score test_f2dq_table_samples f2dq_metrics transformed', db, dbt_vars)
+    # dbt build will "duplicate" saved test result history
+    dbt_build('--select test_f2dq_anomalies test_f2dq_metrics test_f2dq_z_score test_f2dq_table_samples f2dq_metrics transformed', db, dbt_vars)
+
+    # tests test_history seperately, because those are actually added to DB after running
+    # dbt test command
+    dbt_test('--select test_f2dq_test_history', db, dbt_vars)
+
+    op_vars = {
+        'start_date': RUN_TIME.strftime("%Y-%m-%d"),
+        'end_date': (RUN_TIME + timedelta(days=1)).strftime("%Y-%m-%d"),
+        'interval': 'days:1'
+    }
+    op_vars = yaml.dump(op_vars)
+    
+    dbt_command(
+        f'dbt run-operation generate_overview --args "{op_vars}"',
+        db, dbt_vars
+    )
+
+    overview = json.load(open(f'../target/f2dq/overview.json'))
+    expected_types = ['metric', 'schema_change', 'schema', 'alert', 'anomaly']
+    all_types = set()
+
+    # some simple check for now
+    for obj in overview:
+        all_types.add(obj['type'])
+        assert obj['table_name']
+        assert 'column_name' in obj
+        assert 'computed_on' in obj
+
+    assert len(overview) > 100
+    assert sorted(all_types) == sorted(expected_types)
+
+    print (f"Running tests completed for {db}")

integration_tests/python_tests/test_normalizers.py

@@ -0,0 +1,14 @@
+from .utils.run import dbt_seed, dbt_run, dbt_test
+
+def test_normalizers(db, source_schema, debug=True):
+    dbt_vars = {
+        'source_schema': source_schema
+    }
+
+    print (f"Running setup and tests for {db}")
+
+    dbt_seed(f'--select public_macros.normalizing', db, dbt_vars)
+    dbt_run(f'--select us_states_normalized+', db, dbt_vars)
+    dbt_test(f'--models us_states_normalized', db, dbt_vars)
+
+    print (f"Running tests completed for {db}")

integration_tests/python_tests/test_validate.py

@@ -0,0 +1,17 @@
+from .utils.run import dbt_seed, dbt_run, dbt_test
+
+def test_validate_regex(db, source_schema, debug=True):
+    dbt_vars = {
+        'source_schema': source_schema
+    }
+
+    print (f"Running setup and tests for {db}")
+
+    dbt_seed(
+        f'--select public_macros.validating', db, dbt_vars
+    )
+
+    dbt_run(f'--select public_macros.validating', db, dbt_vars)
+    dbt_test(f'--select public_macros.validating', db, dbt_vars)
+
+    print (f"Running tests completed for {db}")

integration_tests/python_tests/utils/run.py

@@ -0,0 +1,23 @@
+import os
+import yaml
+
+def dbt_command(command, for_db, dbt_vars, threads=None):
+    debug = 'DBT_MACRO_DEBUGGING=1 '
+    profile_part = f' --profile f2dq_{for_db}'
+    yaml_vars = yaml.dump(dbt_vars)
+    cmd = f'{debug} {command} --vars "{yaml_vars}" {profile_part}'
+    if threads:
+        cmd += f' --threads {threads}'
+    assert os.system(cmd) == 0
+
+def dbt_seed(args, for_db, dbt_vars):
+    dbt_command(f'dbt seed --full-refresh {args}', for_db, dbt_vars, threads=4)
+
+def dbt_run(args, for_db, dbt_vars):
+    dbt_command(f'dbt run --full-refresh --fail-fast {args}', for_db, dbt_vars, threads=4)
+
+def dbt_test(args, for_db, dbt_vars):
+    dbt_command(f'dbt test --store-failures --fail-fast {args}', for_db, dbt_vars, threads=4)
+
+def dbt_build(args, for_db, dbt_vars):
+    dbt_command(f'dbt build --full-refresh --store-failures --fail-fast {args}', for_db, dbt_vars, threads=4)