Initial publish of dbt_f2dq (f2dq) package v0.12.1.

Clean package repo for Gitea installs via packages.yml git URL. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-11 17:37:40 +03:00
commit ff096b51b5
188 changed files with 5251 additions and 0 deletions
@@ -0,0 +1,7 @@
+
+target/
+dbt_modules/
+dbt_packages/
+logs/
+.user.yml
+package-lock.yml
@@ -0,0 +1 @@
+f2dq integration tests — dbt project for running f2dq dbt-package integration tests
@@ -0,0 +1,116 @@
+name: "f2dq_integration_tests"
+version: "1.0"
+config-version: 2
+
+# Changed when running tests
+profile: "f2dq_postgres"
+
+target-path: "target"
+clean-targets: ["target", "dbt_modules", "dbt_packages"]
+
+models:
+  f2dq:
+    enabled: true
+
+sources:
+  f2dq_integration_tests:
+    +f2dq_monitored: true
+    +f2dq_time_filter: null
+  
+vars:
+  f2dq:store_table_samples: true
+  f2dq:anomaly_detector:
+    name: modified_z_score
+    threshold: 0.6
+  f2dq:max_columns_in_query: 1
+
+  f2dq:select:
+    - tag:testtag
+    - sample_table
+    - sample_without_time_filter
+    - sample_with_anomaly
+    - f2dq_source_test_table
+
+  f2dq:metrics_groups:
+    integration_test_group:
+      table:
+        - row_count
+        - freshness
+        - my_distinct_table_rows
+
+      column:
+        numeric:
+          - min
+          - max
+          - avg
+          - stddev
+          - variance
+          - nulls_count
+          - nulls_percent
+          - diff # my own custom metric
+
+        text:
+          - min_length
+          - max_length
+          - avg_length
+          - nulls_count
+          - nulls_percent
+          - missing_percent
+          - missing_count
+  
+  f2dq:default_metrics:
+    - integration_test_group
+
+seeds:
+  +schema: seeds
+  +quote_columns: false
+
+  f2dq_integration_tests:
+    monitoring:
+      sample_with_anomaly:
+        +f2dq_monitored: true
+        +f2dq_time_filter: creation_time
+
+      sample_without_time_filter:
+        +f2dq_monitored: true
+        +f2dq_time_filter: null
+
+      sample_table:
+        +f2dq_monitored: true
+        +f2dq_time_filter: creation_time
+        
+        +f2dq_columns: 
+          - event_type
+          - value1
+          - value2
+          - null_value
+        
+        +f2dq_metrics:
+          table:
+            - my_custom_table_metric # my own custom metric
+            - distinct_table_rows
+          column:
+            event_type:
+              - regex_test:
+                  regex: ([A-Za-z0-9]+)
+              - match_regex:
+                  regex: ^sell
+              - match_regex_percent:
+                  regex: ^sell
+              - not_match_regex:
+                  regex: ^buy
+              - not_match_regex_percent:
+                  regex: ^buy
+              - distinct_values
+              - duplicate_values
+              - duplicate_rows
+              - unique_rows
+
+      expected_z_score:
+        +column_types:
+          time_window_end: "TIMESTAMP"
+
+      expected_metrics:
+        +column_types:
+          time_window_start: "TIMESTAMP"
+          time_window_end: "TIMESTAMP"
@@ -0,0 +1,28 @@
+
+
+{% macro create_test_source_tables() %}
+
+    {% set create_table %}
+        CREATE SCHEMA IF NOT EXISTS {{target.schema}};
+        DROP TABLE IF EXISTS {{target.schema}}.f2dq_source_test_table;
+        CREATE TABLE IF NOT EXISTS {{target.schema}}.f2dq_source_test_table (
+            number {{ f2dq.integer_type() }},
+            description {{ f2dq.string_type() }},
+            created_at {{ f2dq.timestamp_type() }}
+        );
+        INSERT INTO {{target.schema}}.f2dq_source_test_table (number, description, created_at) VALUES 
+            (1, 'one', current_timestamp),
+            (2, 'two', current_timestamp),
+            (3, 'three', current_timestamp),
+            (4, 'four', current_timestamp),
+            (5, 'five', current_timestamp),
+            (6, 'six', current_timestamp),
+            (7, 'seven', current_timestamp),
+            (8, 'eight', current_timestamp),
+            (9, 'nine', current_timestamp),
+            (10, 'ten', current_timestamp
+        );
+    {% endset %}
+    {% do run_query(create_table) %}
+
+{% endmacro %}
@@ -0,0 +1,48 @@
+{% macro get_schemas_used(schema_name) %}
+    {% set schemas = [
+        schema_name,
+        schema_name + '_re',
+        schema_name + '_re_internal',
+        schema_name + '_raw',
+        schema_name + '_expected',
+        schema_name + '_dbt_test__audit',
+        schema_name + '_seeds'
+    ] %}
+    {{ return (schemas) }}
+{% endmacro %}
+
+{% macro drop_all_schemas(schema_name) %}
+    {% set schemas_to_drop = get_schemas_used(schema_name) %}
+    {{ adapter.dispatch('drop_all_schemas')(schemas_to_drop) }}
+{% endmacro %}
+
+{% macro default__drop_all_schemas(schemas_to_drop) %}
+    {% for schema in schemas_to_drop %}
+        {% set relation = api.Relation.create(database=target.database, schema=schema) %}
+        {% do adapter.drop_schema(relation) %}
+    {% endfor %}
+{% endmacro %}
+
+{% macro redshift__drop_all_schemas(schemas_to_drop) %}
+    {# 
+        dropping schemas with adapter.drop_schema doesn't seem to work with redshift
+        so we default to issuing DDL commands to redshift
+    #}
+    {% set drop_query %}
+        {% for schema in schemas_to_drop %}
+            drop schema if exists {{schema}} cascade;
+        {% endfor %}
+    {% endset %}
+    {% do run_query(drop_query) %}
+{% endmacro %}
+
+{% macro create_required_schemas(schema_name) %}
+    {# required to manually create schemas used for redshift tests #}
+    {% set schemas_to_drop = get_schemas_used(schema_name) %}
+    {% set create_query %}
+        {% for schema in schemas_to_drop %}
+            create schema if not exists {{schema}};
+        {% endfor %}
+    {% endset %}
+    {% do run_query(create_query) %}
+{% endmacro %}
@@ -0,0 +1,29 @@
+{% macro f2dq_metric_diff(context) %}
+    max({{context.column_name}}) - min({{context.column_name}})
+{% endmacro %}
+
+{% macro f2dq_metric_my_custom_table_metric(context) %}
+    1000
+{% endmacro %}
+
+{% macro f2dq_metric_regex_test(context) %}
+    {{ regex_test(context.column_name, context.config.regex) }}
+{% endmacro %}
+
+{% macro regex_test(column_name, pattern) %}
+    coalesce(
+        sum(
+            case when {{ f2dq.regex_match_expression(column_name, pattern) }}
+                then 1
+            else 0
+            end
+        ), 0
+    )
+{% endmacro %}
+
+{% macro f2dq_metric_my_distinct_table_rows(context) %}
+    with temp_table AS (
+            select distinct * from {{ context.table_name }}
+        )
+    select coalesce(count(*), 0) FROM temp_table
+{% endmacro %}
@@ -0,0 +1,20 @@
+
+{% macro to_big_integer(field) %}
+    cast (round({{field}} * 1000) as integer) as {{field}}
+{% endmacro %}
+
+{% macro clean_table_name(field) %}
+    upper(
+        {{-
+            f2dq.clean_blacklist(
+                f2dq.split_and_return_nth_value(field, '.', 3),
+                ['"', '`'],
+                ''
+            )
+        -}}
+    )
+{% endmacro %}
+
+{% macro clean_column_name(field) %}
+    case when ({{ field }} = '' or  {{ field }} is null ) then '---' else  upper({{field}}) end
+{% endmacro %}
@@ -0,0 +1,15 @@
+
+{% macro schema_change_buy_events_add_column() %}
+    {% set alter_table %}
+        alter table {{ ref('buy_events')}} add column sample_column boolean
+    {% endset %}
+    {% do run_query(alter_table) %}
+{% endmacro %}
+
+
+{% macro schema_change_buy_events_drop_column() %}
+    {% set alter_table %}
+        alter table {{ ref('buy_events')}} drop column sample_column
+    {% endset %}
+    {% do run_query(alter_table) %}
+{% endmacro %}
@@ -0,0 +1,30 @@
+
+version: 2
+
+models:
+  - name: f2dq_metrics
+    tests:
+      - f2dq.metric_expression_is_true:
+          table: ref('buy_events')
+          metric: max_length
+          column_name: event_type
+          expression: value = 3
+
+      - f2dq.metric_equal_to:
+          table: ref('buy_events')
+          metric: max_length
+          column_name: event_type
+          value: 3
+
+      - f2dq.metric_in_range:
+          table: ref('buy_events')
+          metric: max_length
+          column_name: event_type
+          min_value: 3
+          max_value: 3
+
+      - f2dq.metric_expression_is_true:
+          table: ref('buy_events')
+          metric: row_count
+          expression: value > 0 and value < 10
+          condition: time_window_start >= '2021-05-02'
@@ -0,0 +1,27 @@
+version: 2
+
+models:
+  - name: test_f2dq_metrics
+    tests:
+      - dbt_utils.equality:
+          compare_model: ref('expected_metrics')
+
+  - name: test_f2dq_z_score
+    tests:
+      - dbt_utils.equality:
+          compare_model: ref('expected_z_score')
+
+  - name: test_f2dq_anomalies
+    tests:
+      - dbt_utils.equality:
+          compare_model: ref('expected_anomalies')
+
+  - name: test_f2dq_test_history
+    tests:
+     - dbt_utils.equality:
+         compare_model: ref('expected_test_history')
+
+  - name: test_f2dq_table_samples
+    tests:
+     - dbt_utils.equality:
+         compare_model: ref('expected_table_samples')
@@ -0,0 +1,9 @@
+
+select
+    {{ clean_table_name('table_name') }} as table_name,
+    {{ clean_column_name('column_name') }} as column_name,
+    metric,
+    anomaly_detector,
+    interval_length_sec
+
+from {{ ref('f2dq_anomalies') }}
@@ -0,0 +1,11 @@
+
+select
+    {{ clean_table_name('table_name') }} as table_name,
+    {{ clean_column_name('column_name') }} as column_name,
+    metric,
+    time_window_start,
+    time_window_end,
+    {{ to_big_integer('value') }},
+    interval_length_sec
+
+from {{ ref('f2dq_metrics') }}
@@ -0,0 +1,9 @@
+
+select 
+    {{ clean_table_name('table_name') }} as table_name,
+    length(sample_data) as sample_data_length
+from {{ ref('f2dq_table_samples') }}
+where {{ clean_table_name('table_name') }} != 'SAMPLE_WITHOUT_TIME_FILTER'
+
+-- SAMPLE_WITHOUT_TIME_FILTER because this table doesn't have a time filter, it's not possible to say how
+-- exactly the sampel of it should look like.
@@ -0,0 +1,10 @@
+
+select
+    {{ clean_table_name('table_name') }} as table_name,
+    {{ clean_column_name('column_name') }} as column_name,
+    right(test_name, 15) as test_name,
+    status,
+    {{ clean_column_name('message') }} as message,
+    cast (failures_count as integer) as failures_count,
+    severity
+from {{ ref('f2dq_test_history') }}
@@ -0,0 +1,22 @@
+
+{% set values_compare = [
+    'z_score_value',
+    'modified_z_score_value', 
+    'last_value',
+    'last_avg',
+    'last_stddev',
+    'last_median',
+    'last_iqr',
+    'last_median_absolute_deviation',
+    'last_mean_absolute_deviation',
+] %}
+
+select
+    {{ clean_table_name('table_name') }} as table_name,
+    {{ clean_column_name('column_name') }} as column_name,
+    metric,
+    time_window_end,
+    {% for col in values_compare %}{{ to_big_integer(col) }},{% endfor %}
+    interval_length_sec
+
+from {{ ref('f2dq_z_score') }}
@@ -0,0 +1,4 @@
+select
+    {{ f2dq.clean_capitalize_words(f2dq.clean_additional_whitespaces('full_name')) }} as full_name,
+    {{ f2dq.clean_blacklist('email', ['^[a-zA-Z0-9_.+-]+'], '*****') }} as email
+from {{ ref('sample_user_data') }}
@@ -0,0 +1,7 @@
+version: 2
+
+models:
+  - name: sanitized_user_data
+    tests:
+      - dbt_utils.equality:
+          compare_model: ref('expected_sample_user_data')
@@ -0,0 +1,16 @@
+with x as 
+    {{ f2dq.filter_remove_duplicates(
+        ref('duplicated'), ['transaction_id'], ['creation_time']) }}
+
+select *, 'take_first' as use_case from x
+
+union all
+
+select *, 'take_last' as use_case from {{ f2dq.filter_remove_duplicates(
+        ref('duplicated'), ['transaction_id'], ['creation_time desc']) }} duplicates
+
+
+union all
+
+select *, 'take_all_statuses' as use_case from {{ f2dq.filter_remove_duplicates(
+    ref('duplicated'), ['transaction_id', 'status'], ['creation_time desc']) }} duplicates
@@ -0,0 +1,2 @@
+ {{ f2dq.filter_get_duplicates(
+        ref('duplicated'), ['transaction_id'], ['creation_time']) }}
@@ -0,0 +1,11 @@
+version: 2
+
+models:
+  - name: deduplicated
+    tests:
+      - dbt_utils.equality:
+          compare_model: ref('expected_deduplicated')
+  - name: duplicates
+    tests:
+      - dbt_utils.equality:
+          compare_model: ref('expected_duplicates')
@@ -0,0 +1,7 @@
+version: 2
+
+models:
+  - name: us_states_normalized
+    tests:
+      - dbt_utils.equality:
+          compare_model: ref('expected_us_states_normalized')
@@ -0,0 +1,29 @@
+with us_states_normalization_cte as (
+    select source, target from {{ ref('us_states_normalization') }}
+)
+
+{% set us_states_mapping = {'Ala.': 'Alabama', 'Alaska': 'Alaska', 'Ariz.': 'Arizona', 'Ark.': 'Arkansas', 'Calif.': 'California', 'Colo.': 'Colorado', 'Conn.': 'Connecticut',
+    'Del.': 'Delaware', 'D.C.': 'District of Columbia', 'Fla.': 'Florida', 'Ga.': 'Georgia', 'Hawaii': 'Hawaii', 'Idaho': 'Idaho', 'Ill.': 'Illinois', 'Ind.': 'Indiana',
+    'Iowa': 'Iowa', 'Kans.': 'Kansas', 'Ky.': 'Kentucky', 'La.': 'Louisiana', 'Maine': 'Maine', 'Md.': 'Maryland', 'Mass.': 'Massachusetts', 'Mich.': 'Michigan',
+    'Minn.': 'Minnesota', 'Miss.': 'Mississippi', 'Mo.': 'Missouri', 'Mont.': 'Montana', 'Nebr.': 'Nebraska', 'Nev.': 'Nevada', 'N.H.': 'New Hampshire', 'N.J.': 'New Jersey',
+    'N.M.': 'New Mexico', 'N.Y.': 'New York', 'N.C.': 'North Carolina', 'N.D.': 'North Dakota', 'Ohio': 'Ohio', 'Okla.': 'Oklahoma', 'Ore.': 'Oregon', 'Pa.': 'Pennsylvania',
+    'R.I.': 'Rhode Island', 'S.C.': 'South Carolina', 'S.D.': 'South Dakota', 'Tenn.': 'Tennessee', 'Tex.': 'Texas', 'Utah': 'Utah', 'Vt.': 'Vermont', 'Va.': 'Virginia',
+    'Wash.': 'Washington', 'W.Va.': 'West Virginia', 'Wis.': 'Wisconsin', 'Wyo.': 'Wyoming'}
+%}
+
+
+{# 
+    We have three ways of passing the source used for normalization
+        1. passing a dbt model using ref('') which is of type Relation.
+        2. passing a common table expression that contains the source mapping
+            Note: model or cte must include "source" and "target" column names used for normalization in 1. & 2. repectively
+        3. passing a dictionary of values that map from source -> target ie {[source]: [target]}
+ #}
+
+select distinct * from (
+    select state, code, state__normalized from {{ f2dq.normalize_values(ref('abbreviated_us_states'), 'state', ref('us_states_normalization')) }} s
+    union all
+    select state, code, state__normalized from {{ f2dq.normalize_values(ref('abbreviated_us_states'), 'state', 'us_states_normalization_cte') }} s
+    union all
+    select state, code, state__normalized from {{ f2dq.normalize_values(ref('abbreviated_us_states'), 'state', us_states_mapping) }} s
+) as normalized
@@ -0,0 +1,27 @@
+version: 2
+
+models:
+  - name: validated_emails
+    tests:
+      - dbt_utils.equality:
+          compare_model: ref('expected_validated_emails')
+
+  - name: validated_numbers
+    tests:
+      - dbt_utils.equality:
+          compare_model: ref('expected_validated_numbers')
+
+  - name: validated_date_and_time
+    tests:
+      - dbt_utils.equality:
+          compare_model: ref('expected_validated_date_and_time')
+
+  - name: validated_ips
+    tests:
+      - dbt_utils.equality:
+          compare_model: ref('expected_validated_ips')
+  
+  - name: validated_uuids
+    tests:
+      - dbt_utils.equality:
+          compare_model: ref('expected_validated_uuids')
@@ -0,0 +1,14 @@
+with 
+    all_rows as (
+        select * from {{ ref('validate_date_and_time') }}
+    )
+
+select *, 
+    case when {{ f2dq.valid_date_eu('date_time') }} then 1 else 0 end as valid_date_eu,
+    case when {{ f2dq.valid_date_us('date_time') }} then 1 else 0 end as valid_date_us,
+    case when {{ f2dq.valid_date_inverse('date_time') }} then 1 else 0 end as valid_date_inverse,
+    case when {{ f2dq.valid_date_iso_8601('date_time') }} then 1 else 0 end as valid_date_iso_8601,
+    case when {{ f2dq.valid_time_24h('date_time') }} then 1 else 0 end as valid_time_24h,
+    case when {{ f2dq.valid_time_12h('date_time') }} then 1 else 0 end as valid_time_12h,
+    case when {{ f2dq.valid_time('date_time') }} then 1 else 0 end as valid_time
+    from all_rows 
@@ -0,0 +1,7 @@
+with 
+    all_emails as (
+        select * from {{ ref('validate_emails') }}
+    )
+
+select *, case when {{ f2dq.valid_email('email') }} then 1 else 0 end as email_valid
+    from all_emails 
@@ -0,0 +1,10 @@
+with 
+    all_rows as (
+        select * from {{ ref('validate_ip') }}
+    )
+
+select *, 
+    case when {{ f2dq.valid_ip_v4('ip_address') }} then 1 else 0 end as valid_ip_v4,
+    case when {{ f2dq.valid_ip_v6('ip_address') }} then 1 else 0 end as valid_ip_v6,
+    case when {{ f2dq.valid_ip('ip_address') }} then 1 else 0 end as valid_ip
+    from all_rows 
@@ -0,0 +1,13 @@
+with 
+    all_num_rows as (
+        select * from {{ ref('validate_numbers') }}
+    )
+
+select *, 
+    case when {{ f2dq.valid_number('number') }} then 1 else 0 end as is_number,
+    case when {{ f2dq.valid_number_decimal_point('number') }} then 1 else 0 end as is_number_decimal_point,
+    case when {{ f2dq.valid_number_decimal_comma('number') }} then 1 else 0 end as is_number_decimal_comma,
+    case when {{ f2dq.valid_number_percentage('number') }} then 1 else 0 end as is_percentage,
+    case when {{ f2dq.valid_number_percentage_point('number') }} then 1 else 0 end as is_percentage_decimal_point,
+    case when {{ f2dq.valid_number_percentage_comma('number') }} then 1 else 0 end as is_percentage_decimal_comma
+    from all_num_rows 
@@ -0,0 +1,8 @@
+with 
+    all_rows as (
+        select * from {{ ref('validate_uuid') }}
+    )
+
+select *, 
+    case when {{ f2dq.valid_uuid('uuid') }} then 1 else 0 end as valid_uuid
+    from all_rows 
@@ -0,0 +1,18 @@
+version: 2
+
+sources:
+  - name:  "{{ var('source_schema', target.schema) }}"
+    tables:
+      - name: f2dq_source_test_table
+        columns:
+          - name: number
+            tests:
+              - not_null
+              - unique
+          
+          - name: description
+            tests:
+              - not_null
+              - unique
+          
+      
@@ -0,0 +1,12 @@
+{{
+    config(
+        f2dq_monitored=true,
+        f2dq_time_filter='creation_time',
+        f2dq_anomaly_detector={'name': 'z_score', 'threshold': 0.5},
+        materialized='table',
+        tags=['testtag']
+    )
+}}
+select *
+from {{ ref('sample_with_anomaly') }}
+where event_type = 'buy'
@@ -0,0 +1,49 @@
+version: 2
+
+models:
+  - name: buy_events
+    tests:
+      - f2dq.assert_in_range:
+          metric: row_count
+          min_value: 0
+          max_value: 10
+
+      - f2dq.assert_equal:
+          metric: row_count
+          value: 1
+
+      - f2dq.assert_false:
+          metric: freshness
+          expression: value is null
+      
+      - f2dq.assert_greater_equal:
+          metric: my_distinct_table_rows
+          value: 10
+
+    columns:
+      - name: value1
+        tests:
+          - f2dq.assert_in_range:
+              metric: nulls_percent
+              min_value: 0
+              max_value: 10
+
+          - f2dq.assert_true:
+              metric: nulls_percent
+              expression: value = 0
+          
+          - f2dq.assert_less:
+              metric: min
+              value: 100
+              condition: time_window_start = '2021-05-02'
+          
+          - f2dq.assert_less_equal:
+              metric: min
+              value: 107
+
+      - name: value2
+        tests:
+          - f2dq.assert_greater_equal:
+              metric: min
+              value: 200
+              condition: time_window_start = '2021-05-02'
@@ -0,0 +1,5 @@
+packages:
+  - local: ../
+
+  - package: dbt-labs/dbt_utils
+    version: [">=1.0.0", "<1.2.0"]
@@ -0,0 +1,5 @@
+[pytest]
+norecursedirs = dbt_modules models target logs data analysis macros snapshots tests
+python_files = test_*.py
+python_functions = test*
+addopts = --capture=no --durations=0
@@ -0,0 +1,14 @@
+import pytest
+
+def pytest_addoption(parser):
+    parser.addoption("--db", action="store")
+    parser.addoption("--source_schema", action="store")
+    
+
+@pytest.fixture()
+def db(pytestconfig):
+    return pytestconfig.getoption("db")
+
+@pytest.fixture()
+def source_schema(pytestconfig):
+    return pytestconfig.getoption("source_schema")
@@ -0,0 +1,5 @@
+pytest --db postgres $@ & 
+pytest --db snowflake $@ &
+pytest --db bigquery $@ &
+pytest --db redshift $@ &
+wait 
@@ -0,0 +1,14 @@
+from .utils.run import dbt_seed, dbt_run, dbt_test
+
+def test_cleaners(db, source_schema, debug=True):
+    dbt_vars = {
+        'source_schema': source_schema
+    }
+
+    print (f"Running setup and tests for {db}")
+
+    dbt_seed(f'--select public_macros.cleaning', db, dbt_vars)
+    dbt_run(f'--select sanitized_user_data+', db, dbt_vars)
+    dbt_test(f'--select sanitized_user_data', db, dbt_vars)
+
+    print (f"Running tests completed for {db}")
@@ -0,0 +1,25 @@
+from .utils.run import dbt_seed, dbt_run, dbt_test
+
+def test_deduplication(db, source_schema, debug=True):
+    dbt_vars = {
+        'source_schema': source_schema
+    }
+
+    print (f"Running setup and tests for {db}")
+
+    dbt_seed(f'--select public_macros.filtering', db, dbt_vars)
+    dbt_run(f'--select deduplicated', db, dbt_vars)
+    dbt_test(f'--select deduplicated', db, dbt_vars)
+
+def test_get_duplicates(db, source_schema, debug=True):
+    dbt_vars = {
+        'source_schema': source_schema
+    }
+
+    print (f"Running setup and tests for {db}")
+
+    dbt_seed(f'--select public_macros.filtering', db, dbt_vars)
+    dbt_run(f'--select duplicates', db, dbt_vars)
+    dbt_test(f'--select duplicates', db, dbt_vars)
+
+    print (f"Running tests completed for {db}")
@@ -0,0 +1,88 @@
+import os
+import copy
+import yaml
+import json
+from datetime import datetime, timedelta
+from .utils.run import dbt_seed, dbt_run, dbt_test, dbt_command, dbt_build
+
+RUN_TIME = datetime(2021, 5, 2, 0, 0, 0)
+
+DBT_VARS = {
+    'f2dq:time_window_start': (RUN_TIME - timedelta(days=1)).strftime("%Y-%m-%d %H:%M:%S"),
+    'f2dq:time_window_end': RUN_TIME.strftime("%Y-%m-%d %H:%M:%S"),
+    'f2dq:save_test_history': True
+}
+
+def test_monitoring(db, source_schema):
+    DBT_VARS.update({'source_schema': source_schema})
+
+    load_deps = 'dbt deps'
+    assert os.system(load_deps) == 0
+
+    dbt_vars = copy.deepcopy(DBT_VARS)
+    
+    print (f"Running setup and tests for {db}")
+
+    dbt_seed('--select monitoring', db, dbt_vars)
+    dbt_run('--models transformed', db, dbt_vars)
+    dbt_command(
+        f'dbt run-operation create_test_source_tables',
+        db, dbt_vars
+    )
+
+    print (f"Computing f2dq metrics for {db}") 
+    dbt_run('--select package:f2dq', db, dbt_vars)
+
+    dbt_command(
+        f'dbt run-operation schema_change_buy_events_add_column',
+        db, dbt_vars
+    )
+
+    # update dbts_vars to run dbt for next day of data
+    dbt_vars['f2dq:time_window_start'] = dbt_vars['f2dq:time_window_end']
+    dbt_vars['f2dq:time_window_end'] = (RUN_TIME + timedelta(days=1)).strftime("%Y-%m-%d %H:%M:%S")
+
+    dbt_command(
+        'dbt run --select package:f2dq --fail-fast',
+        db, dbt_vars
+    )
+
+    dbt_command(
+        'dbt run --select monitoring.*', db, dbt_vars
+    )
+
+    dbt_test('--select test_f2dq_anomalies test_f2dq_metrics test_f2dq_z_score test_f2dq_table_samples f2dq_metrics transformed', db, dbt_vars)
+    # dbt build will "duplicate" saved test result history
+    dbt_build('--select test_f2dq_anomalies test_f2dq_metrics test_f2dq_z_score test_f2dq_table_samples f2dq_metrics transformed', db, dbt_vars)
+
+    # tests test_history seperately, because those are actually added to DB after running
+    # dbt test command
+    dbt_test('--select test_f2dq_test_history', db, dbt_vars)
+
+    op_vars = {
+        'start_date': RUN_TIME.strftime("%Y-%m-%d"),
+        'end_date': (RUN_TIME + timedelta(days=1)).strftime("%Y-%m-%d"),
+        'interval': 'days:1'
+    }
+    op_vars = yaml.dump(op_vars)
+    
+    dbt_command(
+        f'dbt run-operation generate_overview --args "{op_vars}"',
+        db, dbt_vars
+    )
+
+    overview = json.load(open(f'../target/f2dq/overview.json'))
+    expected_types = ['metric', 'schema_change', 'schema', 'alert', 'anomaly']
+    all_types = set()
+
+    # some simple check for now
+    for obj in overview:
+        all_types.add(obj['type'])
+        assert obj['table_name']
+        assert 'column_name' in obj
+        assert 'computed_on' in obj
+
+    assert len(overview) > 100
+    assert sorted(all_types) == sorted(expected_types)
+
+    print (f"Running tests completed for {db}")
@@ -0,0 +1,14 @@
+from .utils.run import dbt_seed, dbt_run, dbt_test
+
+def test_normalizers(db, source_schema, debug=True):
+    dbt_vars = {
+        'source_schema': source_schema
+    }
+
+    print (f"Running setup and tests for {db}")
+
+    dbt_seed(f'--select public_macros.normalizing', db, dbt_vars)
+    dbt_run(f'--select us_states_normalized+', db, dbt_vars)
+    dbt_test(f'--models us_states_normalized', db, dbt_vars)
+
+    print (f"Running tests completed for {db}")
@@ -0,0 +1,17 @@
+from .utils.run import dbt_seed, dbt_run, dbt_test
+
+def test_validate_regex(db, source_schema, debug=True):
+    dbt_vars = {
+        'source_schema': source_schema
+    }
+
+    print (f"Running setup and tests for {db}")
+
+    dbt_seed(
+        f'--select public_macros.validating', db, dbt_vars
+    )
+
+    dbt_run(f'--select public_macros.validating', db, dbt_vars)
+    dbt_test(f'--select public_macros.validating', db, dbt_vars)
+
+    print (f"Running tests completed for {db}")
@@ -0,0 +1,23 @@
+import os
+import yaml
+
+def dbt_command(command, for_db, dbt_vars, threads=None):
+    debug = 'DBT_MACRO_DEBUGGING=1 '
+    profile_part = f' --profile f2dq_{for_db}'
+    yaml_vars = yaml.dump(dbt_vars)
+    cmd = f'{debug} {command} --vars "{yaml_vars}" {profile_part}'
+    if threads:
+        cmd += f' --threads {threads}'
+    assert os.system(cmd) == 0
+
+def dbt_seed(args, for_db, dbt_vars):
+    dbt_command(f'dbt seed --full-refresh {args}', for_db, dbt_vars, threads=4)
+
+def dbt_run(args, for_db, dbt_vars):
+    dbt_command(f'dbt run --full-refresh --fail-fast {args}', for_db, dbt_vars, threads=4)
+
+def dbt_test(args, for_db, dbt_vars):
+    dbt_command(f'dbt test --store-failures --fail-fast {args}', for_db, dbt_vars, threads=4)
+
+def dbt_build(args, for_db, dbt_vars):
+    dbt_command(f'dbt build --full-refresh --store-failures --fail-fast {args}', for_db, dbt_vars, threads=4)
@@ -0,0 +1,3 @@
+protobuf==4.25.3
+pytest==6.2.5
+pyyaml==6.0
@@ -0,0 +1,29 @@
+table_name,column_name,metric,anomaly_detector,interval_length_sec
+BUY_EVENTS,VALUE2,min,"{""name"": ""z_score"", ""threshold"": 0.5}",86400
+BUY_EVENTS,VALUE2,avg,"{""name"": ""z_score"", ""threshold"": 0.5}",86400
+BUY_EVENTS,---,freshness,"{""name"": ""z_score"", ""threshold"": 0.5}",86400
+BUY_EVENTS,VALUE1,min,"{""name"": ""z_score"", ""threshold"": 0.5}",86400
+BUY_EVENTS,VALUE1,max,"{""name"": ""z_score"", ""threshold"": 0.5}",86400
+BUY_EVENTS,VALUE1,avg,"{""name"": ""z_score"", ""threshold"": 0.5}",86400
+BUY_EVENTS,VALUE2,max,"{""name"": ""z_score"", ""threshold"": 0.5}",86400
+SAMPLE_TABLE,VALUE1,avg,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400
+SAMPLE_TABLE,EVENT_TYPE,unique_rows,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400
+SAMPLE_TABLE,EVENT_TYPE,duplicate_rows,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400
+SAMPLE_TABLE,VALUE1,max,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400
+SAMPLE_TABLE,EVENT_TYPE,match_regex_percent,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400
+SAMPLE_TABLE,VALUE1,diff,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400
+SAMPLE_TABLE,VALUE1,stddev,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400
+SAMPLE_TABLE,EVENT_TYPE,not_match_regex,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400
+SAMPLE_TABLE,EVENT_TYPE,avg_length,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400
+SAMPLE_TABLE,EVENT_TYPE,not_match_regex_percent,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400
+SAMPLE_TABLE,EVENT_TYPE,match_regex,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400
+SAMPLE_TABLE,VALUE1,variance,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400
+SAMPLE_TABLE,EVENT_TYPE,max_length,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400
+SAMPLE_TABLE,EVENT_TYPE,distinct_values,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400
+SAMPLE_WITH_ANOMALY,---,freshness,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400
+SAMPLE_WITH_ANOMALY,VALUE2,avg,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400
+SAMPLE_WITH_ANOMALY,VALUE1,max,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400
+SAMPLE_WITH_ANOMALY,VALUE2,min,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400
+SAMPLE_WITH_ANOMALY,VALUE1,avg,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400
+SAMPLE_WITH_ANOMALY,VALUE2,max,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400
+SAMPLE_WITH_ANOMALY,VALUE1,min,"{""name"": ""modified_z_score"", ""threshold"": 0.6}",86400
@@ -0,0 +1,281 @@
+table_name,column_name,metric,time_window_start,time_window_end,value,interval_length_sec
+SAMPLE_WITH_ANOMALY,VALUE1,min,2021-05-01 00:00:00,2021-05-02 00:00:00,107000,86400
+SAMPLE_WITH_ANOMALY,VALUE1,max,2021-05-01 00:00:00,2021-05-02 00:00:00,107000,86400
+SAMPLE_WITH_ANOMALY,VALUE1,avg,2021-05-01 00:00:00,2021-05-02 00:00:00,107000,86400
+SAMPLE_WITH_ANOMALY,VALUE1,stddev,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400
+SAMPLE_WITH_ANOMALY,VALUE1,variance,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400
+SAMPLE_WITH_ANOMALY,VALUE1,nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_WITH_ANOMALY,VALUE1,nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_WITH_ANOMALY,VALUE1,diff,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_WITH_ANOMALY,VALUE2,min,2021-05-01 00:00:00,2021-05-02 00:00:00,200000,86400
+SAMPLE_WITH_ANOMALY,VALUE2,max,2021-05-01 00:00:00,2021-05-02 00:00:00,200000,86400
+SAMPLE_WITH_ANOMALY,VALUE2,avg,2021-05-01 00:00:00,2021-05-02 00:00:00,200000,86400
+SAMPLE_WITH_ANOMALY,VALUE2,stddev,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400
+SAMPLE_WITH_ANOMALY,VALUE2,variance,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400
+SAMPLE_WITH_ANOMALY,VALUE2,nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_WITH_ANOMALY,VALUE2,nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_WITH_ANOMALY,VALUE2,diff,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_WITH_ANOMALY,EVENT_TYPE,min_length,2021-05-01 00:00:00,2021-05-02 00:00:00,3000,86400
+SAMPLE_WITH_ANOMALY,EVENT_TYPE,max_length,2021-05-01 00:00:00,2021-05-02 00:00:00,3000,86400
+SAMPLE_WITH_ANOMALY,EVENT_TYPE,avg_length,2021-05-01 00:00:00,2021-05-02 00:00:00,3000,86400
+SAMPLE_WITH_ANOMALY,EVENT_TYPE,nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_WITH_ANOMALY,EVENT_TYPE,nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_WITH_ANOMALY,EVENT_TYPE,missing_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_WITH_ANOMALY,EVENT_TYPE,missing_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_WITH_ANOMALY,---,row_count,2021-05-01 00:00:00,2021-05-02 00:00:00,1000,86400
+SAMPLE_WITH_ANOMALY,---,freshness,2021-05-01 00:00:00,2021-05-02 00:00:00,41308000,86400
+SAMPLE_WITH_ANOMALY,---,my_distinct_table_rows,2021-05-01 00:00:00,2021-05-02 00:00:00,12000,86400
+BUY_EVENTS,VALUE1,min,2021-05-01 00:00:00,2021-05-02 00:00:00,107000,86400
+BUY_EVENTS,VALUE1,max,2021-05-01 00:00:00,2021-05-02 00:00:00,107000,86400
+BUY_EVENTS,VALUE1,avg,2021-05-01 00:00:00,2021-05-02 00:00:00,107000,86400
+BUY_EVENTS,VALUE1,stddev,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400
+BUY_EVENTS,VALUE1,variance,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400
+BUY_EVENTS,VALUE1,nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+BUY_EVENTS,VALUE1,nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+BUY_EVENTS,VALUE1,diff,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+BUY_EVENTS,VALUE2,min,2021-05-01 00:00:00,2021-05-02 00:00:00,200000,86400
+BUY_EVENTS,VALUE2,max,2021-05-01 00:00:00,2021-05-02 00:00:00,200000,86400
+BUY_EVENTS,VALUE2,avg,2021-05-01 00:00:00,2021-05-02 00:00:00,200000,86400
+BUY_EVENTS,VALUE2,stddev,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400
+BUY_EVENTS,VALUE2,variance,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400
+BUY_EVENTS,VALUE2,nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+BUY_EVENTS,VALUE2,nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+BUY_EVENTS,VALUE2,diff,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+BUY_EVENTS,EVENT_TYPE,min_length,2021-05-01 00:00:00,2021-05-02 00:00:00,3000,86400
+BUY_EVENTS,EVENT_TYPE,max_length,2021-05-01 00:00:00,2021-05-02 00:00:00,3000,86400
+BUY_EVENTS,EVENT_TYPE,avg_length,2021-05-01 00:00:00,2021-05-02 00:00:00,3000,86400
+BUY_EVENTS,EVENT_TYPE,nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+BUY_EVENTS,EVENT_TYPE,nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+BUY_EVENTS,EVENT_TYPE,missing_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+BUY_EVENTS,EVENT_TYPE,missing_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+BUY_EVENTS,---,row_count,2021-05-01 00:00:00,2021-05-02 00:00:00,1000,86400
+BUY_EVENTS,---,freshness,2021-05-01 00:00:00,2021-05-02 00:00:00,41308000,86400
+BUY_EVENTS,---,my_distinct_table_rows,2021-05-01 00:00:00,2021-05-02 00:00:00,10000,86400
+SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__min,2021-05-01 00:00:00,2021-05-02 00:00:00,990,86400
+SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__max,2021-05-01 00:00:00,2021-05-02 00:00:00,4990,86400
+SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__avg,2021-05-01 00:00:00,2021-05-02 00:00:00,3306,86400
+SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__stddev,2021-05-01 00:00:00,2021-05-02 00:00:00,1668,86400
+SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__variance,2021-05-01 00:00:00,2021-05-02 00:00:00,2784,86400
+SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__diff,2021-05-01 00:00:00,2021-05-02 00:00:00,4000,86400
+SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__min_length,2021-05-01 00:00:00,2021-05-02 00:00:00,10000,86400
+SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__max_length,2021-05-01 00:00:00,2021-05-02 00:00:00,17000,86400
+SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__avg_length,2021-05-01 00:00:00,2021-05-02 00:00:00,14105,86400
+SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__missing_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__missing_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,RATING,global__min_length,2021-05-01 00:00:00,2021-05-02 00:00:00,1000,86400
+SAMPLE_WITHOUT_TIME_FILTER,RATING,global__max_length,2021-05-01 00:00:00,2021-05-02 00:00:00,5000,86400
+SAMPLE_WITHOUT_TIME_FILTER,RATING,global__avg_length,2021-05-01 00:00:00,2021-05-02 00:00:00,3053,86400
+SAMPLE_WITHOUT_TIME_FILTER,RATING,global__nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,RATING,global__nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,RATING,global__missing_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,RATING,global__missing_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,---,global__row_count,2021-05-01 00:00:00,2021-05-02 00:00:00,19000,86400
+SAMPLE_WITHOUT_TIME_FILTER,---,global__freshness,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400
+SAMPLE_WITHOUT_TIME_FILTER,---,global__my_distinct_table_rows,2021-05-01 00:00:00,2021-05-02 00:00:00,19000,86400
+F2DQ_SOURCE_TEST_TABLE,NUMBER,global__min,2021-05-01 00:00:00,2021-05-02 00:00:00,1000,86400
+F2DQ_SOURCE_TEST_TABLE,NUMBER,global__max,2021-05-01 00:00:00,2021-05-02 00:00:00,10000,86400
+F2DQ_SOURCE_TEST_TABLE,NUMBER,global__avg,2021-05-01 00:00:00,2021-05-02 00:00:00,5500,86400
+F2DQ_SOURCE_TEST_TABLE,NUMBER,global__stddev,2021-05-01 00:00:00,2021-05-02 00:00:00,3028,86400
+F2DQ_SOURCE_TEST_TABLE,NUMBER,global__variance,2021-05-01 00:00:00,2021-05-02 00:00:00,9167,86400
+F2DQ_SOURCE_TEST_TABLE,NUMBER,global__nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+F2DQ_SOURCE_TEST_TABLE,NUMBER,global__nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+F2DQ_SOURCE_TEST_TABLE,NUMBER,global__diff,2021-05-01 00:00:00,2021-05-02 00:00:00,9000,86400
+F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__min_length,2021-05-01 00:00:00,2021-05-02 00:00:00,3000,86400
+F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__max_length,2021-05-01 00:00:00,2021-05-02 00:00:00,5000,86400
+F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__avg_length,2021-05-01 00:00:00,2021-05-02 00:00:00,3900,86400
+F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__missing_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__missing_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+F2DQ_SOURCE_TEST_TABLE,---,global__row_count,2021-05-01 00:00:00,2021-05-02 00:00:00,10000,86400
+F2DQ_SOURCE_TEST_TABLE,---,global__freshness,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400
+F2DQ_SOURCE_TEST_TABLE,---,global__my_distinct_table_rows,2021-05-01 00:00:00,2021-05-02 00:00:00,10000,86400
+SAMPLE_TABLE,VALUE1,min,2021-05-01 00:00:00,2021-05-02 00:00:00,100000,86400
+SAMPLE_TABLE,VALUE1,max,2021-05-01 00:00:00,2021-05-02 00:00:00,200000,86400
+SAMPLE_TABLE,VALUE1,avg,2021-05-01 00:00:00,2021-05-02 00:00:00,127500,86400
+SAMPLE_TABLE,VALUE1,stddev,2021-05-01 00:00:00,2021-05-02 00:00:00,48563,86400
+SAMPLE_TABLE,VALUE1,variance,2021-05-01 00:00:00,2021-05-02 00:00:00,2358333,86400
+SAMPLE_TABLE,VALUE1,nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_TABLE,VALUE1,nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_TABLE,VALUE1,diff,2021-05-01 00:00:00,2021-05-02 00:00:00,100000,86400
+SAMPLE_TABLE,VALUE2,min,2021-05-01 00:00:00,2021-05-02 00:00:00,109000,86400
+SAMPLE_TABLE,VALUE2,max,2021-05-01 00:00:00,2021-05-02 00:00:00,209000,86400
+SAMPLE_TABLE,VALUE2,avg,2021-05-01 00:00:00,2021-05-02 00:00:00,180750,86400
+SAMPLE_TABLE,VALUE2,stddev,2021-05-01 00:00:00,2021-05-02 00:00:00,47975,86400
+SAMPLE_TABLE,VALUE2,variance,2021-05-01 00:00:00,2021-05-02 00:00:00,2301583,86400
+SAMPLE_TABLE,VALUE2,nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_TABLE,VALUE2,nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_TABLE,VALUE2,diff,2021-05-01 00:00:00,2021-05-02 00:00:00,100000,86400
+SAMPLE_TABLE,NULL_VALUE,min,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400
+SAMPLE_TABLE,NULL_VALUE,max,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400
+SAMPLE_TABLE,NULL_VALUE,avg,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400
+SAMPLE_TABLE,NULL_VALUE,stddev,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400
+SAMPLE_TABLE,NULL_VALUE,variance,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400
+SAMPLE_TABLE,NULL_VALUE,nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,4000,86400
+SAMPLE_TABLE,NULL_VALUE,nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,100000,86400
+SAMPLE_TABLE,NULL_VALUE,diff,2021-05-01 00:00:00,2021-05-02 00:00:00,,86400
+SAMPLE_TABLE,EVENT_TYPE,min_length,2021-05-01 00:00:00,2021-05-02 00:00:00,3000,86400
+SAMPLE_TABLE,EVENT_TYPE,max_length,2021-05-01 00:00:00,2021-05-02 00:00:00,4000,86400
+SAMPLE_TABLE,EVENT_TYPE,avg_length,2021-05-01 00:00:00,2021-05-02 00:00:00,3250,86400
+SAMPLE_TABLE,EVENT_TYPE,nulls_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_TABLE,EVENT_TYPE,nulls_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_TABLE,EVENT_TYPE,missing_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_TABLE,EVENT_TYPE,missing_count,2021-05-01 00:00:00,2021-05-02 00:00:00,0,86400
+SAMPLE_TABLE,EVENT_TYPE,regex_test,2021-05-01 00:00:00,2021-05-02 00:00:00,4000,86400
+SAMPLE_TABLE,EVENT_TYPE,match_regex,2021-05-01 00:00:00,2021-05-02 00:00:00,1000,86400
+SAMPLE_TABLE,EVENT_TYPE,match_regex_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,25000,86400
+SAMPLE_TABLE,EVENT_TYPE,not_match_regex,2021-05-01 00:00:00,2021-05-02 00:00:00,1000,86400
+SAMPLE_TABLE,EVENT_TYPE,not_match_regex_percent,2021-05-01 00:00:00,2021-05-02 00:00:00,25000,86400
+SAMPLE_TABLE,EVENT_TYPE,distinct_values,2021-05-01 00:00:00,2021-05-02 00:00:00,2000,86400
+SAMPLE_TABLE,EVENT_TYPE,duplicate_values,2021-05-01 00:00:00,2021-05-02 00:00:00,1000,86400
+SAMPLE_TABLE,EVENT_TYPE,duplicate_rows,2021-05-01 00:00:00,2021-05-02 00:00:00,3000,86400
+SAMPLE_TABLE,EVENT_TYPE,unique_rows,2021-05-01 00:00:00,2021-05-02 00:00:00,1000,86400
+SAMPLE_TABLE,---,row_count,2021-05-01 00:00:00,2021-05-02 00:00:00,4000,86400
+SAMPLE_TABLE,---,freshness,2021-05-01 00:00:00,2021-05-02 00:00:00,40765000,86400
+SAMPLE_TABLE,---,my_distinct_table_rows,2021-05-01 00:00:00,2021-05-02 00:00:00,9000,86400
+SAMPLE_TABLE,---,my_custom_table_metric,2021-05-01 00:00:00,2021-05-02 00:00:00,1000000,86400
+SAMPLE_TABLE,---,distinct_table_rows,2021-05-01 00:00:00,2021-05-02 00:00:00,4000,86400
+SAMPLE_WITH_ANOMALY,VALUE1,min,2021-05-02 00:00:00,2021-05-03 00:00:00,98000,86400
+SAMPLE_WITH_ANOMALY,VALUE1,max,2021-05-02 00:00:00,2021-05-03 00:00:00,98000,86400
+SAMPLE_WITH_ANOMALY,VALUE1,avg,2021-05-02 00:00:00,2021-05-03 00:00:00,98000,86400
+SAMPLE_WITH_ANOMALY,VALUE1,stddev,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400
+SAMPLE_WITH_ANOMALY,VALUE1,variance,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400
+SAMPLE_WITH_ANOMALY,VALUE1,nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_WITH_ANOMALY,VALUE1,nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_WITH_ANOMALY,VALUE1,diff,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_WITH_ANOMALY,VALUE2,min,2021-05-02 00:00:00,2021-05-03 00:00:00,205000,86400
+SAMPLE_WITH_ANOMALY,VALUE2,max,2021-05-02 00:00:00,2021-05-03 00:00:00,205000,86400
+SAMPLE_WITH_ANOMALY,VALUE2,avg,2021-05-02 00:00:00,2021-05-03 00:00:00,205000,86400
+SAMPLE_WITH_ANOMALY,VALUE2,stddev,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400
+SAMPLE_WITH_ANOMALY,VALUE2,variance,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400
+SAMPLE_WITH_ANOMALY,VALUE2,nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_WITH_ANOMALY,VALUE2,nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_WITH_ANOMALY,VALUE2,diff,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_WITH_ANOMALY,EVENT_TYPE,min_length,2021-05-02 00:00:00,2021-05-03 00:00:00,3000,86400
+SAMPLE_WITH_ANOMALY,EVENT_TYPE,max_length,2021-05-02 00:00:00,2021-05-03 00:00:00,3000,86400
+SAMPLE_WITH_ANOMALY,EVENT_TYPE,avg_length,2021-05-02 00:00:00,2021-05-03 00:00:00,3000,86400
+SAMPLE_WITH_ANOMALY,EVENT_TYPE,nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_WITH_ANOMALY,EVENT_TYPE,nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_WITH_ANOMALY,EVENT_TYPE,missing_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_WITH_ANOMALY,EVENT_TYPE,missing_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_WITH_ANOMALY,---,row_count,2021-05-02 00:00:00,2021-05-03 00:00:00,1000,86400
+SAMPLE_WITH_ANOMALY,---,freshness,2021-05-02 00:00:00,2021-05-03 00:00:00,41065000,86400
+SAMPLE_WITH_ANOMALY,---,my_distinct_table_rows,2021-05-02 00:00:00,2021-05-03 00:00:00,12000,86400
+BUY_EVENTS,VALUE1,min,2021-05-02 00:00:00,2021-05-03 00:00:00,98000,86400
+BUY_EVENTS,VALUE1,max,2021-05-02 00:00:00,2021-05-03 00:00:00,98000,86400
+BUY_EVENTS,VALUE1,avg,2021-05-02 00:00:00,2021-05-03 00:00:00,98000,86400
+BUY_EVENTS,VALUE1,stddev,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400
+BUY_EVENTS,VALUE1,variance,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400
+BUY_EVENTS,VALUE1,nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+BUY_EVENTS,VALUE1,nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+BUY_EVENTS,VALUE1,diff,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+BUY_EVENTS,VALUE2,min,2021-05-02 00:00:00,2021-05-03 00:00:00,205000,86400
+BUY_EVENTS,VALUE2,max,2021-05-02 00:00:00,2021-05-03 00:00:00,205000,86400
+BUY_EVENTS,VALUE2,avg,2021-05-02 00:00:00,2021-05-03 00:00:00,205000,86400
+BUY_EVENTS,VALUE2,stddev,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400
+BUY_EVENTS,VALUE2,variance,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400
+BUY_EVENTS,VALUE2,nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+BUY_EVENTS,VALUE2,nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+BUY_EVENTS,VALUE2,diff,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+BUY_EVENTS,EVENT_TYPE,min_length,2021-05-02 00:00:00,2021-05-03 00:00:00,3000,86400
+BUY_EVENTS,EVENT_TYPE,max_length,2021-05-02 00:00:00,2021-05-03 00:00:00,3000,86400
+BUY_EVENTS,EVENT_TYPE,avg_length,2021-05-02 00:00:00,2021-05-03 00:00:00,3000,86400
+BUY_EVENTS,EVENT_TYPE,nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+BUY_EVENTS,EVENT_TYPE,nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+BUY_EVENTS,EVENT_TYPE,missing_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+BUY_EVENTS,EVENT_TYPE,missing_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+BUY_EVENTS,---,row_count,2021-05-02 00:00:00,2021-05-03 00:00:00,1000,86400
+BUY_EVENTS,---,freshness,2021-05-02 00:00:00,2021-05-03 00:00:00,41065000,86400
+BUY_EVENTS,---,my_distinct_table_rows,2021-05-02 00:00:00,2021-05-03 00:00:00,10000,86400
+SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__min,2021-05-02 00:00:00,2021-05-03 00:00:00,990,86400
+SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__max,2021-05-02 00:00:00,2021-05-03 00:00:00,4990,86400
+SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__avg,2021-05-02 00:00:00,2021-05-03 00:00:00,3306,86400
+SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__stddev,2021-05-02 00:00:00,2021-05-03 00:00:00,1668,86400
+SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__variance,2021-05-02 00:00:00,2021-05-03 00:00:00,2784,86400
+SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__diff,2021-05-02 00:00:00,2021-05-03 00:00:00,4000,86400
+SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__min_length,2021-05-02 00:00:00,2021-05-03 00:00:00,10000,86400
+SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__max_length,2021-05-02 00:00:00,2021-05-03 00:00:00,17000,86400
+SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__avg_length,2021-05-02 00:00:00,2021-05-03 00:00:00,14105,86400
+SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__missing_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__missing_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,RATING,global__min_length,2021-05-02 00:00:00,2021-05-03 00:00:00,1000,86400
+SAMPLE_WITHOUT_TIME_FILTER,RATING,global__max_length,2021-05-02 00:00:00,2021-05-03 00:00:00,5000,86400
+SAMPLE_WITHOUT_TIME_FILTER,RATING,global__avg_length,2021-05-02 00:00:00,2021-05-03 00:00:00,3053,86400
+SAMPLE_WITHOUT_TIME_FILTER,RATING,global__nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,RATING,global__nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,RATING,global__missing_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,RATING,global__missing_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,---,global__row_count,2021-05-02 00:00:00,2021-05-03 00:00:00,19000,86400
+SAMPLE_WITHOUT_TIME_FILTER,---,global__freshness,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400
+SAMPLE_WITHOUT_TIME_FILTER,---,global__my_distinct_table_rows,2021-05-02 00:00:00,2021-05-03 00:00:00,19000,86400
+F2DQ_SOURCE_TEST_TABLE,NUMBER,global__min,2021-05-02 00:00:00,2021-05-03 00:00:00,1000,86400
+F2DQ_SOURCE_TEST_TABLE,NUMBER,global__max,2021-05-02 00:00:00,2021-05-03 00:00:00,10000,86400
+F2DQ_SOURCE_TEST_TABLE,NUMBER,global__avg,2021-05-02 00:00:00,2021-05-03 00:00:00,5500,86400
+F2DQ_SOURCE_TEST_TABLE,NUMBER,global__stddev,2021-05-02 00:00:00,2021-05-03 00:00:00,3028,86400
+F2DQ_SOURCE_TEST_TABLE,NUMBER,global__variance,2021-05-02 00:00:00,2021-05-03 00:00:00,9167,86400
+F2DQ_SOURCE_TEST_TABLE,NUMBER,global__nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+F2DQ_SOURCE_TEST_TABLE,NUMBER,global__nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+F2DQ_SOURCE_TEST_TABLE,NUMBER,global__diff,2021-05-02 00:00:00,2021-05-03 00:00:00,9000,86400
+F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__min_length,2021-05-02 00:00:00,2021-05-03 00:00:00,3000,86400
+F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__max_length,2021-05-02 00:00:00,2021-05-03 00:00:00,5000,86400
+F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__avg_length,2021-05-02 00:00:00,2021-05-03 00:00:00,3900,86400
+F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__missing_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__missing_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+F2DQ_SOURCE_TEST_TABLE,---,global__row_count,2021-05-02 00:00:00,2021-05-03 00:00:00,10000,86400
+F2DQ_SOURCE_TEST_TABLE,---,global__freshness,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400
+F2DQ_SOURCE_TEST_TABLE,---,global__my_distinct_table_rows,2021-05-02 00:00:00,2021-05-03 00:00:00,10000,86400
+SAMPLE_TABLE,VALUE1,min,2021-05-02 00:00:00,2021-05-03 00:00:00,100000,86400
+SAMPLE_TABLE,VALUE1,max,2021-05-02 00:00:00,2021-05-03 00:00:00,210000,86400
+SAMPLE_TABLE,VALUE1,avg,2021-05-02 00:00:00,2021-05-03 00:00:00,142500,86400
+SAMPLE_TABLE,VALUE1,stddev,2021-05-02 00:00:00,2021-05-03 00:00:00,49917,86400
+SAMPLE_TABLE,VALUE1,variance,2021-05-02 00:00:00,2021-05-03 00:00:00,2491667,86400
+SAMPLE_TABLE,VALUE1,nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_TABLE,VALUE1,nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_TABLE,VALUE1,diff,2021-05-02 00:00:00,2021-05-03 00:00:00,110000,86400
+SAMPLE_TABLE,VALUE2,min,2021-05-02 00:00:00,2021-05-03 00:00:00,109000,86400
+SAMPLE_TABLE,VALUE2,max,2021-05-02 00:00:00,2021-05-03 00:00:00,209000,86400
+SAMPLE_TABLE,VALUE2,avg,2021-05-02 00:00:00,2021-05-03 00:00:00,180750,86400
+SAMPLE_TABLE,VALUE2,stddev,2021-05-02 00:00:00,2021-05-03 00:00:00,47975,86400
+SAMPLE_TABLE,VALUE2,variance,2021-05-02 00:00:00,2021-05-03 00:00:00,2301583,86400
+SAMPLE_TABLE,VALUE2,nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_TABLE,VALUE2,nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_TABLE,VALUE2,diff,2021-05-02 00:00:00,2021-05-03 00:00:00,100000,86400
+SAMPLE_TABLE,NULL_VALUE,min,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400
+SAMPLE_TABLE,NULL_VALUE,max,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400
+SAMPLE_TABLE,NULL_VALUE,avg,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400
+SAMPLE_TABLE,NULL_VALUE,stddev,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400
+SAMPLE_TABLE,NULL_VALUE,variance,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400
+SAMPLE_TABLE,NULL_VALUE,nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,4000,86400
+SAMPLE_TABLE,NULL_VALUE,nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,100000,86400
+SAMPLE_TABLE,NULL_VALUE,diff,2021-05-02 00:00:00,2021-05-03 00:00:00,,86400
+SAMPLE_TABLE,EVENT_TYPE,min_length,2021-05-02 00:00:00,2021-05-03 00:00:00,3000,86400
+SAMPLE_TABLE,EVENT_TYPE,max_length,2021-05-02 00:00:00,2021-05-03 00:00:00,3000,86400
+SAMPLE_TABLE,EVENT_TYPE,avg_length,2021-05-02 00:00:00,2021-05-03 00:00:00,3000,86400
+SAMPLE_TABLE,EVENT_TYPE,nulls_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_TABLE,EVENT_TYPE,nulls_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_TABLE,EVENT_TYPE,missing_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_TABLE,EVENT_TYPE,missing_count,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_TABLE,EVENT_TYPE,regex_test,2021-05-02 00:00:00,2021-05-03 00:00:00,4000,86400
+SAMPLE_TABLE,EVENT_TYPE,match_regex,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_TABLE,EVENT_TYPE,match_regex_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_TABLE,EVENT_TYPE,not_match_regex,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_TABLE,EVENT_TYPE,not_match_regex_percent,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_TABLE,EVENT_TYPE,distinct_values,2021-05-02 00:00:00,2021-05-03 00:00:00,1000,86400
+SAMPLE_TABLE,EVENT_TYPE,duplicate_values,2021-05-02 00:00:00,2021-05-03 00:00:00,1000,86400
+SAMPLE_TABLE,EVENT_TYPE,duplicate_rows,2021-05-02 00:00:00,2021-05-03 00:00:00,4000,86400
+SAMPLE_TABLE,EVENT_TYPE,unique_rows,2021-05-02 00:00:00,2021-05-03 00:00:00,0,86400
+SAMPLE_TABLE,---,row_count,2021-05-02 00:00:00,2021-05-03 00:00:00,4000,86400
+SAMPLE_TABLE,---,freshness,2021-05-02 00:00:00,2021-05-03 00:00:00,40765000,86400
+SAMPLE_TABLE,---,my_distinct_table_rows,2021-05-02 00:00:00,2021-05-03 00:00:00,9000,86400
+SAMPLE_TABLE,---,my_custom_table_metric,2021-05-02 00:00:00,2021-05-03 00:00:00,1000000,86400
+SAMPLE_TABLE,---,distinct_table_rows,2021-05-02 00:00:00,2021-05-03 00:00:00,4000,86400
@@ -0,0 +1,5 @@
+table_name,sample_data_length
+BUY_EVENTS,506
+F2DQ_SOURCE_TEST_TABLE,361
+SAMPLE_TABLE,830
+SAMPLE_WITH_ANOMALY,507
@@ -0,0 +1,35 @@
+table_name,column_name,test_name,status,message,failures_count,severity
+TEST_F2DQ_METRICS,---,pected_metrics_,Pass,---,0,ERROR
+TEST_F2DQ_TABLE_SAMPLES,---,_table_samples_,Pass,---,0,ERROR
+TEST_F2DQ_Z_SCORE,---,pected_z_score_,Pass,---,0,ERROR
+TEST_F2DQ_ANOMALIES,---,cted_anomalies_,Pass,---,0,ERROR
+BUY_EVENTS,---,_table_rows__10,Pass,---,0,ERROR
+BUY_EVENTS,---,ts_row_count__1,Pass,---,0,ERROR
+BUY_EVENTS,---,null__freshness,Pass,---,0,ERROR
+BUY_EVENTS,VALUE2,5_02___min__200,Pass,---,0,ERROR
+BUY_EVENTS,VALUE1,alue1__min__107,Pass,---,0,ERROR
+BUY_EVENTS,VALUE1,ulls_percent__0,Pass,---,0,ERROR
+BUY_EVENTS,VALUE1,5_02___min__100,Pass,---,0,ERROR
+BUY_EVENTS,---,0__row_count__0,Pass,---,0,ERROR
+F2DQ_METRICS,---,_buy_events___3,Pass,---,0,ERROR
+F2DQ_METRICS,---,ref_buy_events_,Pass,---,0,ERROR
+F2DQ_METRICS,---,ref_buy_events_,Pass,---,0,ERROR
+BUY_EVENTS,VALUE1,__nulls_percent,Pass,---,0,ERROR
+F2DQ_METRICS,---,ref_buy_events_,Pass,---,0,ERROR
+TEST_F2DQ_TABLE_SAMPLES,---,_table_samples_,Pass,---,0,ERROR
+TEST_F2DQ_ANOMALIES,---,cted_anomalies_,Pass,---,0,ERROR
+BUY_EVENTS,---,ts_row_count__1,Pass,---,0,ERROR
+BUY_EVENTS,---,null__freshness,Pass,---,0,ERROR
+BUY_EVENTS,VALUE2,5_02___min__200,Pass,---,0,ERROR
+BUY_EVENTS,---,_table_rows__10,Pass,---,0,ERROR
+BUY_EVENTS,VALUE1,ulls_percent__0,Pass,---,0,ERROR
+BUY_EVENTS,---,0__row_count__0,Pass,---,0,ERROR
+BUY_EVENTS,VALUE1,5_02___min__100,Pass,---,0,ERROR
+BUY_EVENTS,VALUE1,alue1__min__107,Pass,---,0,ERROR
+BUY_EVENTS,VALUE1,__nulls_percent,Pass,---,0,ERROR
+F2DQ_METRICS,---,_buy_events___3,Pass,---,0,ERROR
+F2DQ_METRICS,---,ref_buy_events_,Pass,---,0,ERROR
+F2DQ_METRICS,---,ref_buy_events_,Pass,---,0,ERROR
+F2DQ_METRICS,---,ref_buy_events_,Pass,---,0,ERROR
+TEST_F2DQ_METRICS,---,pected_metrics_,Pass,---,0,ERROR
+TEST_F2DQ_Z_SCORE,---,pected_z_score_,Pass,---,0,ERROR
@@ -0,0 +1,125 @@
+table_name,column_name,metric,time_window_end,z_score_value,modified_z_score_value,last_value,last_avg,last_stddev,last_median,last_iqr,last_median_absolute_deviation,last_mean_absolute_deviation,interval_length_sec
+SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__avg_length,2021-05-03 00:00:00,0,0,14105,14105,0,14105,0,0,0,86400
+SAMPLE_TABLE,EVENT_TYPE,distinct_values,2021-05-03 00:00:00,-707,-674,1000,1500,707,1500,500,500,500,86400
+BUY_EVENTS,EVENT_TYPE,nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+SAMPLE_TABLE,EVENT_TYPE,max_length,2021-05-03 00:00:00,-707,-674,3000,3500,707,3500,500,500,500,86400
+SAMPLE_TABLE,EVENT_TYPE,nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+SAMPLE_TABLE,EVENT_TYPE,min_length,2021-05-03 00:00:00,0,0,3000,3000,0,3000,0,0,0,86400
+SAMPLE_TABLE,EVENT_TYPE,missing_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+SAMPLE_TABLE,NULL_VALUE,nulls_percent,2021-05-03 00:00:00,0,0,100000,100000,0,100000,0,0,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,RATING,global__avg_length,2021-05-03 00:00:00,0,0,3053,3053,0,3053,0,0,0,86400
+BUY_EVENTS,VALUE1,nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+SAMPLE_WITH_ANOMALY,EVENT_TYPE,nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+SAMPLE_TABLE,---,freshness,2021-05-03 00:00:00,0,0,40765000,40765000,0,40765000,0,0,0,86400
+F2DQ_SOURCE_TEST_TABLE,NUMBER,global__stddev,2021-05-03 00:00:00,0,0,3028,3028,0,3028,0,0,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,RATING,global__missing_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+BUY_EVENTS,EVENT_TYPE,min_length,2021-05-03 00:00:00,0,0,3000,3000,0,3000,0,0,0,86400
+SAMPLE_TABLE,VALUE1,nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+BUY_EVENTS,VALUE2,nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+SAMPLE_TABLE,VALUE1,variance,2021-05-03 00:00:00,707,674,2491667,2425000,94281,2425000,66667,66667,66667,86400
+F2DQ_SOURCE_TEST_TABLE,---,global__row_count,2021-05-03 00:00:00,0,0,10000,10000,0,10000,0,0,0,86400
+BUY_EVENTS,---,my_distinct_table_rows,2021-05-03 00:00:00,0,0,10000,10000,0,10000,0,0,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,---,global__row_count,2021-05-03 00:00:00,0,0,19000,19000,0,19000,0,0,0,86400
+SAMPLE_TABLE,EVENT_TYPE,match_regex,2021-05-03 00:00:00,-707,-674,0,500,707,500,500,500,500,86400
+SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__diff,2021-05-03 00:00:00,0,0,4000,4000,0,4000,0,0,0,86400
+BUY_EVENTS,VALUE2,diff,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+F2DQ_SOURCE_TEST_TABLE,NUMBER,global__avg,2021-05-03 00:00:00,0,0,5500,5500,0,5500,0,0,0,86400
+SAMPLE_TABLE,VALUE2,min,2021-05-03 00:00:00,0,0,109000,109000,0,109000,0,0,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__max_length,2021-05-03 00:00:00,0,0,17000,17000,0,17000,0,0,0,86400
+SAMPLE_WITH_ANOMALY,VALUE1,min,2021-05-03 00:00:00,-707,-674,98000,102500,6364,102500,4500,4500,4500,86400
+SAMPLE_WITH_ANOMALY,VALUE2,max,2021-05-03 00:00:00,707,674,205000,202500,3536,202500,2500,2500,2500,86400
+SAMPLE_WITH_ANOMALY,VALUE2,diff,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+SAMPLE_TABLE,EVENT_TYPE,not_match_regex_percent,2021-05-03 00:00:00,-707,-674,0,12500,17678,12500,12500,12500,12500,86400
+SAMPLE_TABLE,---,distinct_table_rows,2021-05-03 00:00:00,0,0,4000,4000,0,4000,0,0,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,RATING,global__missing_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+SAMPLE_WITH_ANOMALY,VALUE2,nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+SAMPLE_TABLE,VALUE2,stddev,2021-05-03 00:00:00,0,0,47975,47975,0,47975,0,0,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__min,2021-05-03 00:00:00,0,0,990,990,0,990,0,0,0,86400
+BUY_EVENTS,VALUE2,max,2021-05-03 00:00:00,707,674,205000,202500,3536,202500,2500,2500,2500,86400
+SAMPLE_WITHOUT_TIME_FILTER,---,global__my_distinct_table_rows,2021-05-03 00:00:00,0,0,19000,19000,0,19000,0,0,0,86400
+F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__avg_length,2021-05-03 00:00:00,0,0,3900,3900,0,3900,0,0,0,86400
+SAMPLE_WITH_ANOMALY,VALUE1,avg,2021-05-03 00:00:00,-707,-674,98000,102500,6364,102500,4500,4500,4500,86400
+SAMPLE_TABLE,VALUE2,variance,2021-05-03 00:00:00,0,0,2301583,2301583,0,2301583,0,0,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__missing_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+SAMPLE_WITH_ANOMALY,EVENT_TYPE,missing_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+BUY_EVENTS,VALUE1,nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__max,2021-05-03 00:00:00,0,0,4990,4990,0,4990,0,0,0,86400
+SAMPLE_WITH_ANOMALY,---,my_distinct_table_rows,2021-05-03 00:00:00,0,0,12000,12000,0,12000,0,0,0,86400
+BUY_EVENTS,VALUE1,diff,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__variance,2021-05-03 00:00:00,0,0,2784,2784,0,2784,0,0,0,86400
+SAMPLE_TABLE,EVENT_TYPE,duplicate_values,2021-05-03 00:00:00,0,0,1000,1000,0,1000,0,0,0,86400
+SAMPLE_TABLE,NULL_VALUE,nulls_count,2021-05-03 00:00:00,0,0,4000,4000,0,4000,0,0,0,86400
+F2DQ_SOURCE_TEST_TABLE,NUMBER,global__nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+F2DQ_SOURCE_TEST_TABLE,NUMBER,global__variance,2021-05-03 00:00:00,0,0,9167,9167,0,9167,0,0,0,86400
+SAMPLE_TABLE,EVENT_TYPE,avg_length,2021-05-03 00:00:00,-707,-674,3000,3125,177,3125,125,125,125,86400
+F2DQ_SOURCE_TEST_TABLE,NUMBER,global__min,2021-05-03 00:00:00,0,0,1000,1000,0,1000,0,0,0,86400
+F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__missing_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__avg,2021-05-03 00:00:00,0,0,3306,3306,0,3306,0,0,0,86400
+BUY_EVENTS,VALUE1,avg,2021-05-03 00:00:00,-707,-674,98000,102500,6364,102500,4500,4500,4500,86400
+SAMPLE_WITH_ANOMALY,VALUE2,min,2021-05-03 00:00:00,707,674,205000,202500,3536,202500,2500,2500,2500,86400
+F2DQ_SOURCE_TEST_TABLE,NUMBER,global__nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+SAMPLE_TABLE,EVENT_TYPE,not_match_regex,2021-05-03 00:00:00,-707,-674,0,500,707,500,500,500,500,86400
+BUY_EVENTS,VALUE1,max,2021-05-03 00:00:00,-707,-674,98000,102500,6364,102500,4500,4500,4500,86400
+SAMPLE_TABLE,VALUE1,stddev,2021-05-03 00:00:00,707,674,49917,49240,957,49240,677,677,677,86400
+BUY_EVENTS,VALUE1,min,2021-05-03 00:00:00,-707,-674,98000,102500,6364,102500,4500,4500,4500,86400
+SAMPLE_WITH_ANOMALY,VALUE2,nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+SAMPLE_TABLE,---,my_custom_table_metric,2021-05-03 00:00:00,0,0,1000000,1000000,0,1000000,0,0,0,86400
+BUY_EVENTS,EVENT_TYPE,max_length,2021-05-03 00:00:00,0,0,3000,3000,0,3000,0,0,0,86400
+SAMPLE_WITH_ANOMALY,EVENT_TYPE,avg_length,2021-05-03 00:00:00,0,0,3000,3000,0,3000,0,0,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+SAMPLE_WITH_ANOMALY,VALUE1,max,2021-05-03 00:00:00,-707,-674,98000,102500,6364,102500,4500,4500,4500,86400
+SAMPLE_TABLE,---,my_distinct_table_rows,2021-05-03 00:00:00,0,0,9000,9000,0,9000,0,0,0,86400
+SAMPLE_TABLE,---,row_count,2021-05-03 00:00:00,0,0,4000,4000,0,4000,0,0,0,86400
+SAMPLE_TABLE,VALUE2,max,2021-05-03 00:00:00,0,0,209000,209000,0,209000,0,0,0,86400
+SAMPLE_WITH_ANOMALY,VALUE1,diff,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+SAMPLE_WITH_ANOMALY,EVENT_TYPE,nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+BUY_EVENTS,EVENT_TYPE,avg_length,2021-05-03 00:00:00,0,0,3000,3000,0,3000,0,0,0,86400
+SAMPLE_TABLE,VALUE1,diff,2021-05-03 00:00:00,707,674,110000,105000,7071,105000,5000,5000,5000,86400
+F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__missing_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__missing_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+SAMPLE_TABLE,VALUE1,min,2021-05-03 00:00:00,0,0,100000,100000,0,100000,0,0,0,86400
+SAMPLE_TABLE,EVENT_TYPE,regex_test,2021-05-03 00:00:00,0,0,4000,4000,0,4000,0,0,0,86400
+SAMPLE_TABLE,EVENT_TYPE,nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,RATING,global__min_length,2021-05-03 00:00:00,0,0,1000,1000,0,1000,0,0,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,TITLE,global__min_length,2021-05-03 00:00:00,0,0,10000,10000,0,10000,0,0,0,86400
+SAMPLE_WITH_ANOMALY,VALUE1,nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+F2DQ_SOURCE_TEST_TABLE,NUMBER,global__max,2021-05-03 00:00:00,0,0,10000,10000,0,10000,0,0,0,86400
+SAMPLE_WITH_ANOMALY,EVENT_TYPE,missing_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+SAMPLE_TABLE,VALUE2,nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+SAMPLE_WITH_ANOMALY,VALUE2,avg,2021-05-03 00:00:00,707,674,205000,202500,3536,202500,2500,2500,2500,86400
+BUY_EVENTS,---,freshness,2021-05-03 00:00:00,-707,-674,41065000,41186500,171827,41186500,121500,121500,121500,86400
+BUY_EVENTS,VALUE2,avg,2021-05-03 00:00:00,707,674,205000,202500,3536,202500,2500,2500,2500,86400
+BUY_EVENTS,---,row_count,2021-05-03 00:00:00,0,0,1000,1000,0,1000,0,0,0,86400
+SAMPLE_WITH_ANOMALY,---,freshness,2021-05-03 00:00:00,-707,-674,41065000,41186500,171827,41186500,121500,121500,121500,86400
+SAMPLE_WITHOUT_TIME_FILTER,RATING,global__max_length,2021-05-03 00:00:00,0,0,5000,5000,0,5000,0,0,0,86400
+SAMPLE_WITH_ANOMALY,EVENT_TYPE,min_length,2021-05-03 00:00:00,0,0,3000,3000,0,3000,0,0,0,86400
+SAMPLE_TABLE,EVENT_TYPE,match_regex_percent,2021-05-03 00:00:00,-707,-674,0,12500,17678,12500,12500,12500,12500,86400
+SAMPLE_WITH_ANOMALY,EVENT_TYPE,max_length,2021-05-03 00:00:00,0,0,3000,3000,0,3000,0,0,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__stddev,2021-05-03 00:00:00,0,0,1668,1668,0,1668,0,0,0,86400
+SAMPLE_TABLE,VALUE1,max,2021-05-03 00:00:00,707,674,210000,205000,7071,205000,5000,5000,5000,86400
+SAMPLE_TABLE,VALUE2,avg,2021-05-03 00:00:00,0,0,180750,180750,0,180750,0,0,0,86400
+SAMPLE_WITH_ANOMALY,VALUE1,nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__min_length,2021-05-03 00:00:00,0,0,3000,3000,0,3000,0,0,0,86400
+SAMPLE_TABLE,VALUE1,nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+BUY_EVENTS,EVENT_TYPE,nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+BUY_EVENTS,EVENT_TYPE,missing_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+BUY_EVENTS,EVENT_TYPE,missing_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,RATING,global__nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+BUY_EVENTS,VALUE2,nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+F2DQ_SOURCE_TEST_TABLE,NUMBER,global__diff,2021-05-03 00:00:00,0,0,9000,9000,0,9000,0,0,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,RATING,global__nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+BUY_EVENTS,VALUE2,min,2021-05-03 00:00:00,707,674,205000,202500,3536,202500,2500,2500,2500,86400
+F2DQ_SOURCE_TEST_TABLE,DESCRIPTION,global__max_length,2021-05-03 00:00:00,0,0,5000,5000,0,5000,0,0,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+SAMPLE_TABLE,VALUE2,nulls_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+SAMPLE_TABLE,EVENT_TYPE,duplicate_rows,2021-05-03 00:00:00,707,674,4000,3500,707,3500,500,500,500,86400
+F2DQ_SOURCE_TEST_TABLE,---,global__my_distinct_table_rows,2021-05-03 00:00:00,0,0,10000,10000,0,10000,0,0,0,86400
+SAMPLE_TABLE,VALUE2,diff,2021-05-03 00:00:00,0,0,100000,100000,0,100000,0,0,0,86400
+SAMPLE_WITHOUT_TIME_FILTER,RENTAL_RATE,global__nulls_percent,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
+SAMPLE_WITH_ANOMALY,---,row_count,2021-05-03 00:00:00,0,0,1000,1000,0,1000,0,0,0,86400
+SAMPLE_TABLE,EVENT_TYPE,unique_rows,2021-05-03 00:00:00,-707,-674,0,500,707,500,500,500,500,86400
+SAMPLE_TABLE,VALUE1,avg,2021-05-03 00:00:00,707,674,142500,135000,10607,135000,7500,7500,7500,86400
+SAMPLE_TABLE,EVENT_TYPE,missing_count,2021-05-03 00:00:00,0,0,0,0,0,0,0,0,0,86400
@@ -0,0 +1,10 @@
+creation_time,update_time,event_type,value1,value2,null_value,not_used_colum
+2021-04-30 12:40:35,,buy,100,109,,1
+2021-05-01 12:31:32,,buy,100,200,,1
+2021-05-01 12:35:35,,buy,110,205,,1
+2021-05-01 12:40:35,,sell,200,209,,1
+2021-05-01 12:40:35,2021-05-01 12:40:37,buy,100,109,,1
+2021-05-02 12:31:32,,buy,110,200,,1
+2021-05-02 12:35:35,,buy,150,205,,1
+2021-05-02 12:40:35,,buy,210,209,,1
+2021-05-02 12:40:35,,buy,100,109,,1
@@ -0,0 +1,13 @@
+creation_time,event_type,value1,value2
+2021-04-30 12:40:35,buy,101,109
+2021-05-01 12:31:32,buy,107,200
+2021-05-02 12:35:35,buy,98,205
+2021-05-03 12:40:35,sell,108,209
+2021-05-04 12:40:35,buy,100,109
+2021-05-05 12:31:32,buy,110,200
+2021-05-06 12:35:35,buy,99,205
+2021-05-07 12:40:35,buy,94,209
+2021-05-08 12:40:35,buy,104,109
+2021-05-09 12:31:32,buy,10,200
+2021-05-10 12:35:35,buy,23,205
+2021-05-11 12:40:35,sell,10,209
@@ -0,0 +1,20 @@
+title,rental_rate,rating
+Academy Dinosaur,0.99,PG-13
+Alamo Videotape,0.99,G
+Affair Prejudice,2.99,G
+African Egg,2.99,G
+Ace Goldfinger,4.99,G
+Alice Fantasia,0.99,NC-17
+Adaptation Holes,2.99,NC-17
+Alien Center,2.99,NC-17
+Aladdin Calendar,4.99,NC-17
+Chamber Italian,4.99,NC-17
+Alaska Phantom,0.99,PG
+Agent Truman,2.99,PG
+Ali Forever,4.99,PG
+Alabama Devil,2.99,PG-13
+Bright Encounters,4.99,PG-13
+Airplane Sierra,4.99,PG-13
+Date Speed,0.99,R
+Grosse Wonderful,4.99,R
+Airport Pollock,4.99,R
@@ -0,0 +1,21 @@
+"full_name","email"
+"Lizzie Effertz","*****@fakemail.com"
+"Orlando Abbott","*****@fakemail.com"
+"Kelley Harann","*****@fakemail.com"
+"Ruth Langworth","*****@fakemail.com"
+"Lane Swift","*****@fakemail.com"
+"Bertha Corwin","*****@fakemail.com"
+"Manuela Kling","*****@fakemail.com"
+"Mose Balistreri","*****@fakemail.com"
+"Robin Halvorson","*****@fakemail.com"
+"Osbaldo Parker I","*****@fakemail.com"
+"Javier Runolfsson","*****@fakemail.net"
+"Amelia Batz","*****@fakemail.com"
+"Abby Pouros","*****@fakemail.com"
+"Markus Homenick","*****@fakemail.com"
+"Braeden Turner","*****@fakemail.com"
+"Horacio Parker","*****@fakemail.info"
+"Ms. Stacy Padberg","*****@fakemail.com"
+"Dr. Deshawn Stracke","*****@fakemail.com"
+"Pascale Grady","*****@fakemail.com"
+"Lacy Brekke","*****@fakemail.com"
@@ -0,0 +1,21 @@
+full_name,email
+  lizzie effertz,torp.trisha@fakemail.com
+ orlando abbott,dayton.hermiston@fakemail.com
+kelley     harann,borer.blake@fakemail.com
+ruth   langworth,garett66@fakemail.com
+lane swift  ,nienow.coralie@fakemail.com
+bertha corwin ,tstroman@fakemail.com
+manuela   kling,shawn.langworth@fakemail.com
+mose balistreri,dorris70@fakemail.com
+robin    halvorson,murazik.americo@fakemail.com
+     osbaldo parker i  ,friesen.angeline@fakemail.com
+javier runolfsson  ,benjamin.bailey@fakemail.net
+amelia batz,garrison60@fakemail.com
+abby  pouros,dominique.leannon@fakemail.com
+markus homenick,piper73@fakemail.com
+braeden turner,kozey.jace@fakemail.com
+horacio   parker,vtillman@fakemail.info
+ms. stacy       padberg,erdman.elaina@fakemail.com
+dr.     deshawn stracke,rosendo.beer@fakemail.com
+  pascale grady,princess60@fakemail.com
+lacy     brekke,romaguera.darrell@fakemail.com
@@ -0,0 +1,9 @@
+transaction_id,creation_time,status,value
+1,2021-05-01 12:31:32,pending,100
+2,2021-05-01 12:35:35,pending,200
+1,2021-05-01 12:40:35,completed,100
+3,2021-05-01 12:40:35,pending,300
+3,2021-05-02 12:31:32,completed,300
+4,2021-05-02 12:35:35,completed,10
+5,2021-05-02 12:40:35,pending,100
+4,2021-05-02 12:40:35,completed,40
@@ -0,0 +1,18 @@
+transaction_id,creation_time,status,value,use_case
+1,2021-05-01 12:31:32,pending,100,take_first
+2,2021-05-01 12:35:35,pending,200,take_first
+3,2021-05-01 12:40:35,pending,300,take_first
+4,2021-05-02 12:35:35,completed,10,take_first
+5,2021-05-02 12:40:35,pending,100,take_first
+1,2021-05-01 12:40:35,completed,100,take_last
+2,2021-05-01 12:35:35,pending,200,take_last
+3,2021-05-02 12:31:32,completed,300,take_last
+4,2021-05-02 12:40:35,completed,40,take_last
+5,2021-05-02 12:40:35,pending,100,take_last
+1,2021-05-01 12:40:35,completed,100,take_all_statuses
+1,2021-05-01 12:31:32,pending,100,take_all_statuses
+2,2021-05-01 12:35:35,pending,200,take_all_statuses
+3,2021-05-02 12:31:32,completed,300,take_all_statuses
+3,2021-05-01 12:40:35,pending,300,take_all_statuses
+4,2021-05-02 12:40:35,completed,40,take_all_statuses
+5,2021-05-02 12:40:35,pending,100,take_all_statuses
@@ -0,0 +1,7 @@
+"transaction_id","creation_time","status","value","f2dq_duplicate_group_row_count","f2dq_duplicate_group_row_number"
+1,"2021-05-01 12:31:32","pending",100,2,1
+1,"2021-05-01 12:40:35","completed",100,2,2
+3,"2021-05-01 12:40:35","pending",300,2,1
+3,"2021-05-02 12:31:32","completed",300,2,2
+4,"2021-05-02 12:35:35","completed",10,2,1
+4,"2021-05-02 12:40:35","completed",40,2,2
@@ -0,0 +1,52 @@
+state,code
+Ala.,AL
+Alaska,AK
+Ariz.,AZ
+Ark.,AR
+Calif.,CA
+Colo.,CO
+Conn.,CT
+Del.,DE
+D.C.,DC
+Fla.,FL
+Ga.,GA
+Hawaii,HI
+Idaho,ID
+Ill.,IL
+Ind.,IN
+Iowa,IA
+Kans.,KS
+Ky.,KY
+La.,LA
+Maine,ME
+Md.,MD
+Mass.,MA
+Mich.,MI
+Minn.,MN
+Miss.,MS
+Mo.,MO
+Mont.,MT
+Nebr.,NE
+Nev.,NV
+N.H.,NH
+N.J.,NJ
+N.M.,NM
+N.Y.,NY
+N.C.,NC
+N.D.,ND
+Ohio,OH
+Okla.,OK
+Ore.,OR
+Pa.,PA
+R.I.,RI
+S.C.,SC
+S.D.,SD
+Tenn.,TN
+Tex.,TX
+Utah,UT
+Vt.,VT
+Va.,VA
+Wash.,WA
+W.Va.,WV
+Wis.,WI
+Wyo.,WY
@@ -0,0 +1,52 @@
+"state","code","state__normalized"
+"Ariz.","AZ","Arizona"
+"Ind.","IN","Indiana"
+"La.","LA","Louisiana"
+"W.Va.","WV","West Virginia"
+"Nebr.","NE","Nebraska"
+"Pa.","PA","Pennsylvania"
+"Iowa","IA","Iowa"
+"N.H.","NH","New Hampshire"
+"S.C.","SC","South Carolina"
+"Ore.","OR","Oregon"
+"Conn.","CT","Connecticut"
+"R.I.","RI","Rhode Island"
+"Minn.","MN","Minnesota"
+"D.C.","DC","District of Columbia"
+"Wyo.","WY","Wyoming"
+"Hawaii","HI","Hawaii"
+"Wash.","WA","Washington"
+"N.D.","ND","North Dakota"
+"Mass.","MA","Massachusetts"
+"N.Y.","NY","New York"
+"N.M.","NM","New Mexico"
+"Colo.","CO","Colorado"
+"Ohio","OH","Ohio"
+"Idaho","ID","Idaho"
+"Ala.","AL","Alabama"
+"Ark.","AR","Arkansas"
+"S.D.","SD","South Dakota"
+"Mo.","MO","Missouri"
+"N.J.","NJ","New Jersey"
+"Miss.","MS","Mississippi"
+"Kans.","KS","Kansas"
+"Vt.","VT","Vermont"
+"Calif.","CA","California"
+"Mich.","MI","Michigan"
+"Alaska","AK","Alaska"
+"Nev.","NV","Nevada"
+"Okla.","OK","Oklahoma"
+"Tenn.","TN","Tennessee"
+"Ga.","GA","Georgia"
+"Wis.","WI","Wisconsin"
+"Ky.","KY","Kentucky"
+"N.C.","NC","North Carolina"
+"Mont.","MT","Montana"
+"Fla.","FL","Florida"
+"Va.","VA","Virginia"
+"Tex.","TX","Texas"
+"Md.","MD","Maryland"
+"Utah","UT","Utah"
+"Maine","ME","Maine"
+"Del.","DE","Delaware"
+"Ill.","IL","Illinois"
@@ -0,0 +1,52 @@
+source,target
+Ala.,Alabama
+Alaska,Alaska
+Ariz.,Arizona
+Ark.,Arkansas
+Calif.,California
+Colo.,Colorado
+Conn.,Connecticut
+Del.,Delaware
+D.C.,District of Columbia
+Fla.,Florida
+Ga.,Georgia
+Hawaii,Hawaii
+Idaho,Idaho
+Ill.,Illinois
+Ind.,Indiana
+Iowa,Iowa
+Kans.,Kansas
+Ky.,Kentucky
+La.,Louisiana
+Maine,Maine
+Md.,Maryland
+Mass.,Massachusetts
+Mich.,Michigan
+Minn.,Minnesota
+Miss.,Mississippi
+Mo.,Missouri
+Mont.,Montana
+Nebr.,Nebraska
+Nev.,Nevada
+N.H.,New Hampshire
+N.J.,New Jersey
+N.M.,New Mexico
+N.Y.,New York
+N.C.,North Carolina
+N.D.,North Dakota
+Ohio,Ohio
+Okla.,Oklahoma
+Ore.,Oregon
+Pa.,Pennsylvania
+R.I.,Rhode Island
+S.C.,South Carolina
+S.D.,South Dakota
+Tenn.,Tennessee
+Tex.,Texas
+Utah,Utah
+Vt.,Vermont
+Va.,Virginia
+Wash.,Washington
+W.Va.,West Virginia
+Wis.,Wisconsin
+Wyo.,Wyoming
@@ -0,0 +1,11 @@
+credit_card_number,valid_credit_card
+4941533405630082,1
+5476749195896614,1
+3568497486294461,1
+not_a_card_number,0
+3434-4351-4234-3234,0
+3434 4351 4234 3234,0
+34344 42344 43455 43456,0
+43423432,0
+234343443434,0
+2343423423423423423423423423,0
@@ -0,0 +1,13 @@
+date_time,valid_date_eu,valid_date_us,valid_date_inverse,valid_date_iso_8601,valid_time_24h,valid_time_12h,valid_time
+31-01-2020,1,0,0,0,0,0,0
+01/31/2020,0,1,0,0,0,0,0
+05.05.2020,1,1,0,0,0,0,0
+2020-01-31,0,0,1,0,0,0,0
+23:59,0,0,0,0,1,0,1
+12:59,0,0,0,0,1,1,1
+13:59:01,0,0,0,0,0,0,1
+"12:59:01,55",0,0,0,0,0,0,1
+11:59:00,0,0,0,0,0,0,1
+midnight,0,0,0,0,0,0,0
+2020-01-31T12:59:00+02:00,0,0,0,1,0,0,0
+2020-01-31T12:59:00,0,0,0,1,0,0,0
@@ -0,0 +1,5 @@
+user_id,email,email_valid
+1,test@fakemail.com,1
+2,novalidemail@,0
+3,novalidemail@com,0
+4,test+alovalidemail@fakemail.com,0
@@ -0,0 +1,29 @@
+ip_address,valid_ip_v4,valid_ip_v6,valid_ip
+1.2.3.4,1,0,1
+01.102.103.104,1,0,1
+124.171.228.4,1,0,1
+192.168.1.35,1,0,1
+192.168.1.198,1,0,1
+127.248.111.240,1,0,1
+01.1.1,0,0,0
+12325412,0,0,0
+notvalidatall,0,0,0
+232.232.33,0,0,0
+232.3232.232.232+2312,0,0,0
+::::erwerwe,0,0,0
+::3343:4343434343:34343:343434343:443,0,0,0
+,0,0,0
+2001:db8:3333:4444:5555:6666:7777:8888,0,1,1
+2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF,0,1,1
+2001:db8::,0,1,1
+::1234:5678,0,1,1
+2001:db8::1234:5678,0,1,1
+::11.22.33.44,0,1,1
+2001:db8::123.123.123.123,0,1,1
+2001:db8::1234:5678:5.6.7.8,0,1,1
+2001:db8:3333:4444:5555:6666:1.2.3.4,0,1,1
+::11.22.33.44,0,1,1
+2001:db8::123.123.123.123,0,1,1
+::1234:5678:91.123.4.56,0,1,1
+::1234:5678:1.2.3.4,0,1,1
+2001:db8::1234:5678:5.6.7.8,0,1,1
@@ -0,0 +1,12 @@
+number,is_number,is_number_decimal_point,is_number_decimal_comma,is_percentage,is_percentage_decimal_point,is_percentage_decimal_comma
+133,1,0,0,0,0,0
+1232.232,0,1,0,0,0,0
+"2332,123",0,0,1,0,0,0
+not a number,0,0,0,0,0,0
+"1,3%",0,0,0,1,0,1
+123%,0,0,0,1,1,1
+13  %,0,0,0,0,0,0
+76.234%,0,0,0,1,1,0
+not a number,0,0,0,0,0,0
+x,0,0,0,0,0,0
+123partly987,0,0,0,0,0,0
@@ -0,0 +1,8 @@
+uuid,valid_uuid
+ace1245c-3af5-11ec-8d3d-0242ac130003,1
+a568464e-a05d-412c-8b30-517a46c57d88,1
+notanuid,0
+d0d61836-3af5-11ec-8d3d-0242ac130003,1
+d0d61c6e-3af5-11ec-8d3d-0242ac130003,1
+343422-234324-234234-4234234-23432,0
+343422-234324-234234-4234234-234xxx32,0
@@ -0,0 +1,13 @@
+date_time
+31-01-2020
+01/31/2020
+05.05.2020
+2020-01-31
+23:59
+12:59
+13:59:01
+"12:59:01,55"
+11:59:00
+midnight
+2020-01-31T12:59:00+02:00
+2020-01-31T12:59:00
@@ -0,0 +1,5 @@
+user_id,email
+1,test@fakemail.com
+2,novalidemail@
+3,novalidemail@com
+4,test+alovalidemail@fakemail.com
@@ -0,0 +1,29 @@
+ip_address
+1.2.3.4
+01.102.103.104
+124.171.228.4
+192.168.1.35
+192.168.1.198
+127.248.111.240
+01.1.1
+12325412
+notvalidatall
+232.232.33
+232.3232.232.232+2312
+::::erwerwe
+::3343:4343434343:34343:343434343:443
+
+2001:db8:3333:4444:5555:6666:7777:8888
+2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF
+2001:db8::
+::1234:5678
+2001:db8::1234:5678
+::11.22.33.44
+2001:db8::123.123.123.123
+2001:db8::1234:5678:5.6.7.8
+2001:db8:3333:4444:5555:6666:1.2.3.4
+::11.22.33.44
+2001:db8::123.123.123.123
+::1234:5678:91.123.4.56
+::1234:5678:1.2.3.4
+2001:db8::1234:5678:5.6.7.8
@@ -0,0 +1,12 @@
+number
+"133"
+"1232.232"
+"2332,123"
+"not a number"
+"1,3%"
+"123%"
+"13  %"
+"76.234%"
+"not a number"
+"x"
+"123partly987"
@@ -0,0 +1,8 @@
+uuid
+ace1245c-3af5-11ec-8d3d-0242ac130003
+a568464e-a05d-412c-8b30-517a46c57d88
+notanuid
+d0d61836-3af5-11ec-8d3d-0242ac130003
+d0d61c6e-3af5-11ec-8d3d-0242ac130003
+343422-234324-234234-4234234-23432
+343422-234324-234234-4234234-234xxx32
				`@@ -0,0 +1 @@`
				`f2dq integration tests — dbt project for running f2dq dbt-package integration tests`