Initial publish of dbt_f2dq (f2dq) package v0.12.1.

Clean package repo for Gitea installs via packages.yml git URL.

Co-authored-by: Cursor <cursoragent@cursor.com>

integration_tests/models/metrics/f2dq_metrics.yml

@@ -0,0 +1,30 @@
+
+version: 2
+
+models:
+  - name: f2dq_metrics
+    tests:
+      - f2dq.metric_expression_is_true:
+          table: ref('buy_events')
+          metric: max_length
+          column_name: event_type
+          expression: value = 3
+
+      - f2dq.metric_equal_to:
+          table: ref('buy_events')
+          metric: max_length
+          column_name: event_type
+          value: 3
+
+      - f2dq.metric_in_range:
+          table: ref('buy_events')
+          metric: max_length
+          column_name: event_type
+          min_value: 3
+          max_value: 3
+
+      - f2dq.metric_expression_is_true:
+          table: ref('buy_events')
+          metric: row_count
+          expression: value > 0 and value < 10
+          condition: time_window_start >= '2021-05-02'

integration_tests/models/monitoring/schema.yml

@@ -0,0 +1,27 @@
+version: 2
+
+models:
+  - name: test_f2dq_metrics
+    tests:
+      - dbt_utils.equality:
+          compare_model: ref('expected_metrics')
+
+  - name: test_f2dq_z_score
+    tests:
+      - dbt_utils.equality:
+          compare_model: ref('expected_z_score')
+
+  - name: test_f2dq_anomalies
+    tests:
+      - dbt_utils.equality:
+          compare_model: ref('expected_anomalies')
+
+  - name: test_f2dq_test_history
+    tests:
+     - dbt_utils.equality:
+         compare_model: ref('expected_test_history')
+
+  - name: test_f2dq_table_samples
+    tests:
+     - dbt_utils.equality:
+         compare_model: ref('expected_table_samples')

integration_tests/models/monitoring/test_f2dq_anomalies.sql

@@ -0,0 +1,9 @@
+
+select
+    {{ clean_table_name('table_name') }} as table_name,
+    {{ clean_column_name('column_name') }} as column_name,
+    metric,
+    anomaly_detector,
+    interval_length_sec
+
+from {{ ref('f2dq_anomalies') }}

integration_tests/models/monitoring/test_f2dq_metrics.sql

@@ -0,0 +1,11 @@
+
+select
+    {{ clean_table_name('table_name') }} as table_name,
+    {{ clean_column_name('column_name') }} as column_name,
+    metric,
+    time_window_start,
+    time_window_end,
+    {{ to_big_integer('value') }},
+    interval_length_sec
+
+from {{ ref('f2dq_metrics') }}

integration_tests/models/monitoring/test_f2dq_table_samples.sql

@@ -0,0 +1,9 @@
+
+select 
+    {{ clean_table_name('table_name') }} as table_name,
+    length(sample_data) as sample_data_length
+from {{ ref('f2dq_table_samples') }}
+where {{ clean_table_name('table_name') }} != 'SAMPLE_WITHOUT_TIME_FILTER'
+
+-- SAMPLE_WITHOUT_TIME_FILTER because this table doesn't have a time filter, it's not possible to say how
+-- exactly the sampel of it should look like.

integration_tests/models/monitoring/test_f2dq_test_history.sql

@@ -0,0 +1,10 @@
+
+select
+    {{ clean_table_name('table_name') }} as table_name,
+    {{ clean_column_name('column_name') }} as column_name,
+    right(test_name, 15) as test_name,
+    status,
+    {{ clean_column_name('message') }} as message,
+    cast (failures_count as integer) as failures_count,
+    severity
+from {{ ref('f2dq_test_history') }}

integration_tests/models/monitoring/test_f2dq_z_score.sql

@@ -0,0 +1,22 @@
+
+{% set values_compare = [
+    'z_score_value',
+    'modified_z_score_value', 
+    'last_value',
+    'last_avg',
+    'last_stddev',
+    'last_median',
+    'last_iqr',
+    'last_median_absolute_deviation',
+    'last_mean_absolute_deviation',
+] %}
+
+select
+    {{ clean_table_name('table_name') }} as table_name,
+    {{ clean_column_name('column_name') }} as column_name,
+    metric,
+    time_window_end,
+    {% for col in values_compare %}{{ to_big_integer(col) }},{% endfor %}
+    interval_length_sec
+
+from {{ ref('f2dq_z_score') }}

integration_tests/models/public_macros/cleaning/sanitized_user_data.sql

@@ -0,0 +1,4 @@
+select
+    {{ f2dq.clean_capitalize_words(f2dq.clean_additional_whitespaces('full_name')) }} as full_name,
+    {{ f2dq.clean_blacklist('email', ['^[a-zA-Z0-9_.+-]+'], '*****') }} as email
+from {{ ref('sample_user_data') }}

integration_tests/models/public_macros/cleaning/schema.yml

@@ -0,0 +1,7 @@
+version: 2
+
+models:
+  - name: sanitized_user_data
+    tests:
+      - dbt_utils.equality:
+          compare_model: ref('expected_sample_user_data')

integration_tests/models/public_macros/filtering/deduplicated.sql

@@ -0,0 +1,16 @@
+with x as 
+    {{ f2dq.filter_remove_duplicates(
+        ref('duplicated'), ['transaction_id'], ['creation_time']) }}
+
+select *, 'take_first' as use_case from x
+
+union all
+
+select *, 'take_last' as use_case from {{ f2dq.filter_remove_duplicates(
+        ref('duplicated'), ['transaction_id'], ['creation_time desc']) }} duplicates
+
+
+union all
+
+select *, 'take_all_statuses' as use_case from {{ f2dq.filter_remove_duplicates(
+    ref('duplicated'), ['transaction_id', 'status'], ['creation_time desc']) }} duplicates

integration_tests/models/public_macros/filtering/duplicates.sql

@@ -0,0 +1,2 @@
+ {{ f2dq.filter_get_duplicates(
+        ref('duplicated'), ['transaction_id'], ['creation_time']) }}

integration_tests/models/public_macros/filtering/schema.yml

@@ -0,0 +1,11 @@
+version: 2
+
+models:
+  - name: deduplicated
+    tests:
+      - dbt_utils.equality:
+          compare_model: ref('expected_deduplicated')
+  - name: duplicates
+    tests:
+      - dbt_utils.equality:
+          compare_model: ref('expected_duplicates')

integration_tests/models/public_macros/normalizing/schema.yml

@@ -0,0 +1,7 @@
+version: 2
+
+models:
+  - name: us_states_normalized
+    tests:
+      - dbt_utils.equality:
+          compare_model: ref('expected_us_states_normalized')

integration_tests/models/public_macros/normalizing/us_states_normalized.sql

@@ -0,0 +1,29 @@
+with us_states_normalization_cte as (
+    select source, target from {{ ref('us_states_normalization') }}
+)
+
+{% set us_states_mapping = {'Ala.': 'Alabama', 'Alaska': 'Alaska', 'Ariz.': 'Arizona', 'Ark.': 'Arkansas', 'Calif.': 'California', 'Colo.': 'Colorado', 'Conn.': 'Connecticut',
+    'Del.': 'Delaware', 'D.C.': 'District of Columbia', 'Fla.': 'Florida', 'Ga.': 'Georgia', 'Hawaii': 'Hawaii', 'Idaho': 'Idaho', 'Ill.': 'Illinois', 'Ind.': 'Indiana',
+    'Iowa': 'Iowa', 'Kans.': 'Kansas', 'Ky.': 'Kentucky', 'La.': 'Louisiana', 'Maine': 'Maine', 'Md.': 'Maryland', 'Mass.': 'Massachusetts', 'Mich.': 'Michigan',
+    'Minn.': 'Minnesota', 'Miss.': 'Mississippi', 'Mo.': 'Missouri', 'Mont.': 'Montana', 'Nebr.': 'Nebraska', 'Nev.': 'Nevada', 'N.H.': 'New Hampshire', 'N.J.': 'New Jersey',
+    'N.M.': 'New Mexico', 'N.Y.': 'New York', 'N.C.': 'North Carolina', 'N.D.': 'North Dakota', 'Ohio': 'Ohio', 'Okla.': 'Oklahoma', 'Ore.': 'Oregon', 'Pa.': 'Pennsylvania',
+    'R.I.': 'Rhode Island', 'S.C.': 'South Carolina', 'S.D.': 'South Dakota', 'Tenn.': 'Tennessee', 'Tex.': 'Texas', 'Utah': 'Utah', 'Vt.': 'Vermont', 'Va.': 'Virginia',
+    'Wash.': 'Washington', 'W.Va.': 'West Virginia', 'Wis.': 'Wisconsin', 'Wyo.': 'Wyoming'}
+%}
+
+
+{# 
+    We have three ways of passing the source used for normalization
+        1. passing a dbt model using ref('') which is of type Relation.
+        2. passing a common table expression that contains the source mapping
+            Note: model or cte must include "source" and "target" column names used for normalization in 1. & 2. repectively
+        3. passing a dictionary of values that map from source -> target ie {[source]: [target]}
+ #}
+
+select distinct * from (
+    select state, code, state__normalized from {{ f2dq.normalize_values(ref('abbreviated_us_states'), 'state', ref('us_states_normalization')) }} s
+    union all
+    select state, code, state__normalized from {{ f2dq.normalize_values(ref('abbreviated_us_states'), 'state', 'us_states_normalization_cte') }} s
+    union all
+    select state, code, state__normalized from {{ f2dq.normalize_values(ref('abbreviated_us_states'), 'state', us_states_mapping) }} s
+) as normalized

integration_tests/models/public_macros/validating/schema.yml

@@ -0,0 +1,27 @@
+version: 2
+
+models:
+  - name: validated_emails
+    tests:
+      - dbt_utils.equality:
+          compare_model: ref('expected_validated_emails')
+
+  - name: validated_numbers
+    tests:
+      - dbt_utils.equality:
+          compare_model: ref('expected_validated_numbers')
+
+  - name: validated_date_and_time
+    tests:
+      - dbt_utils.equality:
+          compare_model: ref('expected_validated_date_and_time')
+
+  - name: validated_ips
+    tests:
+      - dbt_utils.equality:
+          compare_model: ref('expected_validated_ips')
+  
+  - name: validated_uuids
+    tests:
+      - dbt_utils.equality:
+          compare_model: ref('expected_validated_uuids')

integration_tests/models/public_macros/validating/validated_date_and_time.sql

@@ -0,0 +1,14 @@
+with 
+    all_rows as (
+        select * from {{ ref('validate_date_and_time') }}
+    )
+
+select *, 
+    case when {{ f2dq.valid_date_eu('date_time') }} then 1 else 0 end as valid_date_eu,
+    case when {{ f2dq.valid_date_us('date_time') }} then 1 else 0 end as valid_date_us,
+    case when {{ f2dq.valid_date_inverse('date_time') }} then 1 else 0 end as valid_date_inverse,
+    case when {{ f2dq.valid_date_iso_8601('date_time') }} then 1 else 0 end as valid_date_iso_8601,
+    case when {{ f2dq.valid_time_24h('date_time') }} then 1 else 0 end as valid_time_24h,
+    case when {{ f2dq.valid_time_12h('date_time') }} then 1 else 0 end as valid_time_12h,
+    case when {{ f2dq.valid_time('date_time') }} then 1 else 0 end as valid_time
+    from all_rows 

integration_tests/models/public_macros/validating/validated_emails.sql

@@ -0,0 +1,7 @@
+with 
+    all_emails as (
+        select * from {{ ref('validate_emails') }}
+    )
+
+select *, case when {{ f2dq.valid_email('email') }} then 1 else 0 end as email_valid
+    from all_emails 

integration_tests/models/public_macros/validating/validated_ips.sql

@@ -0,0 +1,10 @@
+with 
+    all_rows as (
+        select * from {{ ref('validate_ip') }}
+    )
+
+select *, 
+    case when {{ f2dq.valid_ip_v4('ip_address') }} then 1 else 0 end as valid_ip_v4,
+    case when {{ f2dq.valid_ip_v6('ip_address') }} then 1 else 0 end as valid_ip_v6,
+    case when {{ f2dq.valid_ip('ip_address') }} then 1 else 0 end as valid_ip
+    from all_rows 

integration_tests/models/public_macros/validating/validated_numbers.sql

@@ -0,0 +1,13 @@
+with 
+    all_num_rows as (
+        select * from {{ ref('validate_numbers') }}
+    )
+
+select *, 
+    case when {{ f2dq.valid_number('number') }} then 1 else 0 end as is_number,
+    case when {{ f2dq.valid_number_decimal_point('number') }} then 1 else 0 end as is_number_decimal_point,
+    case when {{ f2dq.valid_number_decimal_comma('number') }} then 1 else 0 end as is_number_decimal_comma,
+    case when {{ f2dq.valid_number_percentage('number') }} then 1 else 0 end as is_percentage,
+    case when {{ f2dq.valid_number_percentage_point('number') }} then 1 else 0 end as is_percentage_decimal_point,
+    case when {{ f2dq.valid_number_percentage_comma('number') }} then 1 else 0 end as is_percentage_decimal_comma
+    from all_num_rows 

integration_tests/models/public_macros/validating/validated_uuids.sql

@@ -0,0 +1,8 @@
+with 
+    all_rows as (
+        select * from {{ ref('validate_uuid') }}
+    )
+
+select *, 
+    case when {{ f2dq.valid_uuid('uuid') }} then 1 else 0 end as valid_uuid
+    from all_rows 

integration_tests/models/sources/schema.yml

@@ -0,0 +1,18 @@
+version: 2
+
+sources:
+  - name:  "{{ var('source_schema', target.schema) }}"
+    tables:
+      - name: f2dq_source_test_table
+        columns:
+          - name: number
+            tests:
+              - not_null
+              - unique
+          
+          - name: description
+            tests:
+              - not_null
+              - unique
+          
+      

integration_tests/models/transformed/buy_events.sql

@@ -0,0 +1,12 @@
+{{
+    config(
+        f2dq_monitored=true,
+        f2dq_time_filter='creation_time',
+        f2dq_anomaly_detector={'name': 'z_score', 'threshold': 0.5},
+        materialized='table',
+        tags=['testtag']
+    )
+}}
+select *
+from {{ ref('sample_with_anomaly') }}
+where event_type = 'buy'

integration_tests/models/transformed/schema.yml

@@ -0,0 +1,49 @@
+version: 2
+
+models:
+  - name: buy_events
+    tests:
+      - f2dq.assert_in_range:
+          metric: row_count
+          min_value: 0
+          max_value: 10
+
+      - f2dq.assert_equal:
+          metric: row_count
+          value: 1
+
+      - f2dq.assert_false:
+          metric: freshness
+          expression: value is null
+      
+      - f2dq.assert_greater_equal:
+          metric: my_distinct_table_rows
+          value: 10
+
+    columns:
+      - name: value1
+        tests:
+          - f2dq.assert_in_range:
+              metric: nulls_percent
+              min_value: 0
+              max_value: 10
+
+          - f2dq.assert_true:
+              metric: nulls_percent
+              expression: value = 0
+          
+          - f2dq.assert_less:
+              metric: min
+              value: 100
+              condition: time_window_start = '2021-05-02'
+          
+          - f2dq.assert_less_equal:
+              metric: min
+              value: 107
+
+      - name: value2
+        tests:
+          - f2dq.assert_greater_equal:
+              metric: min
+              value: 200
+              condition: time_window_start = '2021-05-02'