Initial publish of dbt_f2dq (f2dq) package v0.12.1.

Clean package repo for Gitea installs via packages.yml git URL. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-11 17:37:40 +03:00
commit ff096b51b5
188 changed files with 5251 additions and 0 deletions
@@ -0,0 +1,91 @@
+
+{% macro f2dq_metric_max(context) %}
+    max({{context.column_name}})
+{% endmacro %}
+
+{% macro f2dq_metric_min(context) %}
+    min({{context.column_name}})
+{% endmacro %}
+
+{% macro f2dq_metric_avg(context) %}
+    avg(cast ({{context.column_name}} as {{ numeric_type() }}))
+{% endmacro %}
+
+{% macro f2dq_metric_stddev(context) %}
+    stddev(cast ( {{context.column_name}} as {{ numeric_type() }}))
+{% endmacro %}
+
+{% macro f2dq_metric_variance(context) %}
+    variance(cast ( {{context.column_name}} as {{ numeric_type() }}))
+{% endmacro %}
+
+{% macro f2dq_metric_max_length(context) %}
+    max(length({{context.column_name}}))
+{% endmacro %}
+
+{% macro f2dq_metric_min_length(context) %}
+    min(length({{context.column_name}}))
+{% endmacro %}
+
+{% macro f2dq_metric_avg_length(context) %}
+    avg(cast (length( {{context.column_name}} ) as {{ numeric_type() }}))
+{% endmacro %}
+
+{% macro f2dq_metric_nulls_count(context) %}
+    coalesce(
+        sum(
+            case when {{context.column_name}} is null
+                then 1
+            else 0
+            end
+        ), 0
+    )
+{% endmacro %}
+
+{% macro f2dq_metric_missing_count(context) %}
+    coalesce(
+        sum(
+            case 
+            when {{context.column_name}} is null
+                then 1
+            when {{context.column_name}} = ''
+                then 1
+            else 0
+            end
+        ), 0
+    )
+{% endmacro %}
+
+{% macro f2dq_metric_nulls_percent(context) %}
+    {{ percentage_formula(f2dq_metric_nulls_count(context), f2dq_metric_row_count()) }}
+{% endmacro %}
+
+{% macro f2dq_metric_missing_percent(context) %}
+    {{ percentage_formula(f2dq_metric_missing_count(context), f2dq_metric_row_count()) }}
+{% endmacro %}
+
+{% macro f2dq_metric_count_true(context) %}
+    COALESCE(
+        SUM(
+            CASE
+                WHEN {{ context.column_name }} IS TRUE THEN 1
+                ELSE 0
+            END
+        ),
+        0
+    )
+{% endmacro %}
+
+{% macro f2dq_metric_count_false(context) %}
+    COALESCE(
+        SUM(
+            CASE
+                WHEN {{ context.column_name }} IS FALSE THEN 1
+                ELSE 0
+            END
+        ),
+        0
+    )
+{% endmacro %}
+
+
@@ -0,0 +1,104 @@
+{% macro f2dq_metric_regex_count(column_name, pattern) %}
+    coalesce(
+        sum(
+            case when {{ regex_match_expression(column_name, pattern) }}
+                then 1
+            else 0
+            end
+        ), 0
+    )
+{% endmacro %}
+
+{% macro f2dq_metric_match_regex(context) %}
+    {{ f2dq_metric_regex_count(context.column_name, context.config.regex) }}
+{% endmacro %}
+
+{% macro f2dq_metric_match_regex_percent(context) %}
+    {{ percentage_formula(f2dq_metric_match_regex(context), f2dq_metric_row_count()) }}
+{% endmacro %}
+
+{% macro f2dq_metric_not_match_regex(context) %}
+    {{ f2dq_metric_row_count() }} - {{ f2dq_metric_regex_count(context.column_name, context.config.regex) }}
+{% endmacro %}
+
+{% macro f2dq_metric_not_match_regex_percent(context) %}
+    {{ percentage_formula(f2dq_metric_not_match_regex(context), f2dq_metric_row_count()) }}
+{% endmacro %}
+
+{% macro f2dq_metric_distinct_values(context) %}
+    {{ distinct_values(context) }}
+{% endmacro %}
+
+{% macro distinct_values(context) %}
+    {{ adapter.dispatch('distinct_values', 'f2dq')(context) }}
+{% endmacro %}
+
+{% macro default__distinct_values(context) %}
+    coalesce(
+        count(distinct {{ context.column_name }} )
+    , 0)
+{% endmacro %}
+
+{% macro postgres__distinct_values(context) %}
+    {# /* In postgres, its faster to count distinct values in a column by selecting then counting in separate steps */ #}
+    with temp_table as (
+            select distinct {{ context.column_name }} from {{ context.table_name }}
+            where {{ in_time_window(context.time_filter) }}
+        )
+    select coalesce(count(*), 0) from temp_table
+{% endmacro %}
+
+{% macro f2dq_metric_approx_distinct_values(context) %}
+    {{ approx_distinct_values(context) }}
+{% endmacro %}
+
+{% macro approx_distinct_values(context) %}
+    {{ adapter.dispatch('approx_distinct_values', 'f2dq')(context) }}
+{% endmacro %}
+
+{% macro default__approx_distinct_values(context) %}
+    {# /* No approximate distinct count in postgres so we default to using a distinct count */ #}
+    {{ f2dq_metric_distinct_values(context) }}
+{% endmacro %}
+
+{% macro redshift__approx_distinct_values(context) %}
+    approximate {{ f2dq_metric_distinct_values(context.column_name) }}
+{% endmacro %}
+
+{% macro bigquery__approx_distinct_values(context) %}
+    approx_count_distinct({{ context.column_name }})
+{% endmacro %}
+
+{% macro snowflake__approx_distinct_values(context) %}
+    approx_count_distinct({{ context.column_name }})
+{% endmacro %}
+
+{% macro f2dq_metric_duplicate_values(context) %}
+        with temp_table as (
+            select {{ context.column_name }} from {{ context.table_name }}
+            where {{ in_time_window(context.time_filter) }}
+            group by {{ context.column_name }}
+            having count(1) > 1
+        )
+        select coalesce(count(*), 0) from temp_table
+{% endmacro %}
+
+{% macro f2dq_metric_duplicate_rows(context) %}
+        with temp_table as (
+            select {{ context.column_name }}, count(1) as row_count from {{ context.table_name }}
+            where {{ in_time_window(context.time_filter) }}
+            group by {{ context.column_name }}
+            having count(1) > 1
+        )
+        select coalesce(sum(row_count), 0) from temp_table
+{% endmacro %}
+
+{% macro f2dq_metric_unique_rows(context) %}
+        with temp_table as (
+            select {{ context.column_name }}, count(1) as row_count from {{ context.table_name }}
+            where {{ in_time_window(context.time_filter) }}
+            group by {{ context.column_name }}
+            having count(1) = 1
+        )
+        select coalesce(sum(row_count), 0) from temp_table
+{% endmacro %}
@@ -0,0 +1,7 @@
+{% macro f2dq_metric_distinct_table_rows(context) %}
+    with temp_table AS (
+            select distinct * from {{ context.table_name }}
+            where {{ in_time_window(context.time_filter) }}
+        )
+    select coalesce(count(*), 0) FROM temp_table
+{% endmacro %}
@@ -0,0 +1,33 @@
+
+{% macro f2dq_metric_row_count(context) %}
+    count(1)
+{% endmacro %}
+
+{% macro f2dq_metric_freshness(context) %}
+    {{ freshness_expression(context.time_filter) }}
+{% endmacro %}
+
+{% macro freshness_expression(time_filter) %}
+    {# /* If time_filter is none, we are computing the metric globally. we set the value as null since a table without use of a time filter doesn't really have a freshness metric */ #}
+    {% if time_filter is none %}
+        cast(null as {{ numeric_type() }})
+    {% else %}
+        {{ adapter.dispatch('freshness_expression', 'f2dq')(time_filter) }}
+    {% endif %}
+{% endmacro %}
+
+{% macro default__freshness_expression(time_filter) %}
+    EXTRACT(EPOCH FROM ({{time_window_end()}} - max({{time_filter}})))
+{% endmacro %}
+
+{% macro bigquery__freshness_expression(time_filter) %}
+    TIMESTAMP_DIFF ( timestamp({{ time_window_end()}}), timestamp(max({{time_filter}})), SECOND)
+{% endmacro %}
+
+{% macro snowflake__freshness_expression(time_filter) %}
+    timediff(second, max({{time_filter}}), {{- time_window_end() -}})
+{% endmacro %}
+
+{% macro redshift__freshness_expression(time_filter) %}
+    DATEDIFF(second, max({{time_filter}}), {{- time_window_end() -}})
+{% endmacro %}
@@ -0,0 +1,102 @@
+{% macro metrics_base_expressions(model, columns, table_level=False) %}
+
+    {% set col_expr = [] %}
+
+    {% for col in columns %}
+        {% set column_name = f2dq.row_value(col, 'column_name') %}
+        {% do col_expr.extend(f2dq.metrics_base_expression_column_all(model, col)) %}
+    {% endfor %}
+
+    {% if table_level %}
+        {% do col_expr.extend(f2dq.metrics_base_expresion_table_all(model)) %}
+    {% endif %}
+
+    {{ return (col_expr) }}
+
+{% endmacro %}
+
+{% macro metrics_base_expression_column_all(model, column) %}
+
+    {%- set col_expr = [] %}
+    {%- set metrics_to_compute = [] %}
+    {% set column_name = f2dq.row_value(column, 'column_name') %}
+    {% set data_type = model.columns_info[column_name].data_type %}
+    {% do metrics_to_compute.extend(model.metrics.get('group').get('column', {}).get(data_type, [])) %}
+    {% do metrics_to_compute.extend(model.metrics.get('additional').get('column', {}).get(column_name, [])) %} 
+
+    {% for metric_value in metrics_to_compute %}
+        {% set metric_obj = f2dq.extract_metric_config(metric_value) %}
+        {% set expression = f2dq.metrics_base_expression_column(model, column_name, metric_obj['metric'], metric_obj['config']) %}
+        {% do col_expr.append({ 'expr': expression, 'col_name': column_name, 'metric': metric_obj['metric']}) %}
+    {% endfor %}
+
+    {{ return (col_expr) }}
+
+{% endmacro %}
+
+
+{% macro metrics_base_expresion_table_all(model) %}
+    {%- set table_expr = [] %}
+    {%- set metrics_to_compute = [] %}
+    {% do metrics_to_compute.extend(model.metrics.get('group').get('table', [])) %}
+    {% do metrics_to_compute.extend(model.metrics.get('additional').get('table', [])) %}
+
+    {% for metric_value in metrics_to_compute %}
+        {% set metric_obj = f2dq.extract_metric_config(metric_value) %}
+        {% set expression = f2dq.metrics_base_expression_table(model, metric_obj['metric'], metric_obj['config']) %}
+        {% do table_expr.append({ 'expr': expression, 'col_name': '', 'metric': metric_obj['metric']}) %}
+    {% endfor %}
+
+    {{ return (table_expr) }}
+
+{% endmacro %}
+
+{% macro metrics_base_expression_table(model, metric_name, config) %}
+    {% set metric_macro = f2dq.get_metric_macro(metric_name) %}
+    {% set context = {'time_filter': model.time_filter, 'metric_name': metric_name, 'config': config, 'table_name': model.table_name, 'column_name': none} %}
+
+    {{ metric_macro(context) }}
+
+{% endmacro %}
+
+
+{%- macro metrics_base_expression_column(model, column_name, metric_name, config) %}
+    {% set metric_macro = f2dq.get_metric_macro(metric_name) %}
+    {% set context = {'time_filter': model.time_filter, 'metric_name': metric_name, 'config': config, 'table_name': model.table_name, 'column_name': f2dq.quote_column_name(column_name)} %}
+
+    {{ metric_macro(context) }}
+
+{% endmacro %}
+
+{% macro extract_metric_config(metric_value) %}
+
+    {% set config = none %}
+
+    {% if metric_value is mapping %}
+        {% set metric = metric_value.keys() | first %}
+        {% if metric_value[metric] is none %}
+            {{ exceptions.raise_compiler_error("Empty configuration passed for metric: " ~ metric ~ ". If the metric doesn't use a config, please use the column name as a string.") }}
+        {% endif %}
+
+        {% set config = metric_value[metric] %}
+    {%- else %}
+        {% set metric = metric_value %}
+    {% endif %}
+
+    {{ return ({'metric': metric, 'config': config}) }}
+
+{% endmacro %}
+
+{%- macro get_metric_macro(metric_name) %}
+    {% set macro_name = 'f2dq_metric' + '_' + metric_name %}
+
+    {% if context['f2dq'].get(macro_name) %}
+        {% set metric_macro = context['f2dq'][macro_name] %}
+    {%- else %}
+        {% set metric_macro = context[project_name][macro_name] %}
+    {% endif %}
+
+    {{ return (metric_macro) }}
+
+{% endmacro %}
+
@@ -0,0 +1,31 @@
+{% macro f2dq_last_base_metrics_part() %}
+
+-- depends_on: {{ ref('f2dq_columns') }}
+
+{{
+    config(
+        materialized='table',
+    )
+}}
+
+{{ f2dq.empty_last_base_metrics() }}
+
+{% endmacro %}
+
+{% macro f2dq_last_base_metrics_thread(num) %}
+    {% set part_name = 'f2dq_last_base_metrics_part' ~ num %}
+    {{ f2dq.generate_depends(['f2dq_selected', 'f2dq_monitored', 'f2dq_columns', 'f2dq_run_started_at', part_name]) }}
+
+    {{
+        config(
+            materialized='table',
+        )
+    }}
+
+    {% if not f2dq.in_compile() %}
+        {{ f2dq.metrics_base_compute_for_thread(num, part_name) }}
+    {% endif %}
+
+    {{ f2dq.empty_last_base_metrics() }}
+
+{% endmacro %}
@@ -0,0 +1,75 @@
+{% macro metrics_base_compute_for_thread(thread_value, ref_model) %}
+    {%- set tables =  run_query(f2dq.get_tables()) %}
+    {%- for mtable in tables %}
+        -- we are splitting computing metrics to 4 different threads
+        {% set for_loop_mod = (loop.index % 4) %}
+        {% if for_loop_mod == thread_value %}
+            {% set model = get_model_config(mtable) %}
+
+            {% set columns_to_query = [] %}
+            {% set size = 0 %}
+
+            {% for column in model.columns %}
+                {% set column_name = f2dq.row_value(column, 'column_name') %}
+                
+                {% if should_compute_metric(model, column_name) %}
+                    {% do columns_to_query.append(column) %}
+                {% endif %}
+
+                {% set columns_size = columns_to_query| length %}
+
+                {% if columns_size == var('f2dq:max_columns_in_query') %}
+                    {%- set insert_stats_query = f2dq.metrics_base_insert(model, ref_model, columns_to_query) -%}
+
+                    {% if insert_stats_query %}
+                        {% do run_query(insert_stats_query) %}
+                    {% endif %}
+                    {% do columns_to_query.clear() %}
+                {% endif %}
+            {% endfor %}
+
+            {%- set insert_stats_query = f2dq.metrics_base_insert(model, ref_model, columns_to_query, table_level=True) -%}
+            {% do run_query(insert_stats_query) %}
+
+            {{ dbt_utils.log_info('[f2dq_log] - finished computing metrics for:' ~ model.model_name) }}
+        {% endif %}
+    {% endfor %}
+{% endmacro %}
+
+{% macro metrics_base_insert(model, ref_model, columns, table_level=False) %}
+
+    {% set col_exprs = f2dq.metrics_base_expressions(model, columns, table_level) %}
+    {% if col_exprs == [] %}
+        {{ return ('') }}
+    {% endif %}
+
+    insert into {{ ref(ref_model) }}
+    with temp_table_metrics as (
+    select 
+        {%- for col_expr in col_exprs %}
+            ( {{ col_expr.expr }} ) as {{ f2dq.quote_column_name(col_expr.col_name + '___' + col_expr.metric) }}
+            {%- if not loop.last %},{%- endif %}
+        {% endfor %}
+    from 
+        {{ model.table_name }}
+    where
+        {{ in_time_window(model.time_filter) }}
+    )
+
+    {%- for col_expr in col_exprs %}
+        {% set final_metric_name = get_final_metric_name(col_expr.metric, model.time_filter) %}
+        
+        select '{{model.table_name}}' as table_name, '{{ col_expr.col_name }}' as column_name, '{{ final_metric_name }}' as metric, {{ f2dq.quote_column_name(col_expr.col_name + '___' + col_expr.metric) }} as value
+        from temp_table_metrics
+        {% if not loop.last %}union all{% endif %}
+    {% endfor %}
+
+{% endmacro %}
+
+{% macro get_final_metric_name(metric_name, time_filter) %}
+    {% if time_filter is none %}
+        {{ return ('global__' + metric_name) }}
+    {% else %}
+        {{ return (metric_name) }}
+    {% endif %}
+{% endmacro %}