Initial publish of dbt_f2dq (f2dq) package v0.12.1.

Clean package repo for Gitea installs via packages.yml git URL. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-10-11 17:37:40 +03:00
commit ff096b51b5
188 changed files with 5251 additions and 0 deletions
@@ -0,0 +1,22 @@
+{% macro export_alerts(start_date, end_date, alerts_path=None, monitored_path=None) %}
+    {% set alerts_query %}
+        select
+            type as {{ f2dq.quote_column('type') }},
+            model as {{ f2dq.quote_column('model') }},
+            message as {{ f2dq.quote_column('message') }},
+            value as {{ f2dq.quote_column('value') }},
+            {{ format_timestamp('time_window_end')}} as {{ f2dq.quote_column('time_window_end') }} 
+        from {{ ref('f2dq_alerts') }}
+        where
+            case
+                when type = 'anomaly' then {{ in_date_window('time_window_end', start_date, end_date) }}
+                else {{ in_date_window('time_window_end', start_date, none) }}
+            end
+        order by time_window_end desc
+    {% endset %}
+
+    {% set query_result = run_query(alerts_query) %}
+    {% set alerts_file_path = alerts_path or 'target/f2dq/alerts.json' %}
+    {% do query_result.to_json(alerts_file_path) %}
+    {{ save_monitored(monitored_path) }}
+{% endmacro %}
@@ -0,0 +1,15 @@
+{% macro export_table_samples(start_date, end_date, table_samples_path=None) %}
+    {% set table_samples_query %}
+        select
+            lower(table_name) as table_name,
+            sample_data,
+            sampled_on
+        from
+            {{ ref('f2dq_table_samples') }}
+    {% endset %}
+
+    {% set query_result = run_query(table_samples_query) %}
+    {% set table_samples_file_path = table_samples_path or 'target/f2dq/table_samples.json' %}
+    {% do query_result.to_json(table_samples_file_path) %}
+
+{% endmacro %}
@@ -0,0 +1,25 @@
+{% macro export_tests_history(start_date, end_date, tests_history_path=None) %}
+    {% set tests_history_query %}
+        select
+            table_name,
+            column_name,
+            test_name,
+            run_at,
+            status,
+            execution_time, 
+            message, 
+            failures_count, 
+            failures_json, 
+            failures_table,
+            severity, 
+            compiled_sql
+        from
+            {{ ref('f2dq_test_history') }}
+        where {{ in_date_window('run_at', start_date, end_date) }}
+    {% endset %}
+
+    {% set query_result = run_query(tests_history_query) %}
+    {% set tests_history_file_path = tests_history_path or 'target/f2dq/tests_history.json' %}
+    {% do query_result.to_json(tests_history_file_path) %}
+
+{% endmacro %}
@@ -0,0 +1,93 @@
+
+{% macro overview_select_base(type, timestamp_col) %}
+    '{{ type }}' as {{ f2dq.quote_column('type') }},
+    table_name as {{ f2dq.quote_column('table_name') }},
+    column_name as {{ f2dq.quote_column('column_name') }},
+    {{ timestamp_col }} as {{ f2dq.quote_column('computed_on') }},
+{% endmacro %}
+
+{% macro generate_overview(start_date, end_date, interval, overview_path=None, monitored_path=None) %}
+-- depends_on: {{ ref('f2dq_anomalies') }}
+-- depends_on: {{ ref('f2dq_base_metrics') }}
+-- depends_on: {{ ref('f2dq_schema_changes') }}
+-- depends_on: {{ ref('f2dq_columns') }}
+
+    {# time grain is either days or hour #}
+    {% set time_grain, num_str = interval.split(':') %}
+    {% set num = num_str | int %}
+    {% if time_grain == 'hours' %}
+        {% set interval_length_sec = num * 3600 %}
+    {% elif time_grain == 'days'%}
+        {% set interval_length_sec = num * 3600 * 24 %}
+    {% else %}
+        {{ exceptions.raise_compiler_error("Invalid interval. Got: " ~ interval) }}
+    {% endif %}
+    {{ dbt_utils.log_info('[f2dq] interval length in seconds is ' ~ interval_length_sec) }}
+    {% set overview_query %}
+        with schema_changes_casted as (
+            select id, table_name, operation, column_name, data_type, {{ bool_to_string('is_nullable') }}, prev_column_name, prev_data_type, {{ bool_to_string('prev_is_nullable') }}, detected_time
+            from {{ ref('f2dq_schema_changes') }}
+        ),
+        columns_casted as (
+            select {{ full_table_name('name', 'schema', 'database') }} as table_name, column_name, data_type, {{ bool_to_string('is_nullable') }}, computed_on
+            from {{ ref('f2dq_columns') }} 
+        )
+        
+    (    
+        select
+            {{ overview_select_base('metric', 'computed_on')}}
+            {{ to_single_json(['metric', 'value', 'time_window_end', 'interval_length_sec']) }} as {{ f2dq.quote_column('data') }}
+        from
+            {{ ref('f2dq_base_metrics') }}
+            where {{ in_date_window('time_window_end', start_date, end_date) }}
+            and interval_length_sec = {{interval_length_sec}}
+    ) union all 
+    (
+        select
+            {{ overview_select_base('anomaly', 'computed_on')}}
+            {{ to_single_json(['id', 'metric', 'z_score_value', 'last_value', 'last_avg', 'last_stddev', 'time_window_end', 'interval_length_sec']) }} as {{ f2dq.quote_column('data') }}
+        from
+            {{ ref('f2dq_anomalies') }}
+            where {{ in_date_window('time_window_end', start_date, end_date) }}
+            and interval_length_sec = {{interval_length_sec}}
+    ) union all
+    (
+        select
+            {{ overview_select_base('schema_change', 'detected_time')}}
+            {{ to_single_json(['id', 'operation', 'data_type', 'is_nullable', 'prev_column_name', 'prev_data_type', 'prev_is_nullable', 'detected_time']) }} as {{ f2dq.quote_column('data') }}
+        from
+            schema_changes_casted
+            where {{ in_date_window('detected_time', start_date, none) }}
+    ) union all
+    (
+        select
+            {{ overview_select_base('schema', 'computed_on')}}
+            {{ to_single_json(['data_type', 'is_nullable']) }} as {{ f2dq.quote_column('data') }}
+        from
+            columns_casted
+    )
+    union all 
+    (
+        select 
+            'alert' as {{ f2dq.quote_column('type') }},
+            model as {{ f2dq.quote_column('table_name') }},
+            null as {{ f2dq.quote_column('column_name') }},
+            time_window_end as {{ f2dq.quote_column('computed_on') }},
+            {{ to_single_json(['type', 'model', 'message', 'value', 'time_window_end']) }} as {{ f2dq.quote_column('data') }}
+        from
+            {{ ref('f2dq_alerts') }}
+        where
+            case
+                when type = 'anomaly' then {{ in_date_window('time_window_end', start_date, end_date)  }} 
+                else {{ in_date_window('time_window_end', start_date, none) }}
+            end
+    )
+    order by {{ f2dq.quote_column('computed_on')}} desc
+    {% endset %}
+
+    {% set overview_result = run_query(overview_query) %}
+    {% set overview_file_path = overview_path or '../target/f2dq/overview.json' %}
+    {% do overview_result.to_json(overview_file_path) %}
+    {{ save_monitored(monitored_path) }}
+
+{% endmacro %}