Initial publish of dbt_f2dq (f2dq) package v0.12.1.

Clean package repo for Gitea installs via packages.yml git URL.

Co-authored-by: Cursor <cursoragent@cursor.com>

models/metrics/final/f2dq_metrics.sql

@@ -0,0 +1 @@
+select * from {{ ref('f2dq_base_metrics') }}

models/metrics/for_anomalies/f2dq_last_metrics.sql

@@ -0,0 +1,12 @@
+select
+    table_name,
+    column_name,
+    metric,
+    value as last_value,
+    interval_length_sec,
+    computed_on
+from 
+    {{ ref('f2dq_base_metrics') }}
+where
+    time_window_end = {{- time_window_end() -}}
+

models/metrics/for_anomalies/f2dq_last_stats.sql

@@ -0,0 +1,104 @@
+{% set columns_to_group_by = 'table_name, column_name, metric, interval_length_sec' %}
+
+with median_value as (
+    select distinct
+        table_name,
+        column_name,
+        metric,
+        interval_length_sec,
+        avg(value) {% if target.type not in postgres_type_db() %} over(partition by {{ columns_to_group_by }}) {% endif %} as last_avg,
+        {{ percentile(percentile_field='value', partition_field=columns_to_group_by, percent='0.25') }} as last_first_quartile,
+        {{ percentile(percentile_field='value', partition_field=columns_to_group_by, percent='0.5') }} as last_median,
+        {{ percentile(percentile_field='value', partition_field=columns_to_group_by, percent='0.75') }} as last_third_quartile
+    from
+        {{ ref('f2dq_base_metrics') }}
+    where
+        time_window_end > {{- anamaly_detection_time_window_start() -}} and
+        time_window_end <= {{- time_window_end() -}}
+    {% if target.type in postgres_type_db() %} 
+        group by
+            {{ columns_to_group_by }}
+    {% endif %}
+    
+), abs_deviation as (
+    select 
+        s.table_name,
+        s.column_name,
+        s.metric,
+        s.interval_length_sec,
+        abs( s.value - mv.last_avg ) as absolute_deviation_from_mean,
+        abs( s.value - mv.last_median ) as absolute_deviation_from_median
+    from
+        {{ ref('f2dq_base_metrics') }} s
+    left join 
+        median_value mv
+        on
+            s.table_name = mv.table_name and
+            s.column_name = mv.column_name and
+            s.metric = mv.metric and
+            s.interval_length_sec = mv.interval_length_sec
+    where
+        s.time_window_end > {{- anamaly_detection_time_window_start() -}} and
+        s.time_window_end <= {{- time_window_end() -}}
+), median_abs_deviation as (
+    select distinct
+        table_name,
+        column_name,
+        metric,
+        interval_length_sec,
+        avg(absolute_deviation_from_mean) {% if target.type not in postgres_type_db() %} over(partition by {{ columns_to_group_by }}) {% endif %} as mean_absolute_deviation,
+        {{ percentile(percentile_field='absolute_deviation_from_median', partition_field=columns_to_group_by, percent='0.5') }} as median_absolute_deviation
+    from
+        abs_deviation
+    {% if target.type in postgres_type_db() %} 
+        group by
+            {{ columns_to_group_by }}
+    {% endif %}
+), stats as (
+    select
+        table_name,
+        column_name,
+        metric,
+        stddev(value) as last_stddev,
+        max(time_window_end) as last_metric_time,
+        interval_length_sec,
+        max(computed_on) as computed_on
+    from
+        {{ ref('f2dq_base_metrics') }}
+    where
+        time_window_end > {{- anamaly_detection_time_window_start() -}} and
+        time_window_end <= {{- time_window_end() -}}
+    group by
+        {{ columns_to_group_by }}
+)
+
+select
+    s.table_name,
+    s.column_name,
+    s.metric,
+    mv.last_avg,
+    s.last_stddev,
+    s.last_metric_time,
+    s.interval_length_sec,
+    s.computed_on,
+    mv.last_median,
+    mv.last_first_quartile,
+    mv.last_third_quartile,
+    md.median_absolute_deviation last_median_absolute_deviation,
+    md.mean_absolute_deviation last_mean_absolute_deviation
+from
+    stats s
+left join
+    median_value mv
+    on
+        s.table_name = mv.table_name and
+        s.column_name = mv.column_name and
+        s.metric = mv.metric and
+        s.interval_length_sec = mv.interval_length_sec
+left join
+    median_abs_deviation md
+    on 
+        s.table_name = md.table_name and
+        s.column_name = md.column_name and
+        s.metric = md.metric and
+        s.interval_length_sec = md.interval_length_sec

models/metrics/types/base/f2dq_base_metrics.sql

@@ -0,0 +1,54 @@
+{{
+    config(
+        materialized='incremental',
+        unique_key = 'id',
+        on_schema_change='sync_all_columns',
+    )
+}}
+
+-- depends_on: {{ ref('f2dq_columns') }}
+-- depends_on: {{ ref('f2dq_last_base_metrics_thread0') }}
+-- depends_on: {{ ref('f2dq_last_base_metrics_thread1') }}
+-- depends_on: {{ ref('f2dq_last_base_metrics_thread2') }}
+-- depends_on: {{ ref('f2dq_last_base_metrics_thread3') }}
+-- depends_on: {{ ref('f2dq_last_base_metrics_part0') }}
+-- depends_on: {{ ref('f2dq_last_base_metrics_part1') }}
+-- depends_on: {{ ref('f2dq_last_base_metrics_part2') }}
+-- depends_on: {{ ref('f2dq_last_base_metrics_part3') }}
+-- depends_on: {{ ref('f2dq_run_started_at') }}
+-- depends_on: {{ ref('f2dq_monitored') }}
+-- depends_on: {{ ref('f2dq_selected') }}
+
+with 
+
+with_time_window as (
+    {% set parts = ['0','1','2','3'] %}
+    {% for part in parts %}
+        {% set ref_name = 'f2dq_last_base_metrics_part' + part %}
+        select
+            *,
+            {{ time_window_start() }} as time_window_start,
+            {{ time_window_end() }} as time_window_end
+        from {{ ref(ref_name) }}
+        {%- if not loop.last %} union all {%- endif %}
+    {% endfor %}
+)
+select
+    cast ({{ dbt_utils.generate_surrogate_key([
+        'table_name',
+        'column_name',
+        'metric',
+        'time_window_start',
+        'time_window_end'
+    ]) }} as {{ string_type() }} ) as id,
+    cast (table_name as {{ string_type() }} ) as table_name,
+    cast (column_name as {{ string_type() }} ) as column_name,
+    cast (metric as {{ string_type() }} ) as metric,
+    cast (value as {{ numeric_type() }} ) as value,
+    cast (time_window_start as {{ timestamp_type() }} ) as time_window_start,
+    cast (time_window_end as {{ timestamp_type() }} ) as time_window_end,
+    cast (
+        {{ interval_length_sec('time_window_start', 'time_window_end') }} as {{ integer_type() }}
+    ) as interval_length_sec,
+    cast ({{dbt.current_timestamp_backcompat()}} as {{ timestamp_type() }}) as computed_on
+from with_time_window

models/metrics/types/samples/f2dq_table_samples.sql

@@ -0,0 +1,17 @@
+{{
+    config(
+        materialized='table',
+        unique_key = 'table_name',
+        on_schema_change='sync_all_columns',
+    )
+}}
+
+-- depends_on: {{ ref('f2dq_last_table_samples') }}
+-- depends_on: {{ ref('f2dq_last_table_samples_part') }}
+
+select
+    table_name,
+    sample_data,
+    cast ({{dbt.current_timestamp_backcompat()}} as {{ timestamp_type() }}) as sampled_on
+
+from {{ ref('f2dq_last_table_samples_part') }}

models/metrics/types/schema/f2dq_columns_over_time.sql

@@ -0,0 +1,35 @@
+{{
+    config(
+        materialized='incremental',
+        unique_key = 'id',
+        on_schema_change='sync_all_columns',
+    )
+}}
+
+
+with columns as (
+
+select
+    {{ full_table_name('cols.name', 'cols.schema', 'cols.database') }} as table_name,
+    cols.column_name,
+    cols.data_type,
+    cols.is_nullable,
+    cast ({{dbt.current_timestamp_backcompat()}} as {{ timestamp_type() }} ) as detected_time
+from
+    {{ ref('f2dq_columns')}} cols, {{ ref('f2dq_selected')}} tables
+where
+    cols.name = tables.name and cols.schema = tables.schema and cols.database = tables.database
+)
+
+select
+    cast ({{ dbt_utils.generate_surrogate_key([
+      'table_name',
+      'column_name',
+      'detected_time'
+    ]) }} as {{ string_type() }} ) as id,
+    table_name,
+    column_name,
+    data_type,
+    is_nullable,
+    detected_time
+from columns