Initial publish of dbt_f2dq (f2dq) package v0.12.1.
Clean package repo for Gitea installs via packages.yml git URL. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
M7XBeast
Cursor
@@ -0,0 +1,34 @@
|
||||
select
|
||||
'anomaly' as type,
|
||||
{{ f2dq.clean_blacklist('table_name', ['"', '`'], '') }} as model,
|
||||
message,
|
||||
last_value_text as value,
|
||||
time_window_end
|
||||
from
|
||||
{{ ref(var('f2dq:anomalies_filtered')) }}
|
||||
union all
|
||||
|
||||
select
|
||||
'schema_change' as type,
|
||||
{{ f2dq.clean_blacklist('table_name', ['"', '`'], '') }} as model,
|
||||
{{ generate_schema_change_message('operation', 'column_name', 'prev_column_name', 'prev_data_type', 'data_type', 'detected_time') }} as message,
|
||||
'' as value,
|
||||
detected_time as time_window_end
|
||||
from {{ ref('f2dq_schema_changes') }}
|
||||
|
||||
union all
|
||||
|
||||
select
|
||||
'test' as type,
|
||||
table_name as model,
|
||||
{{ generate_failed_test_message('test_name', 'column_name') }},
|
||||
status as value,
|
||||
run_at as time_window_end
|
||||
|
||||
from {{ ref('f2dq_test_history') }}
|
||||
where
|
||||
status = 'Fail'
|
||||
or status = 'Error'
|
||||
{% if f2dq.f2dq_var('show_warns_as_alerts', false) %}
|
||||
or status = 'Warn'
|
||||
{% endif %}
|
||||
@@ -0,0 +1,55 @@
|
||||
{{
|
||||
config(
|
||||
materialized='view'
|
||||
)
|
||||
}}
|
||||
select
|
||||
z.id,
|
||||
z.table_name,
|
||||
z.column_name,
|
||||
z.metric,
|
||||
z.z_score_value,
|
||||
z.modified_z_score_value,
|
||||
m.anomaly_detector,
|
||||
z.last_value,
|
||||
z.last_avg,
|
||||
z.last_median,
|
||||
z.last_stddev,
|
||||
z.last_median_absolute_deviation,
|
||||
z.last_mean_absolute_deviation,
|
||||
z.last_iqr,
|
||||
z.last_first_quartile - (cast( {{ json_extract('m.anomaly_detector', 'whisker_boundary_multiplier') }} as {{numeric_type()}} ) * z.last_iqr) lower_bound,
|
||||
z.last_third_quartile + (cast( {{ json_extract('m.anomaly_detector', 'whisker_boundary_multiplier') }} as {{numeric_type()}} ) * z.last_iqr) upper_bound,
|
||||
z.last_first_quartile,
|
||||
z.last_third_quartile,
|
||||
z.time_window_end,
|
||||
z.interval_length_sec,
|
||||
z.computed_on,
|
||||
{{ f2dq.generate_anomaly_message('z.column_name', 'z.metric', 'z.last_value', 'z.last_avg') }} as message,
|
||||
{{ f2dq.generate_metric_value_text('z.metric', 'z.last_value') }} as last_value_text
|
||||
from
|
||||
{{ ref('f2dq_z_score')}} z
|
||||
left join {{ ref('f2dq_selected') }} m
|
||||
on {{ split_and_return_nth_value('table_name', '.', 1) }} = m.database
|
||||
and {{ split_and_return_nth_value('table_name', '.', 2) }} = m.schema
|
||||
and {{ split_and_return_nth_value('table_name', '.', 3) }} = m.name
|
||||
where
|
||||
case when (lower(coalesce({{ json_extract('m.anomaly_detector', 'direction') }}, 'both')) = 'up' and z.last_value > z.last_avg)
|
||||
or (lower(coalesce({{ json_extract('m.anomaly_detector', 'direction') }}, 'both')) = 'down' and z.last_value < z.last_avg)
|
||||
or (lower(coalesce({{ json_extract('m.anomaly_detector', 'direction') }}, 'both')) != 'up' and lower(coalesce({{ json_extract('m.anomaly_detector', 'direction') }}, 'both')) != 'down')
|
||||
then
|
||||
case
|
||||
when {{ json_extract('m.anomaly_detector', 'name') }} = 'z_score'
|
||||
then abs(z_score_value) > cast({{ json_extract('m.anomaly_detector', 'threshold') }} as {{ numeric_type() }})
|
||||
when {{ json_extract('m.anomaly_detector', 'name') }} = 'modified_z_score'
|
||||
then abs(modified_z_score_value) > cast( {{ json_extract('m.anomaly_detector', 'threshold') }} as {{numeric_type()}} )
|
||||
when {{ json_extract('m.anomaly_detector', 'name') }} = 'boxplot'
|
||||
then (
|
||||
z.last_value < z.last_first_quartile - (cast( {{ json_extract('m.anomaly_detector', 'whisker_boundary_multiplier') }} as {{numeric_type()}} ) * z.last_iqr)
|
||||
or
|
||||
z.last_value > z.last_third_quartile + (cast( {{ json_extract('m.anomaly_detector', 'whisker_boundary_multiplier') }} as {{numeric_type()}} ) * z.last_iqr)
|
||||
)
|
||||
else false
|
||||
end
|
||||
else false
|
||||
end
|
||||
@@ -0,0 +1,167 @@
|
||||
{{
|
||||
config(
|
||||
materialized='incremental',
|
||||
unique_key = 'id',
|
||||
on_schema_change='sync_all_columns',
|
||||
)
|
||||
}}
|
||||
|
||||
-- depends_on: {{ ref('f2dq_run_started_at') }}
|
||||
-- depends_on: {{ ref('f2dq_columns_over_time') }}
|
||||
-- depends_on: {{ ref('f2dq_monitored') }}
|
||||
-- depends_on: {{ ref('f2dq_selected') }}
|
||||
|
||||
{% if execute and not f2dq.in_compile() %}
|
||||
{% set last_data_points %}
|
||||
select
|
||||
distinct detected_time
|
||||
from {{ ref('f2dq_columns_over_time') }}
|
||||
order by
|
||||
detected_time desc limit 2;
|
||||
{% endset %}
|
||||
|
||||
{% set detected_times = run_query(last_data_points) %}
|
||||
|
||||
{% set times_list = detected_times.columns[0].values() %}
|
||||
{% set most_recent_time = times_list[0] %}
|
||||
|
||||
{% if times_list | length > 1 %}
|
||||
{% set prev_most_recent = times_list[1] %}
|
||||
{% else %}
|
||||
{% set prev_most_recent = times_list[0] %}
|
||||
{% endif %}
|
||||
{% else %}
|
||||
{% set times_list = () %}
|
||||
{% endif %}
|
||||
|
||||
{% if times_list == () %}
|
||||
{{
|
||||
f2dq.empty_table_generic([
|
||||
('id', 'string'),
|
||||
('table_name', 'string'),
|
||||
('operation', 'string'),
|
||||
('column_name', 'string'),
|
||||
('data_type', 'string'),
|
||||
('is_nullable', 'boolean'),
|
||||
('prev_column_name', 'string'),
|
||||
('prev_data_type', 'string'),
|
||||
('prev_is_nullable', 'boolean'),
|
||||
('detected_time', 'timestamp')
|
||||
])
|
||||
}}
|
||||
{% else %}
|
||||
|
||||
with curr_monitored_schema as (
|
||||
select * from {{ ref('f2dq_columns_over_time')}}
|
||||
where detected_time = cast('{{ most_recent_time }}' as {{ timestamp_type() }})
|
||||
and table_name in (
|
||||
select {{ full_table_name('name', 'schema', 'database') }} from {{ ref('f2dq_selected')}}
|
||||
)
|
||||
),
|
||||
|
||||
|
||||
prev_monitored_schema as (
|
||||
select * from {{ ref('f2dq_columns_over_time')}}
|
||||
where detected_time = cast('{{ prev_most_recent}}' as {{ timestamp_type() }})
|
||||
and table_name in (
|
||||
select {{ full_table_name('name', 'schema', 'database') }} from {{ ref('f2dq_selected')}}
|
||||
)
|
||||
),
|
||||
|
||||
all_changes as (
|
||||
(
|
||||
select
|
||||
curr.table_name as table_name,
|
||||
'type_change' as operation,
|
||||
curr.column_name as column_name,
|
||||
curr.data_type as data_type,
|
||||
curr.is_nullable as is_nullable,
|
||||
|
||||
prev.column_name as prev_column_name,
|
||||
prev.data_type as prev_data_type,
|
||||
prev.is_nullable as prev_is_nullable
|
||||
|
||||
from curr_monitored_schema curr inner join prev_monitored_schema prev on (curr.table_name = prev.table_name and curr.column_name = prev.column_name)
|
||||
where
|
||||
curr.data_type != prev.data_type or
|
||||
curr.is_nullable != prev.is_nullable
|
||||
)
|
||||
|
||||
union all
|
||||
|
||||
(
|
||||
|
||||
select
|
||||
curr.table_name as table_name,
|
||||
'column_added' as operation,
|
||||
curr.column_name as column_name,
|
||||
curr.data_type as data_type,
|
||||
curr.is_nullable as is_nullable,
|
||||
|
||||
null as prev_column_name,
|
||||
null as prev_data_type,
|
||||
null as prev_is_nullable
|
||||
|
||||
from curr_monitored_schema curr left join prev_monitored_schema prev on (curr.table_name = prev.table_name and curr.column_name = prev.column_name)
|
||||
where prev.table_name is null and prev.column_name is null
|
||||
{# note: when a column is added, make sure we only detect for models that were previously monitored,
|
||||
this avoids a situation where a newly monitored model has all its columns detected with 'column_added' operation#}
|
||||
and curr.table_name in (
|
||||
select table_name from prev_monitored_schema
|
||||
)
|
||||
|
||||
)
|
||||
|
||||
union all
|
||||
|
||||
(
|
||||
|
||||
select
|
||||
prev.table_name as table_name,
|
||||
'column_removed' as operation,
|
||||
null as column_name,
|
||||
null as data_type,
|
||||
null as is_nullable,
|
||||
|
||||
prev.column_name as prev_column_name,
|
||||
prev.data_type as prev_data_type,
|
||||
prev.is_nullable as prev_is_nullable
|
||||
|
||||
from prev_monitored_schema prev left join curr_monitored_schema curr on (curr.table_name = prev.table_name and curr.column_name = prev.column_name)
|
||||
where curr.table_name is null and curr.column_name is null
|
||||
|
||||
)
|
||||
),
|
||||
|
||||
all_with_time as (
|
||||
select
|
||||
all_changes.table_name,
|
||||
all_changes.operation,
|
||||
all_changes.column_name,
|
||||
all_changes.data_type,
|
||||
all_changes.is_nullable,
|
||||
all_changes.prev_column_name,
|
||||
all_changes.prev_data_type,
|
||||
all_changes.prev_is_nullable,
|
||||
cast({{dbt.current_timestamp_backcompat()}} as {{ timestamp_type() }}) as detected_time
|
||||
from all_changes
|
||||
)
|
||||
|
||||
select
|
||||
cast ({{ dbt_utils.generate_surrogate_key([
|
||||
'table_name',
|
||||
'column_name',
|
||||
'detected_time'
|
||||
]) }} as {{ string_type() }} ) as id,
|
||||
table_name,
|
||||
cast (operation as {{ string_type() }}) as operation,
|
||||
column_name,
|
||||
data_type,
|
||||
is_nullable,
|
||||
prev_column_name,
|
||||
prev_data_type,
|
||||
prev_is_nullable,
|
||||
detected_time
|
||||
from all_with_time
|
||||
|
||||
{% endif %}
|
||||
@@ -0,0 +1,13 @@
|
||||
{{
|
||||
config(
|
||||
materialized='view'
|
||||
)
|
||||
}}
|
||||
|
||||
select
|
||||
sum(case when status = 'Fail' then 1 else 0 end) as failed,
|
||||
sum(case when status = 'Pass' then 1 else 0 end) as passed,
|
||||
run_at
|
||||
from {{ ref ('f2dq_test_history') }}
|
||||
group by run_at
|
||||
order by run_at desc
|
||||
@@ -0,0 +1,76 @@
|
||||
{{
|
||||
config(
|
||||
materialized='incremental',
|
||||
unique_key = 'id',
|
||||
on_schema_change='sync_all_columns',
|
||||
)
|
||||
}}
|
||||
|
||||
with z_score_without_id as (
|
||||
|
||||
select
|
||||
stats.table_name as table_name,
|
||||
stats.column_name as column_name,
|
||||
stats.metric as metric,
|
||||
stats.interval_length_sec,
|
||||
(last_metric.last_value - stats.last_avg) / (stats.last_stddev + 0.0000000001) as z_score_value,
|
||||
case
|
||||
when stats.last_median_absolute_deviation = 0 then
|
||||
(last_metric.last_value - stats.last_median) / (1.253314 * (stats.last_mean_absolute_deviation + 0.0000000001))
|
||||
else
|
||||
(0.6745 * (last_metric.last_value - stats.last_median)) / (stats.last_median_absolute_deviation + 0.0000000001)
|
||||
end as modified_z_score_value,
|
||||
last_metric.last_value as last_value,
|
||||
stats.last_avg as last_avg,
|
||||
stats.last_median as last_median,
|
||||
stats.last_stddev as last_stddev,
|
||||
stats.last_median_absolute_deviation,
|
||||
stats.last_mean_absolute_deviation,
|
||||
stats.last_third_quartile - stats.last_first_quartile as last_iqr,
|
||||
stats.last_first_quartile,
|
||||
stats.last_third_quartile,
|
||||
{{ time_window_end() }} as time_window_end,
|
||||
cast( {{dbt.current_timestamp_backcompat()}} as {{ timestamp_type() }} ) as computed_on
|
||||
from
|
||||
{{ ref('f2dq_last_stats') }} as stats,
|
||||
{{ ref('f2dq_last_metrics') }} as last_metric
|
||||
where
|
||||
stats.table_name = last_metric.table_name and
|
||||
stats.column_name = last_metric.column_name and
|
||||
stats.metric = last_metric.metric and
|
||||
(
|
||||
stats.interval_length_sec = last_metric.interval_length_sec or
|
||||
(stats.interval_length_sec is null and last_metric.interval_length_sec is null)
|
||||
) and
|
||||
last_metric.last_value is not null and
|
||||
stats.last_avg is not null and
|
||||
stats.last_stddev is not null
|
||||
)
|
||||
|
||||
select
|
||||
cast ({{ dbt_utils.generate_surrogate_key([
|
||||
'table_name',
|
||||
'column_name',
|
||||
'metric',
|
||||
'interval_length_sec',
|
||||
'time_window_end'
|
||||
]) }} as {{ string_type() }} ) as id,
|
||||
table_name,
|
||||
column_name,
|
||||
metric,
|
||||
z_score_value,
|
||||
modified_z_score_value,
|
||||
last_value,
|
||||
last_avg,
|
||||
last_median,
|
||||
last_stddev,
|
||||
last_median_absolute_deviation,
|
||||
last_mean_absolute_deviation,
|
||||
last_iqr,
|
||||
last_first_quartile,
|
||||
last_third_quartile,
|
||||
time_window_end,
|
||||
interval_length_sec,
|
||||
computed_on
|
||||
|
||||
from z_score_without_id
|
||||
Reference in New Issue
Block a user