Initial publish of dbt_f2dq (f2dq) package v0.12.1.
Clean package repo for Gitea installs via packages.yml git URL. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
M7XBeast
Cursor
@@ -0,0 +1,34 @@
|
||||
select
|
||||
'anomaly' as type,
|
||||
{{ f2dq.clean_blacklist('table_name', ['"', '`'], '') }} as model,
|
||||
message,
|
||||
last_value_text as value,
|
||||
time_window_end
|
||||
from
|
||||
{{ ref(var('f2dq:anomalies_filtered')) }}
|
||||
union all
|
||||
|
||||
select
|
||||
'schema_change' as type,
|
||||
{{ f2dq.clean_blacklist('table_name', ['"', '`'], '') }} as model,
|
||||
{{ generate_schema_change_message('operation', 'column_name', 'prev_column_name', 'prev_data_type', 'data_type', 'detected_time') }} as message,
|
||||
'' as value,
|
||||
detected_time as time_window_end
|
||||
from {{ ref('f2dq_schema_changes') }}
|
||||
|
||||
union all
|
||||
|
||||
select
|
||||
'test' as type,
|
||||
table_name as model,
|
||||
{{ generate_failed_test_message('test_name', 'column_name') }},
|
||||
status as value,
|
||||
run_at as time_window_end
|
||||
|
||||
from {{ ref('f2dq_test_history') }}
|
||||
where
|
||||
status = 'Fail'
|
||||
or status = 'Error'
|
||||
{% if f2dq.f2dq_var('show_warns_as_alerts', false) %}
|
||||
or status = 'Warn'
|
||||
{% endif %}
|
||||
@@ -0,0 +1,55 @@
|
||||
{{
|
||||
config(
|
||||
materialized='view'
|
||||
)
|
||||
}}
|
||||
select
|
||||
z.id,
|
||||
z.table_name,
|
||||
z.column_name,
|
||||
z.metric,
|
||||
z.z_score_value,
|
||||
z.modified_z_score_value,
|
||||
m.anomaly_detector,
|
||||
z.last_value,
|
||||
z.last_avg,
|
||||
z.last_median,
|
||||
z.last_stddev,
|
||||
z.last_median_absolute_deviation,
|
||||
z.last_mean_absolute_deviation,
|
||||
z.last_iqr,
|
||||
z.last_first_quartile - (cast( {{ json_extract('m.anomaly_detector', 'whisker_boundary_multiplier') }} as {{numeric_type()}} ) * z.last_iqr) lower_bound,
|
||||
z.last_third_quartile + (cast( {{ json_extract('m.anomaly_detector', 'whisker_boundary_multiplier') }} as {{numeric_type()}} ) * z.last_iqr) upper_bound,
|
||||
z.last_first_quartile,
|
||||
z.last_third_quartile,
|
||||
z.time_window_end,
|
||||
z.interval_length_sec,
|
||||
z.computed_on,
|
||||
{{ f2dq.generate_anomaly_message('z.column_name', 'z.metric', 'z.last_value', 'z.last_avg') }} as message,
|
||||
{{ f2dq.generate_metric_value_text('z.metric', 'z.last_value') }} as last_value_text
|
||||
from
|
||||
{{ ref('f2dq_z_score')}} z
|
||||
left join {{ ref('f2dq_selected') }} m
|
||||
on {{ split_and_return_nth_value('table_name', '.', 1) }} = m.database
|
||||
and {{ split_and_return_nth_value('table_name', '.', 2) }} = m.schema
|
||||
and {{ split_and_return_nth_value('table_name', '.', 3) }} = m.name
|
||||
where
|
||||
case when (lower(coalesce({{ json_extract('m.anomaly_detector', 'direction') }}, 'both')) = 'up' and z.last_value > z.last_avg)
|
||||
or (lower(coalesce({{ json_extract('m.anomaly_detector', 'direction') }}, 'both')) = 'down' and z.last_value < z.last_avg)
|
||||
or (lower(coalesce({{ json_extract('m.anomaly_detector', 'direction') }}, 'both')) != 'up' and lower(coalesce({{ json_extract('m.anomaly_detector', 'direction') }}, 'both')) != 'down')
|
||||
then
|
||||
case
|
||||
when {{ json_extract('m.anomaly_detector', 'name') }} = 'z_score'
|
||||
then abs(z_score_value) > cast({{ json_extract('m.anomaly_detector', 'threshold') }} as {{ numeric_type() }})
|
||||
when {{ json_extract('m.anomaly_detector', 'name') }} = 'modified_z_score'
|
||||
then abs(modified_z_score_value) > cast( {{ json_extract('m.anomaly_detector', 'threshold') }} as {{numeric_type()}} )
|
||||
when {{ json_extract('m.anomaly_detector', 'name') }} = 'boxplot'
|
||||
then (
|
||||
z.last_value < z.last_first_quartile - (cast( {{ json_extract('m.anomaly_detector', 'whisker_boundary_multiplier') }} as {{numeric_type()}} ) * z.last_iqr)
|
||||
or
|
||||
z.last_value > z.last_third_quartile + (cast( {{ json_extract('m.anomaly_detector', 'whisker_boundary_multiplier') }} as {{numeric_type()}} ) * z.last_iqr)
|
||||
)
|
||||
else false
|
||||
end
|
||||
else false
|
||||
end
|
||||
@@ -0,0 +1,167 @@
|
||||
{{
|
||||
config(
|
||||
materialized='incremental',
|
||||
unique_key = 'id',
|
||||
on_schema_change='sync_all_columns',
|
||||
)
|
||||
}}
|
||||
|
||||
-- depends_on: {{ ref('f2dq_run_started_at') }}
|
||||
-- depends_on: {{ ref('f2dq_columns_over_time') }}
|
||||
-- depends_on: {{ ref('f2dq_monitored') }}
|
||||
-- depends_on: {{ ref('f2dq_selected') }}
|
||||
|
||||
{% if execute and not f2dq.in_compile() %}
|
||||
{% set last_data_points %}
|
||||
select
|
||||
distinct detected_time
|
||||
from {{ ref('f2dq_columns_over_time') }}
|
||||
order by
|
||||
detected_time desc limit 2;
|
||||
{% endset %}
|
||||
|
||||
{% set detected_times = run_query(last_data_points) %}
|
||||
|
||||
{% set times_list = detected_times.columns[0].values() %}
|
||||
{% set most_recent_time = times_list[0] %}
|
||||
|
||||
{% if times_list | length > 1 %}
|
||||
{% set prev_most_recent = times_list[1] %}
|
||||
{% else %}
|
||||
{% set prev_most_recent = times_list[0] %}
|
||||
{% endif %}
|
||||
{% else %}
|
||||
{% set times_list = () %}
|
||||
{% endif %}
|
||||
|
||||
{% if times_list == () %}
|
||||
{{
|
||||
f2dq.empty_table_generic([
|
||||
('id', 'string'),
|
||||
('table_name', 'string'),
|
||||
('operation', 'string'),
|
||||
('column_name', 'string'),
|
||||
('data_type', 'string'),
|
||||
('is_nullable', 'boolean'),
|
||||
('prev_column_name', 'string'),
|
||||
('prev_data_type', 'string'),
|
||||
('prev_is_nullable', 'boolean'),
|
||||
('detected_time', 'timestamp')
|
||||
])
|
||||
}}
|
||||
{% else %}
|
||||
|
||||
with curr_monitored_schema as (
|
||||
select * from {{ ref('f2dq_columns_over_time')}}
|
||||
where detected_time = cast('{{ most_recent_time }}' as {{ timestamp_type() }})
|
||||
and table_name in (
|
||||
select {{ full_table_name('name', 'schema', 'database') }} from {{ ref('f2dq_selected')}}
|
||||
)
|
||||
),
|
||||
|
||||
|
||||
prev_monitored_schema as (
|
||||
select * from {{ ref('f2dq_columns_over_time')}}
|
||||
where detected_time = cast('{{ prev_most_recent}}' as {{ timestamp_type() }})
|
||||
and table_name in (
|
||||
select {{ full_table_name('name', 'schema', 'database') }} from {{ ref('f2dq_selected')}}
|
||||
)
|
||||
),
|
||||
|
||||
all_changes as (
|
||||
(
|
||||
select
|
||||
curr.table_name as table_name,
|
||||
'type_change' as operation,
|
||||
curr.column_name as column_name,
|
||||
curr.data_type as data_type,
|
||||
curr.is_nullable as is_nullable,
|
||||
|
||||
prev.column_name as prev_column_name,
|
||||
prev.data_type as prev_data_type,
|
||||
prev.is_nullable as prev_is_nullable
|
||||
|
||||
from curr_monitored_schema curr inner join prev_monitored_schema prev on (curr.table_name = prev.table_name and curr.column_name = prev.column_name)
|
||||
where
|
||||
curr.data_type != prev.data_type or
|
||||
curr.is_nullable != prev.is_nullable
|
||||
)
|
||||
|
||||
union all
|
||||
|
||||
(
|
||||
|
||||
select
|
||||
curr.table_name as table_name,
|
||||
'column_added' as operation,
|
||||
curr.column_name as column_name,
|
||||
curr.data_type as data_type,
|
||||
curr.is_nullable as is_nullable,
|
||||
|
||||
null as prev_column_name,
|
||||
null as prev_data_type,
|
||||
null as prev_is_nullable
|
||||
|
||||
from curr_monitored_schema curr left join prev_monitored_schema prev on (curr.table_name = prev.table_name and curr.column_name = prev.column_name)
|
||||
where prev.table_name is null and prev.column_name is null
|
||||
{# note: when a column is added, make sure we only detect for models that were previously monitored,
|
||||
this avoids a situation where a newly monitored model has all its columns detected with 'column_added' operation#}
|
||||
and curr.table_name in (
|
||||
select table_name from prev_monitored_schema
|
||||
)
|
||||
|
||||
)
|
||||
|
||||
union all
|
||||
|
||||
(
|
||||
|
||||
select
|
||||
prev.table_name as table_name,
|
||||
'column_removed' as operation,
|
||||
null as column_name,
|
||||
null as data_type,
|
||||
null as is_nullable,
|
||||
|
||||
prev.column_name as prev_column_name,
|
||||
prev.data_type as prev_data_type,
|
||||
prev.is_nullable as prev_is_nullable
|
||||
|
||||
from prev_monitored_schema prev left join curr_monitored_schema curr on (curr.table_name = prev.table_name and curr.column_name = prev.column_name)
|
||||
where curr.table_name is null and curr.column_name is null
|
||||
|
||||
)
|
||||
),
|
||||
|
||||
all_with_time as (
|
||||
select
|
||||
all_changes.table_name,
|
||||
all_changes.operation,
|
||||
all_changes.column_name,
|
||||
all_changes.data_type,
|
||||
all_changes.is_nullable,
|
||||
all_changes.prev_column_name,
|
||||
all_changes.prev_data_type,
|
||||
all_changes.prev_is_nullable,
|
||||
cast({{dbt.current_timestamp_backcompat()}} as {{ timestamp_type() }}) as detected_time
|
||||
from all_changes
|
||||
)
|
||||
|
||||
select
|
||||
cast ({{ dbt_utils.generate_surrogate_key([
|
||||
'table_name',
|
||||
'column_name',
|
||||
'detected_time'
|
||||
]) }} as {{ string_type() }} ) as id,
|
||||
table_name,
|
||||
cast (operation as {{ string_type() }}) as operation,
|
||||
column_name,
|
||||
data_type,
|
||||
is_nullable,
|
||||
prev_column_name,
|
||||
prev_data_type,
|
||||
prev_is_nullable,
|
||||
detected_time
|
||||
from all_with_time
|
||||
|
||||
{% endif %}
|
||||
@@ -0,0 +1,13 @@
|
||||
{{
|
||||
config(
|
||||
materialized='view'
|
||||
)
|
||||
}}
|
||||
|
||||
select
|
||||
sum(case when status = 'Fail' then 1 else 0 end) as failed,
|
||||
sum(case when status = 'Pass' then 1 else 0 end) as passed,
|
||||
run_at
|
||||
from {{ ref ('f2dq_test_history') }}
|
||||
group by run_at
|
||||
order by run_at desc
|
||||
@@ -0,0 +1,76 @@
|
||||
{{
|
||||
config(
|
||||
materialized='incremental',
|
||||
unique_key = 'id',
|
||||
on_schema_change='sync_all_columns',
|
||||
)
|
||||
}}
|
||||
|
||||
with z_score_without_id as (
|
||||
|
||||
select
|
||||
stats.table_name as table_name,
|
||||
stats.column_name as column_name,
|
||||
stats.metric as metric,
|
||||
stats.interval_length_sec,
|
||||
(last_metric.last_value - stats.last_avg) / (stats.last_stddev + 0.0000000001) as z_score_value,
|
||||
case
|
||||
when stats.last_median_absolute_deviation = 0 then
|
||||
(last_metric.last_value - stats.last_median) / (1.253314 * (stats.last_mean_absolute_deviation + 0.0000000001))
|
||||
else
|
||||
(0.6745 * (last_metric.last_value - stats.last_median)) / (stats.last_median_absolute_deviation + 0.0000000001)
|
||||
end as modified_z_score_value,
|
||||
last_metric.last_value as last_value,
|
||||
stats.last_avg as last_avg,
|
||||
stats.last_median as last_median,
|
||||
stats.last_stddev as last_stddev,
|
||||
stats.last_median_absolute_deviation,
|
||||
stats.last_mean_absolute_deviation,
|
||||
stats.last_third_quartile - stats.last_first_quartile as last_iqr,
|
||||
stats.last_first_quartile,
|
||||
stats.last_third_quartile,
|
||||
{{ time_window_end() }} as time_window_end,
|
||||
cast( {{dbt.current_timestamp_backcompat()}} as {{ timestamp_type() }} ) as computed_on
|
||||
from
|
||||
{{ ref('f2dq_last_stats') }} as stats,
|
||||
{{ ref('f2dq_last_metrics') }} as last_metric
|
||||
where
|
||||
stats.table_name = last_metric.table_name and
|
||||
stats.column_name = last_metric.column_name and
|
||||
stats.metric = last_metric.metric and
|
||||
(
|
||||
stats.interval_length_sec = last_metric.interval_length_sec or
|
||||
(stats.interval_length_sec is null and last_metric.interval_length_sec is null)
|
||||
) and
|
||||
last_metric.last_value is not null and
|
||||
stats.last_avg is not null and
|
||||
stats.last_stddev is not null
|
||||
)
|
||||
|
||||
select
|
||||
cast ({{ dbt_utils.generate_surrogate_key([
|
||||
'table_name',
|
||||
'column_name',
|
||||
'metric',
|
||||
'interval_length_sec',
|
||||
'time_window_end'
|
||||
]) }} as {{ string_type() }} ) as id,
|
||||
table_name,
|
||||
column_name,
|
||||
metric,
|
||||
z_score_value,
|
||||
modified_z_score_value,
|
||||
last_value,
|
||||
last_avg,
|
||||
last_median,
|
||||
last_stddev,
|
||||
last_median_absolute_deviation,
|
||||
last_mean_absolute_deviation,
|
||||
last_iqr,
|
||||
last_first_quartile,
|
||||
last_third_quartile,
|
||||
time_window_end,
|
||||
interval_length_sec,
|
||||
computed_on
|
||||
|
||||
from z_score_without_id
|
||||
@@ -0,0 +1,7 @@
|
||||
{{
|
||||
config(
|
||||
materialized='table',
|
||||
)
|
||||
}}
|
||||
|
||||
select {{ run_started_at.timestamp() * 1000000 }} as run_started_at
|
||||
@@ -0,0 +1 @@
|
||||
{{ f2dq_last_base_metrics_part() }}
|
||||
@@ -0,0 +1 @@
|
||||
{{ f2dq_last_base_metrics_part() }}
|
||||
@@ -0,0 +1 @@
|
||||
{{ f2dq_last_base_metrics_part() }}
|
||||
@@ -0,0 +1 @@
|
||||
{{ f2dq_last_base_metrics_part() }}
|
||||
@@ -0,0 +1 @@
|
||||
{{ f2dq_last_base_metrics_thread(0)}}
|
||||
@@ -0,0 +1 @@
|
||||
{{ f2dq_last_base_metrics_thread(1)}}
|
||||
@@ -0,0 +1 @@
|
||||
{{ f2dq_last_base_metrics_thread(2)}}
|
||||
@@ -0,0 +1 @@
|
||||
{{ f2dq_last_base_metrics_thread(3)}}
|
||||
@@ -0,0 +1 @@
|
||||
{{ f2dq_last_table_samples() }}
|
||||
@@ -0,0 +1,7 @@
|
||||
{{
|
||||
config(
|
||||
materialized='table',
|
||||
)
|
||||
}}
|
||||
|
||||
{{ f2dq.empty_last_table_samples() }}
|
||||
@@ -0,0 +1,39 @@
|
||||
{{
|
||||
config(
|
||||
materialized='table',
|
||||
)
|
||||
}}
|
||||
|
||||
-- depends_on: {{ ref('f2dq_run_started_at') }}
|
||||
-- depends_on: {{ ref('f2dq_monitored') }}
|
||||
-- depends_on: {{ ref('f2dq_selected') }}
|
||||
|
||||
{% if execute %}
|
||||
{% set schemas = run_query(f2dq.get_schemas()) %}
|
||||
{% if schemas %}
|
||||
|
||||
with columns_from_select as (
|
||||
{% for row in schemas %}
|
||||
{% set schema_name = f2dq.name_in_db(f2dq.row_value(row, 'schema')) %}
|
||||
{{ get_monitored_columns(schema_name, f2dq.row_value(row, 'database')) }}
|
||||
{%- if not loop.last %} union all {%- endif %}
|
||||
{% endfor %}
|
||||
)
|
||||
|
||||
select
|
||||
cast (table_name as {{ string_type() }} ) as name,
|
||||
cast (table_schema as {{ string_type() }} ) as schema,
|
||||
cast (table_catalog as {{ string_type() }} ) as database,
|
||||
cast (column_name as {{ string_type() }} ) as column_name,
|
||||
cast (data_type as {{ string_type() }} ) as data_type,
|
||||
cast (case is_nullable when 'YES' then 1 else 0 end as {{ boolean_type() }} ) as is_nullable,
|
||||
cast ({{dbt.current_timestamp_backcompat()}} as {{ timestamp_type() }}) as computed_on
|
||||
from columns_from_select
|
||||
|
||||
{% else %}
|
||||
{{ empty_columns_table() }}
|
||||
{% endif %}
|
||||
|
||||
{% else %}
|
||||
{{ empty_columns_table() }}
|
||||
{% endif %}
|
||||
@@ -0,0 +1,23 @@
|
||||
{{
|
||||
config(
|
||||
materialized='table',
|
||||
unique_key = 'table_name',
|
||||
post_hook="{% if execute %}{{ pub_insert_into_f2dq_monitored() }}{% endif %}"
|
||||
)
|
||||
}}
|
||||
|
||||
{{
|
||||
f2dq.empty_table_generic([
|
||||
('name', 'string'),
|
||||
('schema', 'string'),
|
||||
('database', 'string'),
|
||||
('time_filter', 'string'),
|
||||
('metrics_groups', 'string'),
|
||||
('additional_metrics', 'string'),
|
||||
('metrics', 'string'),
|
||||
('columns', 'string'),
|
||||
('anomaly_detector', 'string'),
|
||||
('owners', 'string'),
|
||||
('selected', 'boolean')
|
||||
])
|
||||
}}
|
||||
@@ -0,0 +1,6 @@
|
||||
|
||||
select
|
||||
name, schema, database, time_filter, metrics, columns, anomaly_detector, owners
|
||||
from {{ ref('f2dq_monitored')}}
|
||||
where
|
||||
selected = true
|
||||
@@ -0,0 +1 @@
|
||||
select * from {{ ref('f2dq_base_metrics') }}
|
||||
@@ -0,0 +1,12 @@
|
||||
select
|
||||
table_name,
|
||||
column_name,
|
||||
metric,
|
||||
value as last_value,
|
||||
interval_length_sec,
|
||||
computed_on
|
||||
from
|
||||
{{ ref('f2dq_base_metrics') }}
|
||||
where
|
||||
time_window_end = {{- time_window_end() -}}
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
{% set columns_to_group_by = 'table_name, column_name, metric, interval_length_sec' %}
|
||||
|
||||
with median_value as (
|
||||
select distinct
|
||||
table_name,
|
||||
column_name,
|
||||
metric,
|
||||
interval_length_sec,
|
||||
avg(value) {% if target.type not in postgres_type_db() %} over(partition by {{ columns_to_group_by }}) {% endif %} as last_avg,
|
||||
{{ percentile(percentile_field='value', partition_field=columns_to_group_by, percent='0.25') }} as last_first_quartile,
|
||||
{{ percentile(percentile_field='value', partition_field=columns_to_group_by, percent='0.5') }} as last_median,
|
||||
{{ percentile(percentile_field='value', partition_field=columns_to_group_by, percent='0.75') }} as last_third_quartile
|
||||
from
|
||||
{{ ref('f2dq_base_metrics') }}
|
||||
where
|
||||
time_window_end > {{- anamaly_detection_time_window_start() -}} and
|
||||
time_window_end <= {{- time_window_end() -}}
|
||||
{% if target.type in postgres_type_db() %}
|
||||
group by
|
||||
{{ columns_to_group_by }}
|
||||
{% endif %}
|
||||
|
||||
), abs_deviation as (
|
||||
select
|
||||
s.table_name,
|
||||
s.column_name,
|
||||
s.metric,
|
||||
s.interval_length_sec,
|
||||
abs( s.value - mv.last_avg ) as absolute_deviation_from_mean,
|
||||
abs( s.value - mv.last_median ) as absolute_deviation_from_median
|
||||
from
|
||||
{{ ref('f2dq_base_metrics') }} s
|
||||
left join
|
||||
median_value mv
|
||||
on
|
||||
s.table_name = mv.table_name and
|
||||
s.column_name = mv.column_name and
|
||||
s.metric = mv.metric and
|
||||
s.interval_length_sec = mv.interval_length_sec
|
||||
where
|
||||
s.time_window_end > {{- anamaly_detection_time_window_start() -}} and
|
||||
s.time_window_end <= {{- time_window_end() -}}
|
||||
), median_abs_deviation as (
|
||||
select distinct
|
||||
table_name,
|
||||
column_name,
|
||||
metric,
|
||||
interval_length_sec,
|
||||
avg(absolute_deviation_from_mean) {% if target.type not in postgres_type_db() %} over(partition by {{ columns_to_group_by }}) {% endif %} as mean_absolute_deviation,
|
||||
{{ percentile(percentile_field='absolute_deviation_from_median', partition_field=columns_to_group_by, percent='0.5') }} as median_absolute_deviation
|
||||
from
|
||||
abs_deviation
|
||||
{% if target.type in postgres_type_db() %}
|
||||
group by
|
||||
{{ columns_to_group_by }}
|
||||
{% endif %}
|
||||
), stats as (
|
||||
select
|
||||
table_name,
|
||||
column_name,
|
||||
metric,
|
||||
stddev(value) as last_stddev,
|
||||
max(time_window_end) as last_metric_time,
|
||||
interval_length_sec,
|
||||
max(computed_on) as computed_on
|
||||
from
|
||||
{{ ref('f2dq_base_metrics') }}
|
||||
where
|
||||
time_window_end > {{- anamaly_detection_time_window_start() -}} and
|
||||
time_window_end <= {{- time_window_end() -}}
|
||||
group by
|
||||
{{ columns_to_group_by }}
|
||||
)
|
||||
|
||||
select
|
||||
s.table_name,
|
||||
s.column_name,
|
||||
s.metric,
|
||||
mv.last_avg,
|
||||
s.last_stddev,
|
||||
s.last_metric_time,
|
||||
s.interval_length_sec,
|
||||
s.computed_on,
|
||||
mv.last_median,
|
||||
mv.last_first_quartile,
|
||||
mv.last_third_quartile,
|
||||
md.median_absolute_deviation last_median_absolute_deviation,
|
||||
md.mean_absolute_deviation last_mean_absolute_deviation
|
||||
from
|
||||
stats s
|
||||
left join
|
||||
median_value mv
|
||||
on
|
||||
s.table_name = mv.table_name and
|
||||
s.column_name = mv.column_name and
|
||||
s.metric = mv.metric and
|
||||
s.interval_length_sec = mv.interval_length_sec
|
||||
left join
|
||||
median_abs_deviation md
|
||||
on
|
||||
s.table_name = md.table_name and
|
||||
s.column_name = md.column_name and
|
||||
s.metric = md.metric and
|
||||
s.interval_length_sec = md.interval_length_sec
|
||||
@@ -0,0 +1,54 @@
|
||||
{{
|
||||
config(
|
||||
materialized='incremental',
|
||||
unique_key = 'id',
|
||||
on_schema_change='sync_all_columns',
|
||||
)
|
||||
}}
|
||||
|
||||
-- depends_on: {{ ref('f2dq_columns') }}
|
||||
-- depends_on: {{ ref('f2dq_last_base_metrics_thread0') }}
|
||||
-- depends_on: {{ ref('f2dq_last_base_metrics_thread1') }}
|
||||
-- depends_on: {{ ref('f2dq_last_base_metrics_thread2') }}
|
||||
-- depends_on: {{ ref('f2dq_last_base_metrics_thread3') }}
|
||||
-- depends_on: {{ ref('f2dq_last_base_metrics_part0') }}
|
||||
-- depends_on: {{ ref('f2dq_last_base_metrics_part1') }}
|
||||
-- depends_on: {{ ref('f2dq_last_base_metrics_part2') }}
|
||||
-- depends_on: {{ ref('f2dq_last_base_metrics_part3') }}
|
||||
-- depends_on: {{ ref('f2dq_run_started_at') }}
|
||||
-- depends_on: {{ ref('f2dq_monitored') }}
|
||||
-- depends_on: {{ ref('f2dq_selected') }}
|
||||
|
||||
with
|
||||
|
||||
with_time_window as (
|
||||
{% set parts = ['0','1','2','3'] %}
|
||||
{% for part in parts %}
|
||||
{% set ref_name = 'f2dq_last_base_metrics_part' + part %}
|
||||
select
|
||||
*,
|
||||
{{ time_window_start() }} as time_window_start,
|
||||
{{ time_window_end() }} as time_window_end
|
||||
from {{ ref(ref_name) }}
|
||||
{%- if not loop.last %} union all {%- endif %}
|
||||
{% endfor %}
|
||||
)
|
||||
select
|
||||
cast ({{ dbt_utils.generate_surrogate_key([
|
||||
'table_name',
|
||||
'column_name',
|
||||
'metric',
|
||||
'time_window_start',
|
||||
'time_window_end'
|
||||
]) }} as {{ string_type() }} ) as id,
|
||||
cast (table_name as {{ string_type() }} ) as table_name,
|
||||
cast (column_name as {{ string_type() }} ) as column_name,
|
||||
cast (metric as {{ string_type() }} ) as metric,
|
||||
cast (value as {{ numeric_type() }} ) as value,
|
||||
cast (time_window_start as {{ timestamp_type() }} ) as time_window_start,
|
||||
cast (time_window_end as {{ timestamp_type() }} ) as time_window_end,
|
||||
cast (
|
||||
{{ interval_length_sec('time_window_start', 'time_window_end') }} as {{ integer_type() }}
|
||||
) as interval_length_sec,
|
||||
cast ({{dbt.current_timestamp_backcompat()}} as {{ timestamp_type() }}) as computed_on
|
||||
from with_time_window
|
||||
@@ -0,0 +1,17 @@
|
||||
{{
|
||||
config(
|
||||
materialized='table',
|
||||
unique_key = 'table_name',
|
||||
on_schema_change='sync_all_columns',
|
||||
)
|
||||
}}
|
||||
|
||||
-- depends_on: {{ ref('f2dq_last_table_samples') }}
|
||||
-- depends_on: {{ ref('f2dq_last_table_samples_part') }}
|
||||
|
||||
select
|
||||
table_name,
|
||||
sample_data,
|
||||
cast ({{dbt.current_timestamp_backcompat()}} as {{ timestamp_type() }}) as sampled_on
|
||||
|
||||
from {{ ref('f2dq_last_table_samples_part') }}
|
||||
@@ -0,0 +1,35 @@
|
||||
{{
|
||||
config(
|
||||
materialized='incremental',
|
||||
unique_key = 'id',
|
||||
on_schema_change='sync_all_columns',
|
||||
)
|
||||
}}
|
||||
|
||||
|
||||
with columns as (
|
||||
|
||||
select
|
||||
{{ full_table_name('cols.name', 'cols.schema', 'cols.database') }} as table_name,
|
||||
cols.column_name,
|
||||
cols.data_type,
|
||||
cols.is_nullable,
|
||||
cast ({{dbt.current_timestamp_backcompat()}} as {{ timestamp_type() }} ) as detected_time
|
||||
from
|
||||
{{ ref('f2dq_columns')}} cols, {{ ref('f2dq_selected')}} tables
|
||||
where
|
||||
cols.name = tables.name and cols.schema = tables.schema and cols.database = tables.database
|
||||
)
|
||||
|
||||
select
|
||||
cast ({{ dbt_utils.generate_surrogate_key([
|
||||
'table_name',
|
||||
'column_name',
|
||||
'detected_time'
|
||||
]) }} as {{ string_type() }} ) as id,
|
||||
table_name,
|
||||
column_name,
|
||||
data_type,
|
||||
is_nullable,
|
||||
detected_time
|
||||
from columns
|
||||
Reference in New Issue
Block a user