M7XBeastandCursor
109e009210
Generate Documentation / generate-docs (push) Canceled after 0s
Run package tests for all dbs / test-postgres (push) Canceled after 0s
Run package tests for all dbs / test-other-dbs (bigquery) (push) Canceled after 0s
Run package tests for all dbs / test-other-dbs (redshift) (push) Canceled after 0s
Run package tests for all dbs / test-other-dbs (snowflake) (push) Canceled after 0s
Run package tests for all dbs / clean-up-schemas (bigquery) (push) Canceled after 0s
Run package tests for all dbs / clean-up-schemas (redshift) (push) Canceled after 0s
Run package tests for all dbs / clean-up-schemas (snowflake) (push) Canceled after 0s
Metrics reported row_count 0/1 against tables that plainly had 10+ rows. 1. The metric bucket was [run_date - 1d, run_date) - it ENDED at midnight at the start of the run day. A model stamping `dbt_updated_at = CURRENT_DATETIME()` writes rows during the run, so every row fell outside the bucket that run had just computed. Incremental merges that re-stamp rows each run then pushed them into the wrong bucket or out of every bucket. The bucket now covers the run's calendar day: [midnight of run date, +1d). interval_length_sec stays 86400 and the daily series is unchanged. 2. There was no way to see a table as a whole. `metrics_base_insert` now also emits `global__<metric>` computed over the whole table (`where true`) alongside the windowed series, so metrics stay meaningful for models whose time_filter holds historical/backfilled timestamps that miss the current bucket. A model with no time filter already computed globally and is not duplicated. Disable with `f2dq:emit_global_metrics: false`. 3. overview.json metric rows dropped `time_window_start`, so no consumer could tell a bucket's width or spot an empty one. It is exported now. Also: keep the guard that skips save_test_history when the f2dq_test_history relation does not exist. The integration_tests project pins the original re_data bucket anchoring and `f2dq:emit_global_metrics: false` because its expected_*.csv seeds encode the values that bucket produces and cannot be regenerated without the upstream multi-database harness. Co-authored-by: Cursor <cursoragent@cursor.com>
94 lines
4.2 KiB
SQL
94 lines
4.2 KiB
SQL
|
|
{% macro overview_select_base(type, timestamp_col) %}
|
|
'{{ type }}' as {{ f2dq.quote_column('type') }},
|
|
table_name as {{ f2dq.quote_column('table_name') }},
|
|
column_name as {{ f2dq.quote_column('column_name') }},
|
|
{{ timestamp_col }} as {{ f2dq.quote_column('computed_on') }},
|
|
{% endmacro %}
|
|
|
|
{% macro generate_overview(start_date, end_date, interval, overview_path=None, monitored_path=None) %}
|
|
-- depends_on: {{ ref('f2dq_anomalies') }}
|
|
-- depends_on: {{ ref('f2dq_base_metrics') }}
|
|
-- depends_on: {{ ref('f2dq_schema_changes') }}
|
|
-- depends_on: {{ ref('f2dq_columns') }}
|
|
|
|
{# time grain is either days or hour #}
|
|
{% set time_grain, num_str = interval.split(':') %}
|
|
{% set num = num_str | int %}
|
|
{% if time_grain == 'hours' %}
|
|
{% set interval_length_sec = num * 3600 %}
|
|
{% elif time_grain == 'days'%}
|
|
{% set interval_length_sec = num * 3600 * 24 %}
|
|
{% else %}
|
|
{{ exceptions.raise_compiler_error("Invalid interval. Got: " ~ interval) }}
|
|
{% endif %}
|
|
{{ dbt_utils.log_info('[f2dq] interval length in seconds is ' ~ interval_length_sec) }}
|
|
{% set overview_query %}
|
|
with schema_changes_casted as (
|
|
select id, table_name, operation, column_name, data_type, {{ bool_to_string('is_nullable') }}, prev_column_name, prev_data_type, {{ bool_to_string('prev_is_nullable') }}, detected_time
|
|
from {{ ref('f2dq_schema_changes') }}
|
|
),
|
|
columns_casted as (
|
|
select {{ full_table_name('name', 'schema', 'database') }} as table_name, column_name, data_type, {{ bool_to_string('is_nullable') }}, computed_on
|
|
from {{ ref('f2dq_columns') }}
|
|
)
|
|
|
|
(
|
|
select
|
|
{{ overview_select_base('metric', 'computed_on')}}
|
|
{{ to_single_json(['metric', 'value', 'time_window_start', 'time_window_end', 'interval_length_sec']) }} as {{ f2dq.quote_column('data') }}
|
|
from
|
|
{{ ref('f2dq_base_metrics') }}
|
|
where {{ in_date_window('time_window_end', start_date, end_date) }}
|
|
and interval_length_sec = {{interval_length_sec}}
|
|
) union all
|
|
(
|
|
select
|
|
{{ overview_select_base('anomaly', 'computed_on')}}
|
|
{{ to_single_json(['id', 'metric', 'z_score_value', 'last_value', 'last_avg', 'last_stddev', 'time_window_end', 'interval_length_sec']) }} as {{ f2dq.quote_column('data') }}
|
|
from
|
|
{{ ref('f2dq_anomalies') }}
|
|
where {{ in_date_window('time_window_end', start_date, end_date) }}
|
|
and interval_length_sec = {{interval_length_sec}}
|
|
) union all
|
|
(
|
|
select
|
|
{{ overview_select_base('schema_change', 'detected_time')}}
|
|
{{ to_single_json(['id', 'operation', 'data_type', 'is_nullable', 'prev_column_name', 'prev_data_type', 'prev_is_nullable', 'detected_time']) }} as {{ f2dq.quote_column('data') }}
|
|
from
|
|
schema_changes_casted
|
|
where {{ in_date_window('detected_time', start_date, none) }}
|
|
) union all
|
|
(
|
|
select
|
|
{{ overview_select_base('schema', 'computed_on')}}
|
|
{{ to_single_json(['data_type', 'is_nullable']) }} as {{ f2dq.quote_column('data') }}
|
|
from
|
|
columns_casted
|
|
)
|
|
union all
|
|
(
|
|
select
|
|
'alert' as {{ f2dq.quote_column('type') }},
|
|
model as {{ f2dq.quote_column('table_name') }},
|
|
null as {{ f2dq.quote_column('column_name') }},
|
|
time_window_end as {{ f2dq.quote_column('computed_on') }},
|
|
{{ to_single_json(['type', 'model', 'message', 'value', 'time_window_end']) }} as {{ f2dq.quote_column('data') }}
|
|
from
|
|
{{ ref('f2dq_alerts') }}
|
|
where
|
|
case
|
|
when type = 'anomaly' then {{ in_date_window('time_window_end', start_date, end_date) }}
|
|
else {{ in_date_window('time_window_end', start_date, none) }}
|
|
end
|
|
)
|
|
order by {{ f2dq.quote_column('computed_on')}} desc
|
|
{% endset %}
|
|
|
|
{% set overview_result = run_query(overview_query) %}
|
|
{% set overview_file_path = overview_path or '../target/f2dq/overview.json' %}
|
|
{% do overview_result.to_json(overview_file_path) %}
|
|
{{ save_monitored(monitored_path) }}
|
|
|
|
{% endmacro %}
|