fast2dev/dbt-f2dq
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial publish of dbt_f2dq (f2dq) package v0.12.1.

Browse Source 
Clean package repo for Gitea installs via packages.yml git URL.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
M7XBeast
2026-10-11 17:37:40 +03:00
co-authored by Cursor
commit ff096b51b5
188 changed files with 5251 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
macros/public/cleaning/clean_additional_whitespace.sql
+23
View File
@@ -0,0 +1,23 @@
{% macro clean_additional_whitespaces(column_name) %}
{{ adapter.dispatch('clean_additional_whitespaces', 'f2dq')(column_name) }}
{% endmacro %}
{% macro default__clean_additional_whitespaces(column_name) %}
trim(regexp_replace( {{ column_name }}, '\s\s+', ' '))
{% endmacro %}
{% macro postgres__clean_additional_whitespaces(column_name) %}
trim(regexp_replace( {{ column_name }}, '\s\s+', ' ', 'g'))
{% endmacro %}
{% macro redshift__clean_additional_whitespaces(column_name) %}
trim(regexp_replace( {{ column_name }}, '\\s\\s+', ' '))
{% endmacro %}
{% macro bigquery__clean_additional_whitespaces(column_name) %}
trim(regexp_replace( {{ column_name }}, r'\s\s+', ' '))
{% endmacro %}
{% macro snowflake__clean_additional_whitespaces(column_name) %}
trim(regexp_replace( {{ column_name }}, '\\s\\s+', ' '))
{% endmacro %}
macros/public/cleaning/clean_blacklist.sql
+31
View File
@@ -0,0 +1,31 @@
{% macro generate_blacklist_pattern(chars_to_blacklist) %}
{% set pattern = [] %}
{% for char in chars_to_blacklist %}
{% set expr = '(' + char + ')' %}
{% do pattern.append(expr) %}
{% endfor %}
{{ return(pattern | join('|')) }}
{% endmacro %}
{%- macro clean_blacklist(column_name, chars_to_blacklist, replacement) -%}
{% set pattern_string = f2dq.generate_blacklist_pattern(chars_to_blacklist) %}
{{ adapter.dispatch('clean_blacklist', 'f2dq')(column_name, pattern_string, replacement) }}
{%- endmacro -%}
{%- macro default__clean_blacklist(column_name, pattern_string, replacement) -%}
regexp_replace( {{ column_name }}, '{{ pattern_string }}', '{{ replacement }}')
{%- endmacro -%}
{%- macro postgres__clean_blacklist(column_name, pattern_string, replacement) -%}
regexp_replace( {{ column_name }}, '{{ pattern_string }}', '{{ replacement }}', 'g')
{%- endmacro -%}
{%- macro redshift__clean_blacklist(column_name, pattern_string, replacement) -%}
regexp_replace( {{ column_name }}, '{{ pattern_string }}', '{{ replacement }}')
{%- endmacro -%}
{%- macro bigquery__clean_blacklist(column_name, pattern_string, replacement) -%}
regexp_replace( {{ column_name }}, """{{ pattern_string }}""", '{{ replacement }}')
{%- endmacro -%}
macros/public/cleaning/clean_capitalize_words.sql
+3
View File
@@ -0,0 +1,3 @@
{% macro clean_capitalize_words(column_name) %}
initcap( {{column_name}} )
{% endmacro %}
macros/public/filtering/get_duplicates.sql
+23
View File
@@ -0,0 +1,23 @@
{# https://github.com/fast2dev/f2dq/issues/143 #}
{#
macro returns rows with she same key set (unique_cols)
along with the fields of the base model duplicates information added:
f2dq_duplicate_count - total number of duplicates with the same current key set
f2dq_duplicate_row_number - number of a duplicate row inside the group of duplicates with the same current key set
#}
{% macro filter_get_duplicates(relation, unique_cols, sort_columns) %}
(
with duplication_context as (
{{f2dq.add_duplication_context(relation, unique_cols, sort_columns)}}
),
duplicate_rows as (
select * from duplication_context where f2dq_duplicate_group_row_count > 1
)
{# return surrogate key as well? #}
select *
from duplicate_rows
)
{% endmacro %}
macros/public/filtering/remove_duplicates.sql
+15
View File
@@ -0,0 +1,15 @@
{% macro filter_remove_duplicates(relation, unique_cols, sort_columns) %}
(
with with_row_num as (
{{f2dq.add_duplication_context(relation, unique_cols, sort_columns)}}
),
one_row_num as (
select * from with_row_num where f2dq_duplicate_group_row_number = 1
)
select {{ dbt_utils.star(from=relation) }}
from one_row_num
)
{% endmacro %}
macros/public/normalizing/normalize_values.sql
+32
View File
@@ -0,0 +1,32 @@
{% macro is_dbt_relation(obj) %}
{{ return (obj is mapping and obj.get('metadata', {}).get('type', '').endswith('Relation') )}}
{% endmacro %}
{% macro normalize_expression_cte(reference_table) %}
with target_table as (
{% if f2dq.is_dbt_relation(reference_table) or reference_table is string %}
select * from {{ reference_table }}
{% elif reference_table is mapping %}
{% for key, value in reference_table.items() %}
select '{{key}}' as source, '{{value}}' as target
{% if not loop.last %}union all{% endif %}
{% endfor %}
{% endif %}
)
{% endmacro %}
{%- macro normalize_values(source_relation, column_name, reference_table) -%}
(
{{ f2dq.normalize_expression_cte(reference_table) }}
select s.*,
case when t.source is null
then s.{{column_name}}
else t.target
end as {{ column_name + '__normalized'}}
from {{ source_relation }} s
left join target_table t
on t.source = s.{{column_name}}
)
{%- endmacro -%}
macros/public/store/export_alerts.sql
+22
View File
@@ -0,0 +1,22 @@
{% macro export_alerts(start_date, end_date, alerts_path=None, monitored_path=None) %}
{% set alerts_query %}
select
type as {{ f2dq.quote_column('type') }},
model as {{ f2dq.quote_column('model') }},
message as {{ f2dq.quote_column('message') }},
value as {{ f2dq.quote_column('value') }},
{{ format_timestamp('time_window_end')}} as {{ f2dq.quote_column('time_window_end') }}
from {{ ref('f2dq_alerts') }}
where
case
when type = 'anomaly' then {{ in_date_window('time_window_end', start_date, end_date) }}
else {{ in_date_window('time_window_end', start_date, none) }}
end
order by time_window_end desc
{% endset %}
{% set query_result = run_query(alerts_query) %}
{% set alerts_file_path = alerts_path or 'target/f2dq/alerts.json' %}
{% do query_result.to_json(alerts_file_path) %}
{{ save_monitored(monitored_path) }}
{% endmacro %}
macros/public/store/export_table_samples.sql
+15
View File
@@ -0,0 +1,15 @@
{% macro export_table_samples(start_date, end_date, table_samples_path=None) %}
{% set table_samples_query %}
select
lower(table_name) as table_name,
sample_data,
sampled_on
from
{{ ref('f2dq_table_samples') }}
{% endset %}
{% set query_result = run_query(table_samples_query) %}
{% set table_samples_file_path = table_samples_path or 'target/f2dq/table_samples.json' %}
{% do query_result.to_json(table_samples_file_path) %}
{% endmacro %}
macros/public/store/export_tests_history.sql
+25
View File
@@ -0,0 +1,25 @@
{% macro export_tests_history(start_date, end_date, tests_history_path=None) %}
{% set tests_history_query %}
select
table_name,
column_name,
test_name,
run_at,
status,
execution_time,
message,
failures_count,
failures_json,
failures_table,
severity,
compiled_sql
from
{{ ref('f2dq_test_history') }}
where {{ in_date_window('run_at', start_date, end_date) }}
{% endset %}
{% set query_result = run_query(tests_history_query) %}
{% set tests_history_file_path = tests_history_path or 'target/f2dq/tests_history.json' %}
{% do query_result.to_json(tests_history_file_path) %}
{% endmacro %}
macros/public/store/generate_overview.sql
+93
View File
@@ -0,0 +1,93 @@
{% macro overview_select_base(type, timestamp_col) %}
'{{ type }}' as {{ f2dq.quote_column('type') }},
table_name as {{ f2dq.quote_column('table_name') }},
column_name as {{ f2dq.quote_column('column_name') }},
{{ timestamp_col }} as {{ f2dq.quote_column('computed_on') }},
{% endmacro %}
{% macro generate_overview(start_date, end_date, interval, overview_path=None, monitored_path=None) %}
-- depends_on: {{ ref('f2dq_anomalies') }}
-- depends_on: {{ ref('f2dq_base_metrics') }}
-- depends_on: {{ ref('f2dq_schema_changes') }}
-- depends_on: {{ ref('f2dq_columns') }}
{# time grain is either days or hour #}
{% set time_grain, num_str = interval.split(':') %}
{% set num = num_str | int %}
{% if time_grain == 'hours' %}
{% set interval_length_sec = num * 3600 %}
{% elif time_grain == 'days'%}
{% set interval_length_sec = num * 3600 * 24 %}
{% else %}
{{ exceptions.raise_compiler_error("Invalid interval. Got: " ~ interval) }}
{% endif %}
{{ dbt_utils.log_info('[f2dq] interval length in seconds is ' ~ interval_length_sec) }}
{% set overview_query %}
with schema_changes_casted as (
select id, table_name, operation, column_name, data_type, {{ bool_to_string('is_nullable') }}, prev_column_name, prev_data_type, {{ bool_to_string('prev_is_nullable') }}, detected_time
from {{ ref('f2dq_schema_changes') }}
),
columns_casted as (
select {{ full_table_name('name', 'schema', 'database') }} as table_name, column_name, data_type, {{ bool_to_string('is_nullable') }}, computed_on
from {{ ref('f2dq_columns') }}
)
(
select
{{ overview_select_base('metric', 'computed_on')}}
{{ to_single_json(['metric', 'value', 'time_window_end', 'interval_length_sec']) }} as {{ f2dq.quote_column('data') }}
from
{{ ref('f2dq_base_metrics') }}
where {{ in_date_window('time_window_end', start_date, end_date) }}
and interval_length_sec = {{interval_length_sec}}
) union all
(
select
{{ overview_select_base('anomaly', 'computed_on')}}
{{ to_single_json(['id', 'metric', 'z_score_value', 'last_value', 'last_avg', 'last_stddev', 'time_window_end', 'interval_length_sec']) }} as {{ f2dq.quote_column('data') }}
from
{{ ref('f2dq_anomalies') }}
where {{ in_date_window('time_window_end', start_date, end_date) }}
and interval_length_sec = {{interval_length_sec}}
) union all
(
select
{{ overview_select_base('schema_change', 'detected_time')}}
{{ to_single_json(['id', 'operation', 'data_type', 'is_nullable', 'prev_column_name', 'prev_data_type', 'prev_is_nullable', 'detected_time']) }} as {{ f2dq.quote_column('data') }}
from
schema_changes_casted
where {{ in_date_window('detected_time', start_date, none) }}
) union all
(
select
{{ overview_select_base('schema', 'computed_on')}}
{{ to_single_json(['data_type', 'is_nullable']) }} as {{ f2dq.quote_column('data') }}
from
columns_casted
)
union all
(
select
'alert' as {{ f2dq.quote_column('type') }},
model as {{ f2dq.quote_column('table_name') }},
null as {{ f2dq.quote_column('column_name') }},
time_window_end as {{ f2dq.quote_column('computed_on') }},
{{ to_single_json(['type', 'model', 'message', 'value', 'time_window_end']) }} as {{ f2dq.quote_column('data') }}
from
{{ ref('f2dq_alerts') }}
where
case
when type = 'anomaly' then {{ in_date_window('time_window_end', start_date, end_date) }}
else {{ in_date_window('time_window_end', start_date, none) }}
end
)
order by {{ f2dq.quote_column('computed_on')}} desc
{% endset %}
{% set overview_result = run_query(overview_query) %}
{% set overview_file_path = overview_path or '../target/f2dq/overview.json' %}
{% do overview_result.to_json(overview_file_path) %}
{{ save_monitored(monitored_path) }}
{% endmacro %}
macros/public/validating/regex_dict.sql
+65
View File
@@ -0,0 +1,65 @@
{#
# This file contains significant part of code licensed under:
# Copyright 2020 Soda
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#}
{% macro get_regex_for(to_validate) %}
{% set regexp_dict = {
'number_whole': '^\-?[0-9]+$',
'number_decimal_point': '^\-?[0-9]+\.[0-9]+$',
'number_decimal_comma': '^\-?[0-9]+,[0-9]+$',
'number_percentage': '^\-?[0-9]+([\.,][0-9]+)? ?%$',
'number_percentage_point': '^\-?[0-9]+([\.][0-9]+)? ?%$',
'number_percentage_comma': '^\-?[0-9]+([,][0-9]+)? ?%$',
'date_eu': '^([1-9]|0[1-9]|[12][0-9]|3[01])[-\./]([1-9]|0[1-9]|1[012])[-\./](19|20)?[0-9][0-9]$',
'date_us': '^([1-9]|0[1-9]|1[012])[-\./]([1-9]|0[1-9]|[12][0-9]|3[01])[-\./](19|20)?[0-9][0-9]$',
'date_inverse': '^(19|20)[0-9][0-9][-\./]?([1-9]|0[1-9]|1[012])[-\./]?([1-9]|0[1-9]|[12][0-9]|3[01])$',
'time_24h': '^([01][0-9]|2[0-3]):([0-5][0-9])$',
'time_12h': '^(1[0-2]|0?[1-9]):[0-5][0-9]$',
'time': '^([0-9]|1[0-9]|2[0-4])[:-]([0-9]|[0-5][0-9])([:-]([0-9]|[0-5][0-9])(,[0-9]+)?)?$',
'date_iso_8601':
'^'
'([1-9][0-9]{3}-((0[1-9]|1[0-2])-(0[1-9]|1[0-9]|2[0-8])|(0[13-9]|1[0-2])-(29|30)|(0[13578]|1[02])-31)|'
'([1-9][0-9](0[48]|[2468][048]|[13579][26])|([2468][048]|[13579][26])00)-02-29)'
'T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\.[0-9]+)?'
'(Z|[+-][01][0-9]:[0-5][0-9])?'
'$',
'uuid': '^[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}$',
'ipv4_address': '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$',
'ipv6_address': '^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])){3}))|:)))(%.+)?$',
'email': '^[A-Za-z0-9.-_%]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,4}$',
} %}
{% set base_regex = regexp_dict[to_validate] %}
{% set qualifed_regex = adapter.dispatch('get_regex_for', 'f2dq')(base_regex) %}
{{ return(qualifed_regex) }}
{% endmacro %}
{% macro default__get_regex_for(pattern) %}
{{ return (pattern) }}
{% endmacro %}
{% macro redshift__get_regex_for(pattern) %}
{% set changed = modules.re.sub('\.', '\\.', pattern) %}
{% set changed = modules.re.sub('\-', '\\-', changed) %}
{{ return (changed) }}
{% endmacro %}
{% macro snowflake__get_regex_for(pattern) %}
{% set changed = modules.re.sub('\.', '\\.', pattern) %}
{% set changed = modules.re.sub('\-', '\\-', changed) %}
{{ return (changed) }}
{% endmacro %}
macros/public/validating/valid_with_regex.sql
+93
View File
@@ -0,0 +1,93 @@
{% macro valid_regex(column_name, to_validate) %}
{% set pattern = f2dq.get_regex_for(to_validate) %}
case when
{{ column_name }} is null then false
else {{ f2dq.regex_match_expression(column_name, pattern) }}
end
{% endmacro %}
{% macro valid_email(column_name) %}
{{ f2dq.valid_regex(column_name, 'email')}}
{% endmacro %}
{% macro valid_date_eu(column_name) %}
{{ f2dq.valid_regex(column_name, 'date_eu')}}
{% endmacro %}
{% macro valid_date_us(column_name) %}
{{ f2dq.valid_regex(column_name, 'date_us')}}
{% endmacro %}
{% macro valid_date_inverse(column_name) %}
{{ f2dq.valid_regex(column_name, 'date_inverse')}}
{% endmacro %}
{% macro valid_date_iso_8601(column_name) %}
{{ f2dq.valid_regex(column_name, 'date_iso_8601')}}
{% endmacro %}
{% macro valid_time_24h(column_name) %}
{{ f2dq.valid_regex(column_name, 'time_24h')}}
{% endmacro %}
{% macro valid_time_12h(column_name) %}
{{ f2dq.valid_regex(column_name, 'time_12h')}}
{% endmacro %}
{% macro valid_time(column_name) %}
{{ f2dq.valid_regex(column_name, 'time')}}
{% endmacro %}
{% macro valid_ip_v4(column_name) %}
{{ f2dq.valid_regex(column_name, 'ipv4_address')}}
{% endmacro %}
{% macro valid_ip_v6(column_name) %}
{{ f2dq.valid_regex(column_name, 'ipv6_address')}}
{% endmacro %}
{% macro valid_ip(column_name) %}
(
{{ f2dq.valid_regex(column_name, 'ipv4_address')}}
or
{{ f2dq.valid_regex(column_name, 'ipv6_address')}}
)
{% endmacro %}
{% macro valid_number(column_name) %}
{{ f2dq.valid_regex(column_name, 'number_whole')}}
{% endmacro %}
{% macro valid_number_decimal_point(column_name) %}
{{ f2dq.valid_regex(column_name, 'number_decimal_point')}}
{% endmacro %}
{% macro valid_number_decimal_comma(column_name) %}
{{ f2dq.valid_regex(column_name, 'number_decimal_comma')}}
{% endmacro %}
{% macro valid_number_percentage(column_name) %}
{{ f2dq.valid_regex(column_name, 'number_percentage')}}
{% endmacro %}
{% macro valid_number_percentage_point(column_name) %}
{{ f2dq.valid_regex(column_name, 'number_percentage_point')}}
{% endmacro %}
{% macro valid_number_percentage_comma(column_name) %}
{{ f2dq.valid_regex(column_name, 'number_percentage_comma')}}
{% endmacro %}
{% macro valid_phone(column_name) %}
{{ f2dq.valid_regex(column_name, 'phone')}}
{% endmacro %}
{% macro valid_uuid(column_name) %}
{{ f2dq.valid_regex(column_name, 'uuid')}}
{% endmacro %}
{% macro valid_credit_card(column_name) %}
{{ f2dq.valid_regex(column_name, 'credit_card_number')}}
{% endmacro %}