Initial publish of dbt_f2dq (f2dq) package v0.12.1.

Clean package repo for Gitea installs via packages.yml git URL.

Co-authored-by: Cursor <cursoragent@cursor.com>

macros/config/get_model_config.sql

@@ -0,0 +1,41 @@
+
+{% macro get_model_config(monitored) %}
+    {% set model = {} %}
+    {% do model.update({'name': f2dq.row_value(monitored, 'name')}) %}
+    {% do model.update({'schema': f2dq.row_value(monitored, 'schema')}) %}
+    {% do model.update({'database': f2dq.row_value(monitored, 'database')}) %}
+    {% do model.update({'time_filter': f2dq.row_value(monitored, 'time_filter')}) %}    
+    {% do model.update({'metrics': fromjson(f2dq.row_value(monitored, 'metrics'))}) %}
+    {% do model.update({'model_name': model.get('database') + '.' + model.get('schema') + '.' + model.get('name')}) %}
+    {% do model.update({'table_name': full_table_name_values(model.get('name'), model.get('schema'), model.get('database'))}) %}
+
+    {% set columns_db = f2dq.row_value(monitored, 'columns') %}
+
+    {% set column_list = fromjson(columns_db) if columns_db is not none else none %}
+    {% set columns_dict = f2dq.dict_from_list(column_list) %}
+
+    {% do model.update({'columns_dict': columns_dict}) %}
+    {% do model.update({'columns_compute_all': columns_dict is none}) %}
+
+    {% set columns_query %}
+        select * from {{ ref('f2dq_columns') }}
+        where name = '{{ model.name }}' and schema = '{{ model.schema }}' and database = '{{ model.database }}'
+    {% endset %}
+    {% set columns = run_query(columns_query) %}
+
+    {% set columns_info = {} %}
+    {% for col in columns %}
+        {% set column_name = f2dq.row_value(col, 'column_name') %}
+        {% set data_type = f2dq.get_column_type(col) %}
+        {% do columns_info.update({column_name: { 'data_type': data_type }}) %}
+    {% endfor %}
+
+    {% do model.update({'columns_info': columns_info}) %}
+    {% do model.update({'columns': columns}) %}
+
+    {{ return(model) }}
+{% endmacro %}
+
+{% macro should_compute_metric(model, column_name) %}
+    {{ return(model.columns_compute_all or model.columns_dict.get(column_name)) }}
+{% endmacro %}

macros/db/bigquery/quote_column_name.sql

@@ -0,0 +1,4 @@
+{% macro bigquery__quote_column_name(column_name) %}
+    {% set quoted_col_name = '`' + column_name + '`' %}
+    {{ return(quoted_col_name) }}
+{% endmacro %}

macros/db/bigquery/quote_string.sql

@@ -0,0 +1,4 @@
+
+{%- macro bigquery__quote_string(str) %}
+    r"""{{ str }}"""
+{% endmacro %}

macros/db/bigquery/split_and_return_nth_value.sql

@@ -0,0 +1,3 @@
+{% macro bigquery__split_and_return_nth_value(column_name, delimiter, ordinal) %}
+    split({{ f2dq.clean_blacklist(column_name, ['"', '`'], '') }}, '{{ delimiter }}')[ORDINAL( {{ ordinal }} )]
+{% endmacro %}

macros/db/core/escape.sql

@@ -0,0 +1,4 @@
+
+{% macro escape_seq_for_json(chr) %}{{adapter.dispatch('escape_seq_for_json', 'f2dq')(chr)}}{% endmacro %}
+
+{% macro default__escape_seq_for_json(chr) %}'\\\{{chr}}'{% endmacro %}

macros/db/core/identifier_mapping.sql

@@ -0,0 +1,10 @@
+
+{% macro name_in_db(name) %}
+    {% set translated = adapter.dispatch('name_in_db', 'f2dq')(name) %}
+    {{ return(translated) }}
+    
+{% endmacro %}
+
+{% macro default__name_in_db(name) %}
+    {{ return(name) }}
+{% endmacro %}

macros/db/core/quote_column_name.sql

@@ -0,0 +1,10 @@
+{% macro quote_column_name(column_name) %}
+    {% set col_name = adapter.dispatch('quote_column_name', 'f2dq')(column_name) %}
+    {{ return(col_name) }}
+{% endmacro %}
+
+
+{% macro default__quote_column_name(column_name) %}
+    {% set quoted_col_name = '"' + column_name + '"' %}
+    {{ return(quoted_col_name) }}
+{% endmacro %}

macros/db/core/quote_string.sql

@@ -0,0 +1,13 @@
+
+{%- macro quote_string(str) %}
+    {{ adapter.dispatch('quote_string', 'f2dq')(str) }}
+{% endmacro %}
+
+{%- macro default__quote_string(str) %}
+    $${{ str }}$$
+{% endmacro %}
+
+
+{%- macro quote_new_line() %}{{ adapter.dispatch('quote_new_line', 'f2dq')() }}{% endmacro %}
+
+{%- macro default__quote_new_line() %}'\134\134n'{% endmacro %}

macros/db/core/split_and_return_nth_value.sql

@@ -0,0 +1,7 @@
+{% macro split_and_return_nth_value(column_name, delimiter, ordinal) -%}
+    {{ adapter.dispatch('split_and_return_nth_value', 'f2dq')(column_name, delimiter, ordinal) }}
+{%- endmacro %}
+
+{% macro default__split_and_return_nth_value(column_name, delimiter, ordinal) -%}
+    split_part({{ f2dq.clean_blacklist(column_name, ['"', '`'], '') }}, '{{ delimiter }}', {{ ordinal }})
+{%- endmacro %}

macros/db/postgres/escape.sql

@@ -0,0 +1 @@
+{% macro postgres__escape_seq_for_json(chr) %}'\{{chr}}'{% endmacro %}

macros/db/postgres/postgres_type_db.sql

@@ -0,0 +1,3 @@
+{% macro postgres_type_db() %}
+    {{ ('postgres', 'greenplum') }}
+{% endmacro %}

macros/db/postgres/quote_string.sql

@@ -0,0 +1,3 @@
+
+
+{%- macro postgres__quote_new_line() %}'\\n'{% endmacro %}

macros/db/redshift/escape.sql

@@ -0,0 +1,2 @@
+
+{% macro redshift__escape_seq_for_json(chr) %}'\\\{{chr}}'{% endmacro %}

macros/db/redshift/quote_string.sql

@@ -0,0 +1,2 @@
+
+{%- macro redshift__quote_new_line() %}'\134\134n'{% endmacro %}

macros/db/snowflake/identifier_mapping.sql

@@ -0,0 +1,8 @@
+
+{% macro snowflake__name_in_db(name) %}
+    {% if name %}
+        {{ return (name.upper()) }}
+    {% else %}
+        {{ return (name) }}
+    {% endif %}
+{% endmacro %}

macros/meta/get_monitored.sql

@@ -0,0 +1,97 @@
+{% macro pub_monitored_from_graph() %}
+    {% set monitored = [] %}
+    {% set both = []%}
+    {% do both.extend(graph.nodes.values()) %}
+    {% do both.extend(graph.sources.values()) %}
+    {% set owners_config = f2dq.get_owners_config() %}
+
+    {% set select_var = f2dq.f2dq_var('select', none) %}
+    {% set select_all = true %}
+
+    {% set selected_nodes = none %}
+    {% set selected_tags = none %}
+
+    {% if select_var is not none %}
+        {% set select_all = false %}
+        {% set selected_nodes = dict() %}
+        {% set selected_tags = dict() %}
+
+        {% for el in select_var %}
+            {% if el.startswith('tag:') %}
+                {% do selected_tags.update({el[4:]: True}) %}
+            {% else %}
+                {% do selected_nodes.update({el: True}) %}
+            {% endif %}
+        {% endfor %}
+    {% endif %}
+
+    {% for el in both %}
+        {% if el.resource_type in ['model', 'seed', 'source'] %}
+            {% if f2dq.f2dq_node_config(el, 'monitored') %}
+                {% set target_name = el.identifier or el.alias or el.name %}
+
+                {% if select_all %}
+                    {% set selected = true %}
+                {% else %}
+                    {% set selected_name = selected_nodes.get(target_name, false) %}
+                    {% set selected_tag = [] %}
+
+                    {% for tag in el.tags %}
+                        {% if selected_tags.get(tag, false) %}
+                            {% do selected_tag.append(true) %}
+                        {% endif %}
+                    {% endfor %}
+                    
+                    {% set selected = selected_name or (selected_tag | length > 0) %}
+                {% endif %}
+
+                {% set metrics_groups = f2dq.f2dq_node_config(el, 'metrics_groups', f2dq.f2dq_var('default_metrics')) %}
+                {% set additional_metrics = f2dq.f2dq_node_config(el, 'metrics', {}) %}
+
+                {% do monitored.append({
+                    'name': f2dq.name_in_db(target_name),
+                    'schema': f2dq.name_in_db(el.schema),
+                    'database': f2dq.name_in_db(el.database),
+                    'time_filter': f2dq.f2dq_node_config(el, 'time_filter', none),
+                    'metrics_groups': metrics_groups,
+                    'additional_metrics': f2dq.metrics_in_db(additional_metrics),
+                    'metrics': f2dq.metrics_in_db(f2dq.final_metrics(metrics_groups, additional_metrics)),
+                    'columns': f2dq.columns_in_db(f2dq.f2dq_node_config(el, 'columns', none)),
+                    'anomaly_detector': f2dq.f2dq_node_config(el, 'anomaly_detector', f2dq.f2dq_var('anomaly_detector', {})),
+                    'owners': f2dq.prepare_model_owners(f2dq.f2dq_node_config(el, 'owners', []), owners_config),
+                    'selected': selected
+                    })
+                %}
+            {% endif %}
+        {% endif %}
+    {% endfor %}
+
+    {{ return(monitored) }}
+{% endmacro %}
+
+{% macro get_owners_config() %}
+    {% set owners_config = f2dq.f2dq_var('owners_config', {}) %}
+    {{ return (owners_config) }}
+{% endmacro %}
+
+{% macro prepare_model_owners(f2dq_owners, owners_config) %}
+    {% set owners = {} %}
+    {% set seen_identifiers = {} %}
+    {% for owner in f2dq_owners if owners_config.get(owner) %}
+        {% set members = owners_config.get(owner) %}
+        {% for member in members %}
+            {% set identifier = member.get('identifier') %}
+            {% if identifier not in seen_identifiers %}
+            {% do seen_identifiers.update({identifier: true }) %}
+            {% do owners.update({
+                identifier: {
+                    'notify_channel': member.get('type'),
+                    'owner': owner,
+                    'name': member.get('name') 
+                } 
+            }) %}
+            {% endif %}
+        {% endfor %}
+    {% endfor %}
+    {{ return (owners) }}
+{% endmacro %}

macros/meta/information_schema.sql

@@ -0,0 +1,32 @@
+{% macro get_monitored_columns(schema, database) %}
+    {{ adapter.dispatch('get_monitored_columns', 'f2dq')(schema, database) }}
+{% endmacro %}
+
+{% macro default__get_monitored_columns(table_schema, db_name) %}
+    {% set relation = api.Relation.create(database=db_name, schema=table_schema) %}
+    select
+        table_name,
+        table_schema,
+        table_catalog,
+        column_name,
+        data_type,
+        is_nullable
+    from
+        {{ relation.information_schema('COLUMNS') }}
+    where
+        table_schema = '{{ table_schema }}'
+{% endmacro %}
+
+{% macro redshift__get_monitored_columns(table_schema, db_name) %}
+    select
+        table_name,
+        table_schema,
+        table_catalog,
+        column_name,
+        data_type,
+        is_nullable
+    from
+        svv_columns
+    where
+        table_schema = '{{ table_schema }}'
+{% endmacro %}

macros/meta/monitored_model_queries.sql

@@ -0,0 +1,10 @@
+{% macro get_tables() %}
+    select *
+    from {{ ref('f2dq_selected') }}
+    order by name, schema, database, time_filter
+{% endmacro %}
+
+{% macro get_schemas() %}
+    select distinct schema, database
+    from {{ ref('f2dq_selected') }}
+{% endmacro %}

macros/meta/save_monitored.sql

@@ -0,0 +1,17 @@
+{% macro save_monitored(monitored_path) %}
+
+    {% set monitored_query %}
+        select
+            {{ full_table_name('name', 'schema', 'database') }} as {{ f2dq.quote_column('model') }},
+             time_filter as {{ f2dq.quote_column('time_filter') }},
+            metrics as {{ f2dq.quote_column('metrics') }},
+            columns as {{ f2dq.quote_column('columns') }},
+            anomaly_detector as {{ f2dq.quote_column('anomaly_detector') }},
+            owners as {{ f2dq.quote_column('owners') }}
+        from {{ ref('f2dq_selected') }}
+    {% endset %}
+    {% set query_result = run_query(monitored_query) %}
+    {% set monitored_file_path = monitored_path or 'target/f2dq/monitored.json' %}
+    {% do query_result.to_json(monitored_file_path) %}
+
+{% endmacro %}

macros/meta/table_name.sql

@@ -0,0 +1,33 @@
+{% macro full_table_name(table_name, table_schema, table_catalog) %}
+    {{ adapter.dispatch('full_table_name', 'f2dq')(table_name, table_schema, table_catalog) }}
+{% endmacro %}
+
+
+{% macro default__full_table_name(table_name, table_schema, table_catalog) %}
+    '"' || {{table_catalog}} || '"' || '.' || '"' || {{table_schema}} || '"' || '.' || '"' || {{table_name}} || '"'
+{% endmacro %}
+
+
+{% macro bigquery__full_table_name(table_name, table_schema, table_catalog) %}
+    '`' || {{table_catalog}} || '`' || '.' || '`' || {{table_schema}} || '`' || '.' || '`' || {{table_name}} || '`'
+{% endmacro %}
+
+
+{% macro full_table_name_values(table_name, table_schema, table_catalog) %}
+    {% set result = adapter.dispatch('full_table_name_values', 'f2dq')(table_name, table_schema, table_catalog) %}
+    {{ return (result.strip()) }}
+{% endmacro %}
+
+{% macro default__full_table_name_values(table_name, table_schema, table_catalog) %}
+    "{{table_catalog}}"."{{table_schema}}"."{{table_name}}"
+{% endmacro %}
+
+
+{% macro bigquery__full_table_name_values(table_name, table_schema, table_catalog) %}
+    `{{table_catalog}}`.`{{table_schema}}`.`{{table_name}}`
+{% endmacro %}
+
+
+{% macro snowflake__full_table_name_values(table_name, table_schema, table_catalog) %}
+    "{{table_catalog|upper}}"."{{table_schema|upper}}"."{{table_name|upper}}"
+{% endmacro %}

macros/metrics/base/build_in/column_default.sql

@@ -0,0 +1,91 @@
+
+{% macro f2dq_metric_max(context) %}
+    max({{context.column_name}})
+{% endmacro %}
+
+{% macro f2dq_metric_min(context) %}
+    min({{context.column_name}})
+{% endmacro %}
+
+{% macro f2dq_metric_avg(context) %}
+    avg(cast ({{context.column_name}} as {{ numeric_type() }}))
+{% endmacro %}
+
+{% macro f2dq_metric_stddev(context) %}
+    stddev(cast ( {{context.column_name}} as {{ numeric_type() }}))
+{% endmacro %}
+
+{% macro f2dq_metric_variance(context) %}
+    variance(cast ( {{context.column_name}} as {{ numeric_type() }}))
+{% endmacro %}
+
+{% macro f2dq_metric_max_length(context) %}
+    max(length({{context.column_name}}))
+{% endmacro %}
+
+{% macro f2dq_metric_min_length(context) %}
+    min(length({{context.column_name}}))
+{% endmacro %}
+
+{% macro f2dq_metric_avg_length(context) %}
+    avg(cast (length( {{context.column_name}} ) as {{ numeric_type() }}))
+{% endmacro %}
+
+{% macro f2dq_metric_nulls_count(context) %}
+    coalesce(
+        sum(
+            case when {{context.column_name}} is null
+                then 1
+            else 0
+            end
+        ), 0
+    )
+{% endmacro %}
+
+{% macro f2dq_metric_missing_count(context) %}
+    coalesce(
+        sum(
+            case 
+            when {{context.column_name}} is null
+                then 1
+            when {{context.column_name}} = ''
+                then 1
+            else 0
+            end
+        ), 0
+    )
+{% endmacro %}
+
+{% macro f2dq_metric_nulls_percent(context) %}
+    {{ percentage_formula(f2dq_metric_nulls_count(context), f2dq_metric_row_count()) }}
+{% endmacro %}
+
+{% macro f2dq_metric_missing_percent(context) %}
+    {{ percentage_formula(f2dq_metric_missing_count(context), f2dq_metric_row_count()) }}
+{% endmacro %}
+
+{% macro f2dq_metric_count_true(context) %}
+    COALESCE(
+        SUM(
+            CASE
+                WHEN {{ context.column_name }} IS TRUE THEN 1
+                ELSE 0
+            END
+        ),
+        0
+    )
+{% endmacro %}
+
+{% macro f2dq_metric_count_false(context) %}
+    COALESCE(
+        SUM(
+            CASE
+                WHEN {{ context.column_name }} IS FALSE THEN 1
+                ELSE 0
+            END
+        ),
+        0
+    )
+{% endmacro %}
+
+

macros/metrics/base/build_in/optional_column_metrics.sql

@@ -0,0 +1,104 @@
+{% macro f2dq_metric_regex_count(column_name, pattern) %}
+    coalesce(
+        sum(
+            case when {{ regex_match_expression(column_name, pattern) }}
+                then 1
+            else 0
+            end
+        ), 0
+    )
+{% endmacro %}
+
+{% macro f2dq_metric_match_regex(context) %}
+    {{ f2dq_metric_regex_count(context.column_name, context.config.regex) }}
+{% endmacro %}
+
+{% macro f2dq_metric_match_regex_percent(context) %}
+    {{ percentage_formula(f2dq_metric_match_regex(context), f2dq_metric_row_count()) }}
+{% endmacro %}
+
+{% macro f2dq_metric_not_match_regex(context) %}
+    {{ f2dq_metric_row_count() }} - {{ f2dq_metric_regex_count(context.column_name, context.config.regex) }}
+{% endmacro %}
+
+{% macro f2dq_metric_not_match_regex_percent(context) %}
+    {{ percentage_formula(f2dq_metric_not_match_regex(context), f2dq_metric_row_count()) }}
+{% endmacro %}
+
+{% macro f2dq_metric_distinct_values(context) %}
+    {{ distinct_values(context) }}
+{% endmacro %}
+
+{% macro distinct_values(context) %}
+    {{ adapter.dispatch('distinct_values', 'f2dq')(context) }}
+{% endmacro %}
+
+{% macro default__distinct_values(context) %}
+    coalesce(
+        count(distinct {{ context.column_name }} )
+    , 0)
+{% endmacro %}
+
+{% macro postgres__distinct_values(context) %}
+    {# /* In postgres, its faster to count distinct values in a column by selecting then counting in separate steps */ #}
+    with temp_table as (
+            select distinct {{ context.column_name }} from {{ context.table_name }}
+            where {{ in_time_window(context.time_filter) }}
+        )
+    select coalesce(count(*), 0) from temp_table
+{% endmacro %}
+
+{% macro f2dq_metric_approx_distinct_values(context) %}
+    {{ approx_distinct_values(context) }}
+{% endmacro %}
+
+{% macro approx_distinct_values(context) %}
+    {{ adapter.dispatch('approx_distinct_values', 'f2dq')(context) }}
+{% endmacro %}
+
+{% macro default__approx_distinct_values(context) %}
+    {# /* No approximate distinct count in postgres so we default to using a distinct count */ #}
+    {{ f2dq_metric_distinct_values(context) }}
+{% endmacro %}
+
+{% macro redshift__approx_distinct_values(context) %}
+    approximate {{ f2dq_metric_distinct_values(context.column_name) }}
+{% endmacro %}
+
+{% macro bigquery__approx_distinct_values(context) %}
+    approx_count_distinct({{ context.column_name }})
+{% endmacro %}
+
+{% macro snowflake__approx_distinct_values(context) %}
+    approx_count_distinct({{ context.column_name }})
+{% endmacro %}
+
+{% macro f2dq_metric_duplicate_values(context) %}
+        with temp_table as (
+            select {{ context.column_name }} from {{ context.table_name }}
+            where {{ in_time_window(context.time_filter) }}
+            group by {{ context.column_name }}
+            having count(1) > 1
+        )
+        select coalesce(count(*), 0) from temp_table
+{% endmacro %}
+
+{% macro f2dq_metric_duplicate_rows(context) %}
+        with temp_table as (
+            select {{ context.column_name }}, count(1) as row_count from {{ context.table_name }}
+            where {{ in_time_window(context.time_filter) }}
+            group by {{ context.column_name }}
+            having count(1) > 1
+        )
+        select coalesce(sum(row_count), 0) from temp_table
+{% endmacro %}
+
+{% macro f2dq_metric_unique_rows(context) %}
+        with temp_table as (
+            select {{ context.column_name }}, count(1) as row_count from {{ context.table_name }}
+            where {{ in_time_window(context.time_filter) }}
+            group by {{ context.column_name }}
+            having count(1) = 1
+        )
+        select coalesce(sum(row_count), 0) from temp_table
+{% endmacro %}

macros/metrics/base/build_in/optional_table_metrics.sql

@@ -0,0 +1,7 @@
+{% macro f2dq_metric_distinct_table_rows(context) %}
+    with temp_table AS (
+            select distinct * from {{ context.table_name }}
+            where {{ in_time_window(context.time_filter) }}
+        )
+    select coalesce(count(*), 0) FROM temp_table
+{% endmacro %}

macros/metrics/base/build_in/table_default.sql

@@ -0,0 +1,33 @@
+
+{% macro f2dq_metric_row_count(context) %}
+    count(1)
+{% endmacro %}
+
+{% macro f2dq_metric_freshness(context) %}
+    {{ freshness_expression(context.time_filter) }}
+{% endmacro %}
+
+{% macro freshness_expression(time_filter) %}
+    {# /* If time_filter is none, we are computing the metric globally. we set the value as null since a table without use of a time filter doesn't really have a freshness metric */ #}
+    {% if time_filter is none %}
+        cast(null as {{ numeric_type() }})
+    {% else %}
+        {{ adapter.dispatch('freshness_expression', 'f2dq')(time_filter) }}
+    {% endif %}
+{% endmacro %}
+
+{% macro default__freshness_expression(time_filter) %}
+    EXTRACT(EPOCH FROM ({{time_window_end()}} - max({{time_filter}})))
+{% endmacro %}
+
+{% macro bigquery__freshness_expression(time_filter) %}
+    TIMESTAMP_DIFF ( timestamp({{ time_window_end()}}), timestamp(max({{time_filter}})), SECOND)
+{% endmacro %}
+
+{% macro snowflake__freshness_expression(time_filter) %}
+    timediff(second, max({{time_filter}}), {{- time_window_end() -}})
+{% endmacro %}
+
+{% macro redshift__freshness_expression(time_filter) %}
+    DATEDIFF(second, max({{time_filter}}), {{- time_window_end() -}})
+{% endmacro %}

macros/metrics/base/expression.sql

@@ -0,0 +1,102 @@
+{% macro metrics_base_expressions(model, columns, table_level=False) %}
+
+    {% set col_expr = [] %}
+
+    {% for col in columns %}
+        {% set column_name = f2dq.row_value(col, 'column_name') %}
+        {% do col_expr.extend(f2dq.metrics_base_expression_column_all(model, col)) %}
+    {% endfor %}
+
+    {% if table_level %}
+        {% do col_expr.extend(f2dq.metrics_base_expresion_table_all(model)) %}
+    {% endif %}
+
+    {{ return (col_expr) }}
+
+{% endmacro %}
+
+{% macro metrics_base_expression_column_all(model, column) %}
+
+    {%- set col_expr = [] %}
+    {%- set metrics_to_compute = [] %}
+    {% set column_name = f2dq.row_value(column, 'column_name') %}
+    {% set data_type = model.columns_info[column_name].data_type %}
+    {% do metrics_to_compute.extend(model.metrics.get('group').get('column', {}).get(data_type, [])) %}
+    {% do metrics_to_compute.extend(model.metrics.get('additional').get('column', {}).get(column_name, [])) %} 
+
+    {% for metric_value in metrics_to_compute %}
+        {% set metric_obj = f2dq.extract_metric_config(metric_value) %}
+        {% set expression = f2dq.metrics_base_expression_column(model, column_name, metric_obj['metric'], metric_obj['config']) %}
+        {% do col_expr.append({ 'expr': expression, 'col_name': column_name, 'metric': metric_obj['metric']}) %}
+    {% endfor %}
+
+    {{ return (col_expr) }}
+
+{% endmacro %}
+
+
+{% macro metrics_base_expresion_table_all(model) %}
+    {%- set table_expr = [] %}
+    {%- set metrics_to_compute = [] %}
+    {% do metrics_to_compute.extend(model.metrics.get('group').get('table', [])) %}
+    {% do metrics_to_compute.extend(model.metrics.get('additional').get('table', [])) %}
+
+    {% for metric_value in metrics_to_compute %}
+        {% set metric_obj = f2dq.extract_metric_config(metric_value) %}
+        {% set expression = f2dq.metrics_base_expression_table(model, metric_obj['metric'], metric_obj['config']) %}
+        {% do table_expr.append({ 'expr': expression, 'col_name': '', 'metric': metric_obj['metric']}) %}
+    {% endfor %}
+
+    {{ return (table_expr) }}
+
+{% endmacro %}
+
+{% macro metrics_base_expression_table(model, metric_name, config) %}
+    {% set metric_macro = f2dq.get_metric_macro(metric_name) %}
+    {% set context = {'time_filter': model.time_filter, 'metric_name': metric_name, 'config': config, 'table_name': model.table_name, 'column_name': none} %}
+
+    {{ metric_macro(context) }}
+
+{% endmacro %}
+
+
+{%- macro metrics_base_expression_column(model, column_name, metric_name, config) %}
+    {% set metric_macro = f2dq.get_metric_macro(metric_name) %}
+    {% set context = {'time_filter': model.time_filter, 'metric_name': metric_name, 'config': config, 'table_name': model.table_name, 'column_name': f2dq.quote_column_name(column_name)} %}
+
+    {{ metric_macro(context) }}
+
+{% endmacro %}
+
+{% macro extract_metric_config(metric_value) %}
+
+    {% set config = none %}
+
+    {% if metric_value is mapping %}
+        {% set metric = metric_value.keys() | first %}
+        {% if metric_value[metric] is none %}
+            {{ exceptions.raise_compiler_error("Empty configuration passed for metric: " ~ metric ~ ". If the metric doesn't use a config, please use the column name as a string.") }}
+        {% endif %}
+
+        {% set config = metric_value[metric] %}
+    {%- else %}
+        {% set metric = metric_value %}
+    {% endif %}
+
+    {{ return ({'metric': metric, 'config': config}) }}
+
+{% endmacro %}
+
+{%- macro get_metric_macro(metric_name) %}
+    {% set macro_name = 'f2dq_metric' + '_' + metric_name %}
+
+    {% if context['f2dq'].get(macro_name) %}
+        {% set metric_macro = context['f2dq'][macro_name] %}
+    {%- else %}
+        {% set metric_macro = context[project_name][macro_name] %}
+    {% endif %}
+
+    {{ return (metric_macro) }}
+
+{% endmacro %}
+

macros/metrics/base/internal_model_template.sql

@@ -0,0 +1,31 @@
+{% macro f2dq_last_base_metrics_part() %}
+
+-- depends_on: {{ ref('f2dq_columns') }}
+
+{{
+    config(
+        materialized='table',
+    )
+}}
+
+{{ f2dq.empty_last_base_metrics() }}
+
+{% endmacro %}
+
+{% macro f2dq_last_base_metrics_thread(num) %}
+    {% set part_name = 'f2dq_last_base_metrics_part' ~ num %}
+    {{ f2dq.generate_depends(['f2dq_selected', 'f2dq_monitored', 'f2dq_columns', 'f2dq_run_started_at', part_name]) }}
+
+    {{
+        config(
+            materialized='table',
+        )
+    }}
+
+    {% if not f2dq.in_compile() %}
+        {{ f2dq.metrics_base_compute_for_thread(num, part_name) }}
+    {% endif %}
+
+    {{ f2dq.empty_last_base_metrics() }}
+
+{% endmacro %}

macros/metrics/base/queries.sql

@@ -0,0 +1,75 @@
+{% macro metrics_base_compute_for_thread(thread_value, ref_model) %}
+    {%- set tables =  run_query(f2dq.get_tables()) %}
+    {%- for mtable in tables %}
+        -- we are splitting computing metrics to 4 different threads
+        {% set for_loop_mod = (loop.index % 4) %}
+        {% if for_loop_mod == thread_value %}
+            {% set model = get_model_config(mtable) %}
+
+            {% set columns_to_query = [] %}
+            {% set size = 0 %}
+
+            {% for column in model.columns %}
+                {% set column_name = f2dq.row_value(column, 'column_name') %}
+                
+                {% if should_compute_metric(model, column_name) %}
+                    {% do columns_to_query.append(column) %}
+                {% endif %}
+
+                {% set columns_size = columns_to_query| length %}
+
+                {% if columns_size == var('f2dq:max_columns_in_query') %}
+                    {%- set insert_stats_query = f2dq.metrics_base_insert(model, ref_model, columns_to_query) -%}
+
+                    {% if insert_stats_query %}
+                        {% do run_query(insert_stats_query) %}
+                    {% endif %}
+                    {% do columns_to_query.clear() %}
+                {% endif %}
+            {% endfor %}
+
+            {%- set insert_stats_query = f2dq.metrics_base_insert(model, ref_model, columns_to_query, table_level=True) -%}
+            {% do run_query(insert_stats_query) %}
+
+            {{ dbt_utils.log_info('[f2dq_log] - finished computing metrics for:' ~ model.model_name) }}
+        {% endif %}
+    {% endfor %}
+{% endmacro %}
+
+{% macro metrics_base_insert(model, ref_model, columns, table_level=False) %}
+
+    {% set col_exprs = f2dq.metrics_base_expressions(model, columns, table_level) %}
+    {% if col_exprs == [] %}
+        {{ return ('') }}
+    {% endif %}
+
+    insert into {{ ref(ref_model) }}
+    with temp_table_metrics as (
+    select 
+        {%- for col_expr in col_exprs %}
+            ( {{ col_expr.expr }} ) as {{ f2dq.quote_column_name(col_expr.col_name + '___' + col_expr.metric) }}
+            {%- if not loop.last %},{%- endif %}
+        {% endfor %}
+    from 
+        {{ model.table_name }}
+    where
+        {{ in_time_window(model.time_filter) }}
+    )
+
+    {%- for col_expr in col_exprs %}
+        {% set final_metric_name = get_final_metric_name(col_expr.metric, model.time_filter) %}
+        
+        select '{{model.table_name}}' as table_name, '{{ col_expr.col_name }}' as column_name, '{{ final_metric_name }}' as metric, {{ f2dq.quote_column_name(col_expr.col_name + '___' + col_expr.metric) }} as value
+        from temp_table_metrics
+        {% if not loop.last %}union all{% endif %}
+    {% endfor %}
+
+{% endmacro %}
+
+{% macro get_final_metric_name(metric_name, time_filter) %}
+    {% if time_filter is none %}
+        {{ return ('global__' + metric_name) }}
+    {% else %}
+        {{ return (metric_name) }}
+    {% endif %}
+{% endmacro %}

macros/post_hook/f2dq_monitored.sql

@@ -0,0 +1,11 @@
+{% macro pub_insert_into_f2dq_monitored() %}
+    {% set monitored = f2dq.pub_monitored_from_graph() %}
+    {% do f2dq.insert_list_to_table(
+        this,
+        monitored,
+        ['name', 'schema', 'database', 'time_filter', 'metrics_groups', 'additional_metrics', 'metrics', 'columns', 'anomaly_detector', 'owners', 'selected']
+    ) %}
+
+    {{ return('') }}
+
+{% endmacro %}

macros/public/cleaning/clean_additional_whitespace.sql

@@ -0,0 +1,23 @@
+{% macro clean_additional_whitespaces(column_name) %}
+    {{ adapter.dispatch('clean_additional_whitespaces', 'f2dq')(column_name) }}
+{% endmacro %}
+
+{% macro default__clean_additional_whitespaces(column_name) %}
+    trim(regexp_replace( {{ column_name }}, '\s\s+', ' '))
+{% endmacro %}
+
+{% macro postgres__clean_additional_whitespaces(column_name) %}
+    trim(regexp_replace( {{ column_name }}, '\s\s+', ' ', 'g'))
+{% endmacro %}
+
+{% macro redshift__clean_additional_whitespaces(column_name) %}
+    trim(regexp_replace( {{ column_name }}, '\\s\\s+', ' '))
+{% endmacro %}
+
+{% macro bigquery__clean_additional_whitespaces(column_name) %}
+    trim(regexp_replace( {{ column_name }}, r'\s\s+', ' '))
+{% endmacro %}
+
+{% macro snowflake__clean_additional_whitespaces(column_name) %}
+    trim(regexp_replace( {{ column_name }}, '\\s\\s+', ' '))
+{% endmacro %}

macros/public/cleaning/clean_blacklist.sql

@@ -0,0 +1,31 @@
+{% macro generate_blacklist_pattern(chars_to_blacklist) %}
+    {% set pattern = [] %}
+    {% for char in chars_to_blacklist %}
+        {% set expr = '(' + char + ')' %}
+        {% do pattern.append(expr) %}
+    {% endfor %}
+
+    {{ return(pattern | join('|')) }}
+{% endmacro %}
+
+{%- macro clean_blacklist(column_name, chars_to_blacklist, replacement) -%}
+    {% set pattern_string = f2dq.generate_blacklist_pattern(chars_to_blacklist) %}
+
+    {{ adapter.dispatch('clean_blacklist', 'f2dq')(column_name, pattern_string, replacement) }}
+{%- endmacro -%}
+
+{%- macro default__clean_blacklist(column_name, pattern_string, replacement) -%}
+    regexp_replace( {{ column_name }}, '{{ pattern_string }}', '{{ replacement }}')
+{%- endmacro -%}
+
+{%- macro postgres__clean_blacklist(column_name, pattern_string, replacement) -%}
+    regexp_replace( {{ column_name }}, '{{ pattern_string }}', '{{ replacement }}', 'g')
+{%- endmacro -%}
+
+{%- macro redshift__clean_blacklist(column_name, pattern_string, replacement) -%}
+    regexp_replace( {{ column_name }}, '{{ pattern_string }}', '{{ replacement }}')
+{%- endmacro -%}
+
+{%- macro bigquery__clean_blacklist(column_name, pattern_string, replacement) -%}
+    regexp_replace( {{ column_name }}, """{{ pattern_string }}""", '{{ replacement }}')
+{%- endmacro -%}

macros/public/cleaning/clean_capitalize_words.sql

@@ -0,0 +1,3 @@
+{% macro clean_capitalize_words(column_name) %}
+    initcap( {{column_name}} )
+{% endmacro %}

macros/public/filtering/get_duplicates.sql

@@ -0,0 +1,23 @@
+{# https://github.com/fast2dev/f2dq/issues/143 #}
+
+{#
+macro returns rows with she same key set (unique_cols)
+
+along with the fields of the base model duplicates information added:
+f2dq_duplicate_count - total number of duplicates with the same current key set
+f2dq_duplicate_row_number - number of a duplicate row inside the group of duplicates with the same current key set
+#}
+
+{% macro filter_get_duplicates(relation, unique_cols, sort_columns) %}
+    (
+        with duplication_context as (
+            {{f2dq.add_duplication_context(relation, unique_cols, sort_columns)}}
+        ),
+        duplicate_rows as (
+            select * from duplication_context where f2dq_duplicate_group_row_count > 1
+        )
+        {# return surrogate key as well? #}
+        select *
+        from duplicate_rows
+    ) 
+{% endmacro %}

macros/public/filtering/remove_duplicates.sql

@@ -0,0 +1,15 @@
+
+
+
+{% macro filter_remove_duplicates(relation, unique_cols, sort_columns) %}
+    (
+        with with_row_num as (
+            {{f2dq.add_duplication_context(relation, unique_cols, sort_columns)}}
+        ),
+        one_row_num as (
+            select * from with_row_num where f2dq_duplicate_group_row_number = 1
+        )
+        select {{ dbt_utils.star(from=relation) }}
+        from one_row_num
+    ) 
+{% endmacro %}

macros/public/normalizing/normalize_values.sql

@@ -0,0 +1,32 @@
+{% macro is_dbt_relation(obj) %}
+    {{ return (obj is mapping and obj.get('metadata', {}).get('type', '').endswith('Relation') )}}
+{% endmacro %}
+
+{% macro normalize_expression_cte(reference_table) %}
+    with target_table as (
+        {% if f2dq.is_dbt_relation(reference_table) or reference_table is string %}
+            select * from {{ reference_table }}
+        {% elif reference_table is mapping %}
+            {% for key, value in reference_table.items() %}
+                select '{{key}}' as source, '{{value}}' as target
+                {% if not loop.last %}union all{% endif %}
+            {% endfor %}
+        {% endif %}
+    )
+{% endmacro %}
+
+{%- macro normalize_values(source_relation, column_name, reference_table) -%}
+    ( 
+        {{ f2dq.normalize_expression_cte(reference_table) }}
+        
+        select s.*, 
+        case when t.source is null
+                then s.{{column_name}}
+            else t.target
+            end as {{ column_name + '__normalized'}} 
+        from {{ source_relation }} s
+        left join target_table t 
+        on t.source = s.{{column_name}}
+    )
+{%- endmacro -%}
+

macros/public/store/export_alerts.sql

@@ -0,0 +1,22 @@
+{% macro export_alerts(start_date, end_date, alerts_path=None, monitored_path=None) %}
+    {% set alerts_query %}
+        select
+            type as {{ f2dq.quote_column('type') }},
+            model as {{ f2dq.quote_column('model') }},
+            message as {{ f2dq.quote_column('message') }},
+            value as {{ f2dq.quote_column('value') }},
+            {{ format_timestamp('time_window_end')}} as {{ f2dq.quote_column('time_window_end') }} 
+        from {{ ref('f2dq_alerts') }}
+        where
+            case
+                when type = 'anomaly' then {{ in_date_window('time_window_end', start_date, end_date) }}
+                else {{ in_date_window('time_window_end', start_date, none) }}
+            end
+        order by time_window_end desc
+    {% endset %}
+
+    {% set query_result = run_query(alerts_query) %}
+    {% set alerts_file_path = alerts_path or 'target/f2dq/alerts.json' %}
+    {% do query_result.to_json(alerts_file_path) %}
+    {{ save_monitored(monitored_path) }}
+{% endmacro %}

macros/public/store/export_table_samples.sql

@@ -0,0 +1,15 @@
+{% macro export_table_samples(start_date, end_date, table_samples_path=None) %}
+    {% set table_samples_query %}
+        select
+            lower(table_name) as table_name,
+            sample_data,
+            sampled_on
+        from
+            {{ ref('f2dq_table_samples') }}
+    {% endset %}
+
+    {% set query_result = run_query(table_samples_query) %}
+    {% set table_samples_file_path = table_samples_path or 'target/f2dq/table_samples.json' %}
+    {% do query_result.to_json(table_samples_file_path) %}
+
+{% endmacro %}

macros/public/store/export_tests_history.sql

@@ -0,0 +1,25 @@
+{% macro export_tests_history(start_date, end_date, tests_history_path=None) %}
+    {% set tests_history_query %}
+        select
+            table_name,
+            column_name,
+            test_name,
+            run_at,
+            status,
+            execution_time, 
+            message, 
+            failures_count, 
+            failures_json, 
+            failures_table,
+            severity, 
+            compiled_sql
+        from
+            {{ ref('f2dq_test_history') }}
+        where {{ in_date_window('run_at', start_date, end_date) }}
+    {% endset %}
+
+    {% set query_result = run_query(tests_history_query) %}
+    {% set tests_history_file_path = tests_history_path or 'target/f2dq/tests_history.json' %}
+    {% do query_result.to_json(tests_history_file_path) %}
+
+{% endmacro %}

macros/public/store/generate_overview.sql

@@ -0,0 +1,93 @@
+
+{% macro overview_select_base(type, timestamp_col) %}
+    '{{ type }}' as {{ f2dq.quote_column('type') }},
+    table_name as {{ f2dq.quote_column('table_name') }},
+    column_name as {{ f2dq.quote_column('column_name') }},
+    {{ timestamp_col }} as {{ f2dq.quote_column('computed_on') }},
+{% endmacro %}
+
+{% macro generate_overview(start_date, end_date, interval, overview_path=None, monitored_path=None) %}
+-- depends_on: {{ ref('f2dq_anomalies') }}
+-- depends_on: {{ ref('f2dq_base_metrics') }}
+-- depends_on: {{ ref('f2dq_schema_changes') }}
+-- depends_on: {{ ref('f2dq_columns') }}
+
+    {# time grain is either days or hour #}
+    {% set time_grain, num_str = interval.split(':') %}
+    {% set num = num_str | int %}
+    {% if time_grain == 'hours' %}
+        {% set interval_length_sec = num * 3600 %}
+    {% elif time_grain == 'days'%}
+        {% set interval_length_sec = num * 3600 * 24 %}
+    {% else %}
+        {{ exceptions.raise_compiler_error("Invalid interval. Got: " ~ interval) }}
+    {% endif %}
+    {{ dbt_utils.log_info('[f2dq] interval length in seconds is ' ~ interval_length_sec) }}
+    {% set overview_query %}
+        with schema_changes_casted as (
+            select id, table_name, operation, column_name, data_type, {{ bool_to_string('is_nullable') }}, prev_column_name, prev_data_type, {{ bool_to_string('prev_is_nullable') }}, detected_time
+            from {{ ref('f2dq_schema_changes') }}
+        ),
+        columns_casted as (
+            select {{ full_table_name('name', 'schema', 'database') }} as table_name, column_name, data_type, {{ bool_to_string('is_nullable') }}, computed_on
+            from {{ ref('f2dq_columns') }} 
+        )
+        
+    (    
+        select
+            {{ overview_select_base('metric', 'computed_on')}}
+            {{ to_single_json(['metric', 'value', 'time_window_end', 'interval_length_sec']) }} as {{ f2dq.quote_column('data') }}
+        from
+            {{ ref('f2dq_base_metrics') }}
+            where {{ in_date_window('time_window_end', start_date, end_date) }}
+            and interval_length_sec = {{interval_length_sec}}
+    ) union all 
+    (
+        select
+            {{ overview_select_base('anomaly', 'computed_on')}}
+            {{ to_single_json(['id', 'metric', 'z_score_value', 'last_value', 'last_avg', 'last_stddev', 'time_window_end', 'interval_length_sec']) }} as {{ f2dq.quote_column('data') }}
+        from
+            {{ ref('f2dq_anomalies') }}
+            where {{ in_date_window('time_window_end', start_date, end_date) }}
+            and interval_length_sec = {{interval_length_sec}}
+    ) union all
+    (
+        select
+            {{ overview_select_base('schema_change', 'detected_time')}}
+            {{ to_single_json(['id', 'operation', 'data_type', 'is_nullable', 'prev_column_name', 'prev_data_type', 'prev_is_nullable', 'detected_time']) }} as {{ f2dq.quote_column('data') }}
+        from
+            schema_changes_casted
+            where {{ in_date_window('detected_time', start_date, none) }}
+    ) union all
+    (
+        select
+            {{ overview_select_base('schema', 'computed_on')}}
+            {{ to_single_json(['data_type', 'is_nullable']) }} as {{ f2dq.quote_column('data') }}
+        from
+            columns_casted
+    )
+    union all 
+    (
+        select 
+            'alert' as {{ f2dq.quote_column('type') }},
+            model as {{ f2dq.quote_column('table_name') }},
+            null as {{ f2dq.quote_column('column_name') }},
+            time_window_end as {{ f2dq.quote_column('computed_on') }},
+            {{ to_single_json(['type', 'model', 'message', 'value', 'time_window_end']) }} as {{ f2dq.quote_column('data') }}
+        from
+            {{ ref('f2dq_alerts') }}
+        where
+            case
+                when type = 'anomaly' then {{ in_date_window('time_window_end', start_date, end_date)  }} 
+                else {{ in_date_window('time_window_end', start_date, none) }}
+            end
+    )
+    order by {{ f2dq.quote_column('computed_on')}} desc
+    {% endset %}
+
+    {% set overview_result = run_query(overview_query) %}
+    {% set overview_file_path = overview_path or '../target/f2dq/overview.json' %}
+    {% do overview_result.to_json(overview_file_path) %}
+    {{ save_monitored(monitored_path) }}
+
+{% endmacro %}

macros/public/validating/regex_dict.sql

@@ -0,0 +1,65 @@
+{#
+#  This file contains significant part of code licensed under:
+#  Copyright 2020 Soda
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#   http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#}
+
+{% macro get_regex_for(to_validate) %}
+
+    {% set regexp_dict = {
+        'number_whole': '^\-?[0-9]+$',
+        'number_decimal_point': '^\-?[0-9]+\.[0-9]+$',
+        'number_decimal_comma': '^\-?[0-9]+,[0-9]+$',
+        'number_percentage': '^\-?[0-9]+([\.,][0-9]+)? ?%$',
+        'number_percentage_point': '^\-?[0-9]+([\.][0-9]+)? ?%$',
+        'number_percentage_comma': '^\-?[0-9]+([,][0-9]+)? ?%$',
+        'date_eu': '^([1-9]|0[1-9]|[12][0-9]|3[01])[-\./]([1-9]|0[1-9]|1[012])[-\./](19|20)?[0-9][0-9]$',
+        'date_us': '^([1-9]|0[1-9]|1[012])[-\./]([1-9]|0[1-9]|[12][0-9]|3[01])[-\./](19|20)?[0-9][0-9]$',
+        'date_inverse': '^(19|20)[0-9][0-9][-\./]?([1-9]|0[1-9]|1[012])[-\./]?([1-9]|0[1-9]|[12][0-9]|3[01])$',
+        'time_24h': '^([01][0-9]|2[0-3]):([0-5][0-9])$',
+        'time_12h': '^(1[0-2]|0?[1-9]):[0-5][0-9]$',
+        'time': '^([0-9]|1[0-9]|2[0-4])[:-]([0-9]|[0-5][0-9])([:-]([0-9]|[0-5][0-9])(,[0-9]+)?)?$',
+        'date_iso_8601':
+            '^'
+            '([1-9][0-9]{3}-((0[1-9]|1[0-2])-(0[1-9]|1[0-9]|2[0-8])|(0[13-9]|1[0-2])-(29|30)|(0[13578]|1[02])-31)|'
+            '([1-9][0-9](0[48]|[2468][048]|[13579][26])|([2468][048]|[13579][26])00)-02-29)'
+
+            'T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\.[0-9]+)?'
+
+            '(Z|[+-][01][0-9]:[0-5][0-9])?'
+            '$',
+        'uuid': '^[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}$',
+        'ipv4_address': '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$',
+        'ipv6_address': '^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])(\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])){3}))|:)))(%.+)?$',
+        'email': '^[A-Za-z0-9.-_%]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,4}$',
+        } %}
+
+    {% set base_regex = regexp_dict[to_validate] %}
+    {% set qualifed_regex = adapter.dispatch('get_regex_for', 'f2dq')(base_regex) %}
+    {{ return(qualifed_regex) }}
+
+{% endmacro %}
+
+{% macro default__get_regex_for(pattern) %}
+    {{ return (pattern) }}
+{% endmacro %}
+
+{% macro redshift__get_regex_for(pattern) %}
+    {% set changed = modules.re.sub('\.', '\\.', pattern) %}
+    {% set changed = modules.re.sub('\-', '\\-', changed) %}
+    {{ return (changed) }}
+{% endmacro %}
+
+{% macro snowflake__get_regex_for(pattern) %}
+    {% set changed = modules.re.sub('\.', '\\.', pattern) %}
+    {% set changed = modules.re.sub('\-', '\\-', changed) %}
+    {{ return (changed) }}
+{% endmacro %}

macros/public/validating/valid_with_regex.sql

@@ -0,0 +1,93 @@
+
+
+{% macro valid_regex(column_name, to_validate) %}
+{% set pattern = f2dq.get_regex_for(to_validate) %}
+  case when 
+    {{ column_name }} is null then false 
+    else {{ f2dq.regex_match_expression(column_name, pattern) }}
+  end
+{% endmacro %}
+
+{% macro valid_email(column_name) %}
+    {{ f2dq.valid_regex(column_name, 'email')}}
+{% endmacro %}
+
+{% macro valid_date_eu(column_name) %}
+    {{ f2dq.valid_regex(column_name, 'date_eu')}}
+{% endmacro %}
+
+{% macro valid_date_us(column_name) %}
+    {{ f2dq.valid_regex(column_name, 'date_us')}}
+{% endmacro %}
+
+{% macro valid_date_inverse(column_name) %}
+    {{ f2dq.valid_regex(column_name, 'date_inverse')}}
+{% endmacro %}
+
+{% macro valid_date_iso_8601(column_name) %}
+    {{ f2dq.valid_regex(column_name, 'date_iso_8601')}}
+{% endmacro %}
+
+{% macro valid_time_24h(column_name) %}
+    {{ f2dq.valid_regex(column_name, 'time_24h')}}
+{% endmacro %}
+
+{% macro valid_time_12h(column_name) %}
+    {{ f2dq.valid_regex(column_name, 'time_12h')}}
+{% endmacro %}
+
+{% macro valid_time(column_name) %}
+    {{ f2dq.valid_regex(column_name, 'time')}}
+{% endmacro %}
+
+{% macro valid_ip_v4(column_name) %}
+    {{ f2dq.valid_regex(column_name, 'ipv4_address')}}
+{% endmacro %}
+
+{% macro valid_ip_v6(column_name) %}
+    {{ f2dq.valid_regex(column_name, 'ipv6_address')}}
+{% endmacro %}
+
+{% macro valid_ip(column_name) %}
+    (
+        {{ f2dq.valid_regex(column_name, 'ipv4_address')}}
+    or 
+        {{ f2dq.valid_regex(column_name, 'ipv6_address')}}
+    )
+{% endmacro %}
+
+{% macro valid_number(column_name) %}
+    {{ f2dq.valid_regex(column_name, 'number_whole')}}
+{% endmacro %}
+
+{% macro valid_number_decimal_point(column_name) %}
+    {{ f2dq.valid_regex(column_name, 'number_decimal_point')}}
+{% endmacro %}
+
+{% macro valid_number_decimal_comma(column_name) %}
+    {{ f2dq.valid_regex(column_name, 'number_decimal_comma')}}
+{% endmacro %}
+
+{% macro valid_number_percentage(column_name) %}
+    {{ f2dq.valid_regex(column_name, 'number_percentage')}}
+{% endmacro %}
+
+{% macro valid_number_percentage_point(column_name) %}
+    {{ f2dq.valid_regex(column_name, 'number_percentage_point')}}
+{% endmacro %}
+
+{% macro valid_number_percentage_comma(column_name) %}
+    {{ f2dq.valid_regex(column_name, 'number_percentage_comma')}}
+{% endmacro %}
+
+{% macro valid_phone(column_name) %}
+    {{ f2dq.valid_regex(column_name, 'phone')}}
+{% endmacro %}
+
+{% macro valid_uuid(column_name) %}
+    {{ f2dq.valid_regex(column_name, 'uuid')}}
+{% endmacro %}
+
+{% macro valid_credit_card(column_name) %}
+    {{ f2dq.valid_regex(column_name, 'credit_card_number')}}
+{% endmacro %}

macros/run_end/save_results_history.sql

@@ -0,0 +1,112 @@
+
+{% macro save_test_history(results) -%}
+
+{{ adapter.dispatch('save_test_history_impl', 'f2dq') (results) }}
+
+{%- endmacro %}
+
+{% macro default__save_test_history_impl(results) %}
+    -- depends_on: {{ ref('f2dq_test_history') }}
+    {% set command = flags.WHICH %}
+    {% if execute and results and command in ('test', 'build') %}
+        {% set tests = [] %}
+        {% for el in results %}
+            {% if el.node.resource_type.value == 'test' %}
+                {% do tests.append(f2dq.test_data_dict(el)) %}
+            {% endif %}
+        {% endfor %}
+
+        {% if tests %}
+            {% do f2dq.insert_list_to_table(
+                ref('f2dq_test_history'),
+                tests,
+                ['table_name', 'column_name', 'test_name', 'status', 'execution_time', 'message', 'failures_count', 'failures_json', 'failures_table', 'severity', 'compiled_sql', 'run_at'],
+                { 'run_at': timestamp_type() }
+            ) %}
+        {% endif %}
+
+    {% endif %}
+    {{ return ('') }}
+
+{% endmacro %}
+
+{% macro test_data_dict(el) %}
+
+    {% set run_started_at_str = run_started_at.strftime('%Y-%m-%d %H:%M:%S') %}
+
+    {% if el.node.to_dict().get('test_metadata') %}
+        {% set any_refs = modules.re.findall("ref\(\'(?P<name>.*)\'\)", el.node.test_metadata.kwargs['model']) %}
+        {% set any_source = modules.re.findall("source\(\'(?P<one>.*)\'\,\s+\'(?P<two>.*)\'\)", el.node.test_metadata.kwargs['model']) %}
+
+        {% if any_refs %}
+            {% set name = any_refs[0] %}
+            {% set node_name = f2dq.priv_full_name_from_depends(el.node, name) %}
+            {% set schema = graph.nodes.get(node_name)['schema'] %}
+            {% set database = graph.nodes.get(node_name)['database'] %}
+            {% set table_name = (database + '.' + schema + '.' + name) | lower %} 
+            
+        {% elif any_source %}
+            {% set package_name = any_source[0][0] %}
+            {% set name = any_source[0][1] %}
+            {% set node_name = f2dq.priv_full_name_from_depends(el.node, name) %}
+            {% set schema = graph.sources.get(node_name)['schema'] %}
+            {% set database = graph.sources.get(node_name)['database'] %}
+            {% set table_name = (database + '.' + schema + '.' + name) | lower %}
+        {% else %}
+            {% set table_name = none %}
+        {% endif %}
+    {% else %}
+        {% set table_name = none %}
+    {% endif %}
+
+    {% if var.has_var('f2dq:query_test_failures') %}
+        {% set query_failures = var('f2dq:query_test_failures') %}
+    {% else %}
+        {% set query_failures = true %}
+    {% endif %}
+
+    {% if el.failures and el.failures > 0 and el.node.relation_name and query_failures %}
+        {% if var.has_var('f2dq:test_history_failures_limit') %}
+            {% set limit_count = var('f2dq:test_history_failures_limit')%}
+        {% else %}
+            {% set limit_count = 10 %}
+        {% endif %}
+
+        {% set failures_query %}
+            select * from {{ el.node.relation_name}} limit {{ limit_count }}
+        {% endset %}
+        {% set failures_list = f2dq.agate_to_list(run_query(failures_query)) %}
+    {% endif %}
+
+    {% set failures_json = none %}
+
+    {{ return ({
+        'table_name': table_name,
+        'column_name': el.node.column_name or none,
+        'test_name': el.node.name,
+        'status': el.status.name,
+        'execution_time': el.execution_time,
+        'message': el.message,
+        'failures_count': el.failures,
+        'failures_json': '' ~ failures_list,
+        'failures_table': el.node.relation_name or none,
+        'severity': el.node.config.severity,
+        'compiled_sql': el.node.compiled_sql or el.node.compiled_code or none,
+        'run_at': run_started_at_str,
+        })
+    }}
+
+{% endmacro %}
+
+{% macro priv_full_name_from_depends(node, name) %}
+
+    {% for full_name in node.depends_on.nodes %}
+        {% set node_name = full_name.split('.')[-1] %}
+        {% if node_name == name %}
+            {{ return(full_name) }}
+        {% endif %}
+    {% endfor %}
+
+    {{ return(none) }}
+
+{% endmacro %}

macros/samples/internal_model_template.sql

@@ -0,0 +1,54 @@
+{%- macro order_by_if_time_filter(time_filter) -%}
+    {%- if time_filter is not none -%}
+        order by {{ time_filter }} desc
+    {%- endif -%}
+{%- endmacro -%}
+
+
+{% macro f2dq_last_table_samples() %}
+    {{ f2dq.generate_depends(['f2dq_selected', 'f2dq_monitored', 'f2dq_columns', 'f2dq_run_started_at', 'f2dq_last_table_samples_part']) }}
+
+    {{
+        config(
+            materialized='table',
+        )
+    }}
+
+    {% set store_samples = f2dq.f2dq_var('store_table_samples', false) %}
+    {% if not f2dq.in_compile() and store_samples is sameas true %}
+        {%- set tables = run_query(f2dq.get_tables()) %}
+
+        {% set samples_list = [] %}
+        {%- for sample_table in tables %}
+
+            {% set model = get_model_config(sample_table) %}
+            {% set columns_to_sample = [] %}
+            {% for key, value in model.columns_info.items() | sort %}
+                {% if value.data_type in ['numeric', 'text'] %}
+                    {% do columns_to_sample.append(key) %}
+                {% endif %}
+            {% endfor %}
+
+            {% set samples_query %}
+                select {{ print_list(columns_to_sample)}} from {{ model.table_name }}
+                {{ order_by_if_time_filter(model.time_filter) }}
+                limit 10
+            {% endset %}
+
+            {% set samples = f2dq.agate_to_list(run_query(samples_query)) %}
+            {% do samples_list.append({
+                'table_name': model.model_name,
+                'sample_data': samples,
+            }) %}
+
+        {% endfor %}
+        {% do f2dq.insert_list_to_table(
+                ref('f2dq_last_table_samples_part'),
+                samples_list,
+                ['table_name', 'sample_data']
+            ) %}
+    {% endif %}
+
+    {{ f2dq.empty_last_table_samples() }}
+
+{% endmacro %}

macros/store/insert_list_to_table.sql

@@ -0,0 +1,42 @@
+{% macro insert_list_to_table(table, list, params, dtype=None,insert_size=100) %}
+
+    {% set single_insert_list = [] %}
+    {% for el in list %}
+        {% do single_insert_list.append(el) %}
+        {% set single_insert_list_size = single_insert_list | length %}
+        {% if single_insert_list_size == insert_size or loop.last %}
+
+            {% set insert_query %}
+                insert into {{ table }} ({%- for p in params %}{{p}}{% if not loop.last %}, {% endif %}{% endfor %}) values
+                {%- for row in single_insert_list -%}
+                    (
+                    {%- for p in params -%}
+                        {%- if row[p] is none -%}
+                            NULL
+                        {%- else -%}
+                            {%- if row[p] is string -%}
+                                {%- if dtype and p in dtype -%}
+                                  {% set cast_type = dtype[p] %}
+                                  cast ({{ f2dq.quote_string(row[p]) }} as {{ cast_type }})
+                                {%- else %}
+                                  {{- f2dq.quote_string(row[p]) -}}
+                                {%- endif -%}
+                            {%- elif row[p] is number -%}
+                                {{-row[p]-}}
+                            {%- else -%}
+                                {{- f2dq.quote_string(tojson(row[p])) -}}
+                            {%- endif -%}
+                        {%- endif -%}
+                        {%- if not loop.last -%},{%- endif -%}
+                    {%- endfor -%}
+                    )
+                    {%- if not loop.last -%},{%- endif %}
+                {% endfor -%}
+            {% endset %}
+
+            {% do run_query(insert_query) %}
+            {% do single_insert_list.clear() %}
+        {% endif %}
+    {% endfor %}
+
+{% endmacro %}

macros/tests/test_metrics.sql

@@ -0,0 +1,106 @@
+
+{% macro metric_expression(table, metric, expression, column_name=None, condition=None) %}
+    select * from {{ref('f2dq_base_metrics')}}
+    where
+        table_name = '{{ f2dq.full_table_name_values(table.identifier, table.schema, table.database)}}' and
+        metric = '{{ metric }}' and
+        {% if condition is not none %}
+            {{ condition }} and
+        {% endif %}
+        {% if column_name is none %}
+        not ( {{ expression }} )
+        {% else %}
+        column_name = '{{ column_name }}' and
+        not ( {{ expression }} )
+        {% endif %}
+
+{% endmacro %}
+
+{# old test macros, will be removed after some time #}
+{% test metric_expression_is_true(model, table, metric, expression, column_name=None, condition=None) %}
+    {{ f2dq.metric_expression(table, metric, expression, column_name=None, condition=None) }}
+{% endtest %}
+
+
+{% test metric_equal_to(model, table, metric, value, column_name=None, condition=None) %}
+    {{ f2dq.metric_expression(table, metric, 'value = ' ~ value, column_name, condition) }}
+{% endtest %}
+
+
+{% test metric_in_range(model, table, metric, min_value, max_value, column_name=None, condition=None) %}
+    {{ f2dq.metric_expression(table, metric, 'value >= ' ~ min_value ~ ' and value <= ' ~ max_value, column_name, condition) }}
+{% endtest %}
+
+{# new test macros #}
+
+{% test assert_true(model, column_name=None, metric=None, expression=expression, condition=None) %}
+    -- depends_on: {{ ref('f2dq_base_metrics') }}
+    {% if execute %}
+        {{ f2dq.metric_expression(model, metric, expression, column_name, condition) }}
+    {% else %}
+        {{ f2dq.empty_table() }}
+    {% endif %}
+{% endtest %}
+
+{% test assert_false(model, column_name=None, metric=None, expression=expression, condition=None) %}
+    -- depends_on: {{ ref('f2dq_base_metrics') }}
+    {% if execute %}
+        {{ f2dq.metric_expression(model, metric, 'not (' ~ expression ~ ')', column_name, condition) }}
+    {% else %}
+        {{ f2dq.empty_table() }}
+    {% endif %}
+{% endtest %}
+
+{% test assert_in_range(model, column_name=None, metric=None, min_value=None, max_value=None, condition=None) %}
+    -- depends_on: {{ ref('f2dq_base_metrics') }}
+    {% if execute %}
+        {{ f2dq.metric_expression(model, metric, 'value >= ' ~ min_value ~ ' and value <= ' ~ max_value, column_name, condition) }}
+    {% else %}
+        {{ f2dq.empty_table() }}
+    {% endif %}
+{% endtest %}
+
+{% test assert_equal(model, column_name=None, metric=None, value=value, condition=None) %}
+    -- depends_on: {{ ref('f2dq_base_metrics') }}
+    {% if execute %}
+        {{ f2dq.metric_expression(model, metric, 'value = ' ~ value, column_name, condition) }}
+    {% else %}
+        {{ f2dq.empty_table() }}
+    {% endif %}
+{% endtest %}
+
+{% test assert_greater(model, column_name=None, metric=None, value=None, condition=None) %}
+    -- depends_on: {{ ref('f2dq_base_metrics') }}
+    {% if execute %}
+        {{ f2dq.metric_expression(model, metric, 'value > ' ~ value, column_name, condition) }}
+    {% else %}
+        {{ f2dq.empty_table() }}
+    {% endif %}
+{% endtest %}
+
+{% test assert_greater_equal(model, column_name=None, metric=None, value=None, condition=None) %}
+    -- depends_on: {{ ref('f2dq_base_metrics') }}
+    {% if execute %}
+        {{ f2dq.metric_expression(model, metric, 'value >= ' ~ value, column_name, condition) }}
+    {% else %}
+        {{ f2dq.empty_table() }}
+    {% endif %}
+{% endtest %}
+
+{% test assert_less(model, column_name=None, metric=None, value=None, condition=None) %}
+    -- depends_on: {{ ref('f2dq_base_metrics') }}
+    {% if execute %}
+        {{ f2dq.metric_expression(model, metric, 'value < ' ~ value, column_name, condition) }}
+    {% else %}
+        {{ f2dq.empty_table() }}
+    {% endif %}
+{% endtest %}
+
+{% test assert_less_equal(model, column_name=None, metric=None, value=None, condition=None) %}
+    -- depends_on: {{ ref('f2dq_base_metrics') }}
+    {% if execute %}
+        {{ f2dq.metric_expression(model, metric, 'value <= ' ~ value, column_name, condition) }}
+    {% else %}
+        {{ f2dq.empty_table() }}
+    {% endif %}
+{% endtest %}

macros/utils/agate/row_value.sql

@@ -0,0 +1,20 @@
+
+{% macro row_value(agate_row, column) %}
+    {{ return (agate_row[f2dq.name_in_db(column)]) }}
+{% endmacro %}
+
+{% macro agate_to_list(table) %}
+    {% set col_names = table.column_names %}
+    {% set query_result = [] %}
+    {% for row in table.rows %}
+        {% set pairs = [] %}
+        {% for col_name in col_names %}
+            {% set value = row.get(col_name) | string %}
+            {% do pairs.append('"' ~ (col_name | lower) ~ '":' ~ '"' ~ (value | replace('"', '\\\"') | replace('\n', '\\n') ) ~ '"') %}
+        {% endfor %}
+        {% set joined_pairs = '{' ~ (pairs | join(',')) ~ '}' %}
+        {% do query_result.append(joined_pairs) %}
+    {% endfor %}
+    {% set query_result = '[' ~ (query_result | join(',')) ~ ']' %}
+    {{ return (query_result) }}
+{% endmacro %}

macros/utils/bool_to_string.sql

@@ -0,0 +1,8 @@
+
+{% macro bool_to_string(column) %}
+    (
+    case when {{ column }} = true then 'true'
+         when {{ column }} = false then 'false'
+    end
+    ) as {{ column }}
+{% endmacro %}

macros/utils/column_types.sql

@@ -0,0 +1,102 @@
+
+{% macro get_column_type(column) %}
+    {% set result = adapter.dispatch('get_column_type', 'f2dq')(column) %}
+    {{ return(result) }}
+{% endmacro %}
+
+
+{% macro default__get_column_type(column) %}
+    
+    {% if column.data_type in [
+        'character varying',
+        'varchar',
+        'character',
+        'char',
+        'text'
+    ] %}
+        {{ return('text') }}
+
+    {% elif column.data_type in [
+            'smallint',
+            'integer',
+            'bigint',
+            'decimal',
+            'numeric',
+            'real',
+            'double precision',
+            'enum',
+        ] %}
+        {{ return('numeric') }}
+    
+    {% elif column.data_type in [ 'boolean', 'bool' ] %}
+    
+        {{ return('boolean') }}
+
+    {% else %}
+        {{ return('unknown') }}
+
+    {% endif %}
+
+{% endmacro %}
+
+
+{% macro snowflake__get_column_type(column) %}
+
+    {% if column.DATA_TYPE in [
+        'VARCHAR',
+        'CHAR',
+        'CHARACTER',
+        'STRING',
+        'TEXT'
+    ] %}
+
+        {{ return('text') }}
+
+    {% elif column.DATA_TYPE in [
+            'NUMBER',
+            'DECIMAL',
+            'NUMERIC',
+            'INT',
+            'INTEGER',
+            'BIGINT',
+            'SMALLINT',
+            'TINYINT',
+            'BYTEINT',
+            'FLOAT',
+            'FLOAT4',
+            'FLOAT8',
+            'DOUBLE',
+            'DOUBLE PRECISION',
+            'REAL',
+    ] %}
+
+        {{ return('numeric') }}
+
+    {% elif column.data_type in [ 'BOOLEAN' ] %}
+    
+        {{ return('boolean') }}
+
+    {% else %}
+
+        {{ return('unknown') }}
+
+    {% endif %}
+
+{% endmacro %}
+
+
+{% macro bigquery__get_column_type(column) %}
+
+    {% if column.data_type in [ 'STRING' ] %}
+        {{ return('text') }}
+
+        {% elif column.data_type in [ "INT64", "NUMERIC", "BIGNUMERIC", "FLOAT64", "INTEGER"] %}
+        {{ return('numeric') }}
+
+        {% elif column.data_type in [ "BOOLEAN", "BOOL"] %}
+        {{ return('boolean') }}
+    {% else %}
+        {{ return('unknown') }}
+
+    {% endif %}
+{% endmacro %}

macros/utils/comma_delimited_list.sql

@@ -0,0 +1,5 @@
+{% macro comma_delimited_list(args) %}
+    {%- for arg in args %}
+        {{- arg -}} {{- ", " if not loop.last else "" -}}
+    {% endfor %}
+{% endmacro %}

macros/utils/comparison_text.sql

@@ -0,0 +1,5 @@
+{% macro comparison_text(a, b) %}
+    case when {{a}} > {{b}} then 'greater than' 
+    when {{a}} = {{b}} then 'equal to'
+    else 'less than' end
+{% endmacro %}

macros/utils/compat_dual_read.sql

@@ -0,0 +1,58 @@
+{# Dual-read helpers for migrating from re_data → f2dq without breaking
+   projects that still use re_data_* config keys or re_data:* vars. #}
+
+{% macro f2dq_var(name, default=none) %}
+    {# Prefer f2dq:<name>; fall back to re_data:<name>; then default.
+       For booleans, either namespace set to true wins (so a project that
+       still sets re_data:store_table_samples: true keeps working even
+       though the package default for f2dq:store_table_samples is false). #}
+    {% set fkey = 'f2dq:' ~ name %}
+    {% set rkey = 're_data:' ~ name %}
+    {% set fval = var(fkey) if var.has_var(fkey) else none %}
+    {% set rval = var(rkey) if var.has_var(rkey) else none %}
+    {% if fval is sameas true or rval is sameas true %}
+        {{ return(true) }}
+    {% elif fval is sameas false and rval is sameas false %}
+        {{ return(false) }}
+    {% elif fval is not none and fval is not sameas false %}
+        {{ return(fval) }}
+    {% elif rval is not none and rval is not sameas false %}
+        {{ return(rval) }}
+    {% elif fval is not none %}
+        {{ return(fval) }}
+    {% elif rval is not none %}
+        {{ return(rval) }}
+    {% else %}
+        {{ return(default) }}
+    {% endif %}
+{% endmacro %}
+
+{% macro f2dq_node_config(el, key, default=none) %}
+    {# Read f2dq_<key> / re_data_<key> from node config, then config.meta /
+       node.meta (dbt 1.12 CustomKeyInConfigDeprecation migration path).
+       `key` is the suffix without package prefix, e.g. 'monitored',
+       'time_filter', 'metrics_groups'. #}
+    {% set keys = ['f2dq_' ~ key, 're_data_' ~ key] %}
+    {% for k in keys %}
+        {% if el.config is mapping and el.config.get(k) is not none %}
+            {{ return(el.config.get(k)) }}
+        {% endif %}
+    {% endfor %}
+    {% set conf_meta = el.config.get('meta', {}) if el.config is mapping else {} %}
+    {% if conf_meta is mapping %}
+        {% for k in keys %}
+            {% if conf_meta.get(k) is not none %}
+                {{ return(conf_meta.get(k)) }}
+            {% endif %}
+        {% endfor %}
+    {% endif %}
+    {% set node_meta = el.get('meta', {}) if el is mapping else {} %}
+    {% if node_meta is mapping %}
+        {% for k in keys %}
+            {% if node_meta.get(k) is not none %}
+                {{ return(node_meta.get(k)) }}
+            {% endif %}
+        {% endfor %}
+    {% endif %}
+    {{ return(default) }}
+{% endmacro %}

macros/utils/deduplication/add_duplication_context.sql

@@ -0,0 +1,13 @@
+{% macro add_duplication_context(relation, unique_cols, sort_columns) %}
+
+            select {{ dbt_utils.star(from=relation) }}
+            , count(*) over (
+                 partition by {{ f2dq.comma_delimited_list(unique_cols) }} 
+            ) as f2dq_duplicate_group_row_count
+            , row_number() over (
+                partition by {{ f2dq.comma_delimited_list(unique_cols) }} {% if sort_columns %} order by {{ f2dq.comma_delimited_list(sort_columns) }} {% endif %}
+            ) as f2dq_duplicate_group_row_number
+
+            from {{ relation }}
+
+{% endmacro %}

macros/utils/depends_macro.sql

@@ -0,0 +1,6 @@
+{% macro generate_depends(used_tables) %}
+    {% for t in used_tables %}
+        -- depends_on: {{ ref(t) }}
+    {% endfor %}
+
+{% endmacro %}

macros/utils/dict_from_list.sql

@@ -0,0 +1,14 @@
+
+{% macro dict_from_list(el_list) %}
+
+    {% if el_list is none %}
+        {{ return (none) }}
+    {% endif %}
+
+    {% set for_cols_dict = {} %}
+    {% for col in el_list %}
+        {% do for_cols_dict.update({col: True})%}
+    {% endfor %}
+    {% do return(for_cols_dict) %}
+
+{% endmacro %}

macros/utils/fivetran_utils/json_extract.sql

@@ -0,0 +1,40 @@
+{#
+# This file contains significant part of code derived from 
+# https://github.com/fivetran/dbt_fivetran_utils/tree/v0.4.0 which is licensed under Apache License 2.0.
+#}
+
+{% macro json_extract(string, string_path) -%}
+
+{{ adapter.dispatch('json_extract','f2dq') (string, string_path) }}
+
+{%- endmacro %}
+
+{% macro default__json_extract(string, string_path) %}
+
+  json_extract_path_text({{string}}, {{ "'" ~ string_path ~ "'" }} )
+ 
+{% endmacro %}
+
+{% macro snowflake__json_extract(string, string_path) %}
+
+  json_extract_path_text(try_parse_json( {{string}} ), {{ "'" ~ string_path ~ "'" }} )
+
+{% endmacro %}
+
+{% macro redshift__json_extract(string, string_path) %}
+
+  case when is_valid_json( {{string}} ) then json_extract_path_text({{string}}, {{ "'" ~ string_path ~ "'" }} ) else null end
+ 
+{% endmacro %}
+
+{% macro bigquery__json_extract(string, string_path) %}
+
+  json_extract_scalar({{string}}, {{ "'$." ~ string_path ~ "'" }} )
+
+{% endmacro %}
+
+{% macro postgres__json_extract(string, string_path) %}
+
+  {{string}}::json->>{{"'" ~ string_path ~ "'" }}
+
+{% endmacro %}

macros/utils/fivetran_utils/percentile.sql

@@ -0,0 +1,60 @@
+{#
+# This file contains significant part of code derived from 
+# https://github.com/fivetran/dbt_fivetran_utils/tree/v0.4.0 which is licensed under Apache License 2.0.
+#}
+
+{% macro percentile(percentile_field, partition_field, percent) -%}
+
+{{ adapter.dispatch('percentile','f2dq') (percentile_field, partition_field, percent) }}
+
+{%- endmacro %}
+
+--percentile calculation specific to Redshift
+{% macro default__percentile(percentile_field, partition_field, percent)  %}
+
+    percentile_cont( 
+        {{ percent }} )
+        within group ( order by {{ percentile_field }} )
+        over ( partition by {{ partition_field }} )
+
+{% endmacro %}
+
+--percentile calculation specific to Redshift
+{% macro redshift__percentile(percentile_field, partition_field, percent)  %}
+
+    percentile_cont( 
+        {{ percent }} )
+        within group ( order by {{ percentile_field }} )
+        over ( partition by {{ partition_field }} )
+
+{% endmacro %}
+
+--percentile calculation specific to BigQuery
+{% macro bigquery__percentile(percentile_field, partition_field, percent)  %}
+
+    percentile_cont( 
+        {{ percentile_field }}, 
+        {{ percent }}) 
+        over (partition by {{ partition_field }}    
+        )
+
+{% endmacro %}
+
+{% macro postgres__percentile(percentile_field, partition_field, percent)  %}
+
+    percentile_cont( 
+        {{ percent }} )
+        within group ( order by {{ percentile_field }} )
+    /* have to group by partition field */
+
+{% endmacro %}
+
+{% macro spark__percentile(percentile_field, partition_field, percent)  %}
+
+    percentile( 
+        {{ percentile_field }}, 
+        {{ percent }}) 
+        over (partition by {{ partition_field }}    
+        )
+
+{% endmacro %}

macros/utils/for_loops.sql

@@ -0,0 +1,4 @@
+
+{% macro print_list(l) %}
+    {% for el in l %}{{el}}{% if not loop.last %},{% endif %}{% endfor %}
+{% endmacro %}

macros/utils/formulas.sql

@@ -0,0 +1,10 @@
+{% macro percentage_formula(summation, total) %}
+    abs(
+        ( 
+            cast({{ summation }} as {{ numeric_type() }})
+        ) / 
+        nullif(
+            cast( {{ total }} as {{ numeric_type() }} )
+        , 0) * 100.0
+    )
+{% endmacro %}

macros/utils/generate_alert_message.sql

@@ -0,0 +1,65 @@
+{% macro generate_anomaly_message(column_name, metric, last_value, last_avg) -%}
+  {{ return(adapter.dispatch('generate_anomaly_message', 'f2dq')(column_name, metric, last_value, last_avg)) }}
+{%- endmacro %}
+
+{% macro default__generate_anomaly_message(column_name, metric, last_value, last_avg) %}
+
+    case when {{ column_name }} != '' then metric || '(' || column_name || ')'
+    else metric
+    end 
+    || ' is ' ||
+    {{ to_2dp( percentage_formula('last_value - last_avg', last_avg) ) }}
+    || '% ' ||
+    {{ comparison_text(last_value, last_avg) }}
+    || ' average.'
+{% endmacro %}
+
+{% macro to_2dp(val) %}
+    {{ adapter.dispatch('to_2dp', 'f2dq')(val) }}
+{% endmacro %}
+
+{% macro default__to_2dp(val) %}
+    trim(to_char({{ val }}, '9999999999999999990D00'))
+{% endmacro %}
+
+{% macro bigquery__to_2dp(val) %}
+    format('%.2f', {{ val }})
+{% endmacro %}
+
+{% macro seconds_to_hours(val) %}
+    cast({{ val }} as {{ numeric_type() }}) / 3600
+{% endmacro %}
+
+{% macro generate_metric_value_text(metric, value) %}
+    case 
+        when {{ metric }} = 'freshness' 
+            then cast({{ to_2dp(seconds_to_hours(value)) }} as {{ string_type() }}) || ' hours'
+        when {{ regex_match_expression(metric, 'percent') }} 
+            then cast({{ to_2dp(value) }} as {{ string_type() }}) || '%'
+        when {{ regex_match_expression(metric, 'count') }} 
+            then cast({{ value }} as {{ string_type() }})
+        else cast({{ to_2dp(value) }} as {{ string_type() }})
+    end
+
+{% endmacro %}
+
+{% macro generate_schema_change_message(operation, column_name, prev_column_name, prev_data_type, data_type, detected_time) %}
+    case 
+        when {{ operation }} = 'column_added'
+            then 'column ' || {{ column_name }} || ' of type ' || {{ data_type }} || ' was added.'
+        when {{ operation }} = 'column_removed'
+            then 'column ' || {{ prev_column_name }} || ' of type ' || {{ prev_data_type }} || ' was removed.'
+        when {{ operation }} = 'type_change'
+            then {{ column_name }} || ' column data type was changed from ' || {{ prev_data_type }} || ' to ' || {{ data_type }} || '.'
+        else ''
+    end
+{% endmacro %}
+
+{% macro generate_failed_test_message(test_name, column_name) %}
+    case 
+        when {{ column_name }} is null
+            then 'Test ' || {{ test_name }} || ' failed.'
+        else
+            'Test ' || {{ test_name }} || ' failed for column ' || {{ column_name }} || '.'
+    end
+{% endmacro %}

macros/utils/get_database.sql

@@ -0,0 +1,16 @@
+
+{% macro get_target_database() %}
+    {{- adapter.dispatch('get_target_database', 'f2dq')() -}}
+{% endmacro %}
+
+{% macro default__get_target_database() %}
+    {{- return (target.dbname) -}}
+{% endmacro %}
+
+{% macro bigquery__get_target_database() %}
+    {{- return (target.project) -}}
+{% endmacro %}
+
+{% macro snowflake__get_target_database() %}
+    {{- return (target.database) -}}
+{% endmacro %}

macros/utils/in_compile.sql

@@ -0,0 +1,16 @@
+{% macro in_compile() %}
+
+    {%- call statement('in_compile', fetch_result=True) -%}
+        select * from {{ ref('f2dq_run_started_at') }}
+    {%- endcall -%}
+
+    {% if execute %}
+        {%- set result = load_result('in_compile')['data'][0][0] -%}
+        {% if result == run_started_at.timestamp() * 1000000 %}
+            {{ return(False) }}
+        {% else %}
+            {{ return(True) }}
+        {% endif %}
+    {% endif %}
+
+{% endmacro %}

macros/utils/is_list.sql

@@ -0,0 +1,7 @@
+{% macro is_list(obj) %}
+    {% if not obj %}
+        {{ return (False) }}
+    {% endif %}
+    {% set check = obj is iterable and (obj is not string and obj is not mapping) %}
+    {{ return (check) }}
+{% endmacro %}

macros/utils/json/to_single_json.sql

@@ -0,0 +1,22 @@
+{% macro to_json_string_value_or_null(column) %}
+    (
+        case 
+            when {{ column }} is null then 'null'
+            else '"' ||
+                regexp_replace(
+                    replace(cast({{ column }} as {{ string_type() }}), '"', {{ escape_seq_for_json('"') }}),
+                    '\n', {{ quote_new_line() }} {% if target.type in postgres_type_db() %}, 'g' {% endif %}
+                ) || '"'
+        end
+    )
+{% endmacro %}
+
+{% macro to_single_json(columns) %}
+    '{' ||
+    {%- for column in columns %}
+        '"{{ column }}": ' ||
+        {{ to_json_string_value_or_null(column) }}
+        {%- if not loop.last %} || ',' || {%- endif %}
+    {%- endfor %}
+    || '}'
+{% endmacro %}

macros/utils/mock/empty_tables.sql

@@ -0,0 +1,67 @@
+
+{% macro all_types_select() %}
+    with types_table as (
+        select
+            cast (null as {{ string_type() }}) as string_type,
+            cast (null as {{ long_string_type() }}) as long_string_type,
+            cast (1 as {{ numeric_type() }}) as numeric_type,
+            cast ('2000-01-10' as {{ timestamp_type() }}) as timestamp_type,
+            cast (true as {{ boolean_type() }}) as boolean_type
+    )
+{% endmacro %}
+
+{% macro empty_table_generic(list) %}
+    {{ f2dq.all_types_select() }}
+    select
+    {% for name, type in list %}
+         {{ type }}_type as {{ name }}
+        {%- if not loop.last %}, {%- endif %}
+    {% endfor %}
+    from types_table
+    where string_type is not null
+{% endmacro %}
+
+{% macro empty_last_base_metrics() %}
+    {{
+        f2dq.empty_table_generic([
+            ('table_name', 'string'),
+            ('column_name', 'string'),
+            ('metric', 'string'),
+            ('value', 'numeric')
+        ])
+    }}
+{% endmacro %}
+
+{% macro empty_last_table_samples() %}
+    {{
+        f2dq.empty_table_generic([
+            ('table_name', 'string'),
+            ('sample_data', 'string')
+        ])
+    }}
+{% endmacro %}
+
+{% macro empty_columns_table() %}
+    {{
+        f2dq.empty_table_generic([
+            ('name', 'string'),
+            ('schema', 'string'),
+            ('database', 'string'),
+            ('column_name', 'string'),
+            ('data_type', 'string'),
+            ('is_nullable', 'boolean'),
+            ('time_filter', 'string'),
+            ('computed_on', 'timestamp')
+        ])
+    }}
+{% endmacro %}
+
+
+{% macro empty_table() %}
+    {{
+        f2dq.empty_table_generic([
+            ('name', 'string')
+        ])
+    }}
+{% endmacro %}
+

macros/utils/monitored_config.sql

@@ -0,0 +1,38 @@
+
+
+{% macro columns_in_db(columns) %}
+    {% set translated = [] %}
+    {% if columns is none %}
+        {{ return (none) }}
+    {% endif %}
+    {% for col in columns %}
+        {% do translated.append(f2dq.name_in_db(col))%}
+    {% endfor %}
+    {{ return (translated) }}
+{% endmacro %}
+
+{% macro metrics_in_db(metrics) %}
+    {% set translated = metrics %}
+    {% set column_metrics = {} %}
+    {% for col in metrics.column %}
+        {% do column_metrics.update({f2dq.name_in_db(col): metrics.column[col]}) %}
+    {% endfor %}
+    {% if column_metrics %}
+        {% do metrics.update({'column': column_metrics}) %}
+    {% endif %}
+    {{ return (metrics) }}
+{% endmacro %}
+
+{% macro final_metrics(metrics_groups, additional_metrics) %}
+    {% set final_metrics_dict = dict([('group', {}), ('additional', {})]) %}
+    {% set all_metrics_groups = f2dq.f2dq_var('metrics_groups') %}
+
+    {% for group in metrics_groups %}
+        {% set value = all_metrics_groups.get(group) %}
+        {% do final_metrics_dict['group'].update(value) %}
+    {% endfor %}
+
+    {% do final_metrics_dict['additional'].update(additional_metrics) %}
+    {{ return (final_metrics_dict) }}
+
+{% endmacro %}

macros/utils/quote.sql

@@ -0,0 +1,13 @@
+
+
+{% macro quote_column(col_name) %}
+    {{ adapter.dispatch('quote_column', 'f2dq')(col_name) }}
+{% endmacro %}
+
+{% macro default__quote_column(col_name) %}
+    "{{ col_name }}"
+{% endmacro %}
+
+{% macro bigquery__quote_column(col_name) %}
+    `{{ col_name }}`
+{% endmacro %}

macros/utils/regular_expression.sql

@@ -0,0 +1,15 @@
+{% macro regex_match_expression(column_name, pattern) %}
+    {{ adapter.dispatch('regex_match_expression', 'f2dq')(column_name, pattern) }}
+{% endmacro %}
+
+{% macro default__regex_match_expression(column_name, pattern) %}
+    ({{column_name}} ~ '{{pattern}}')
+{% endmacro %}
+
+{% macro bigquery__regex_match_expression(column_name, pattern) %}
+    regexp_contains({{column_name}}, r'{{pattern}}')
+{% endmacro %}
+
+{% macro snowflake__regex_match_expression(column_name, pattern) %}
+    regexp_like({{column_name | upper}}, '{{pattern}}')
+{% endmacro %}

macros/utils/time_macros.sql

@@ -0,0 +1,100 @@
+
+{% macro time_window_start() %}
+    cast('{{- f2dq.f2dq_var('time_window_start') -}}' as timestamp)
+{% endmacro %}
+
+
+{% macro time_window_end() %}
+    cast('{{- f2dq.f2dq_var('time_window_end') -}}' as timestamp)
+{% endmacro %}
+
+
+{% macro anamaly_detection_time_window_start() %}
+   {{ adapter.dispatch('anamaly_detection_time_window_start', 'f2dq')() }}
+{% endmacro %}
+
+{% macro default__anamaly_detection_time_window_start() %}
+    {{ time_window_start() }} - interval '{{ f2dq.f2dq_var('anomaly_detection_look_back_days') }} days'
+{% endmacro %}
+
+{% macro bigquery__anamaly_detection_time_window_start() %}
+    DATE_ADD({{ time_window_start() }}, INTERVAL -{{ f2dq.f2dq_var('anomaly_detection_look_back_days') }} DAY)
+{% endmacro %}
+
+{% macro snowflake__anamaly_detection_time_window_start() %}
+    DATEADD('DAY', -{{- f2dq.f2dq_var('anomaly_detection_look_back_days') -}}, {{ time_window_start() }})
+{% endmacro %}
+
+
+{% macro interval_length_sec(start_timestamp, end_timestamp) %}
+    {{ adapter.dispatch('interval_length_sec', 'f2dq')(start_timestamp, end_timestamp) }}
+{% endmacro %}
+
+{% macro default__interval_length_sec(start_timestamp, end_timestamp) %}
+   EXTRACT(EPOCH FROM ({{ end_timestamp }} - {{ start_timestamp }} ))
+{% endmacro %}
+
+{% macro bigquery__interval_length_sec(start_timestamp, end_timestamp) %}
+    TIMESTAMP_DIFF ({{ end_timestamp }}, {{ start_timestamp }}, SECOND)
+{% endmacro %}
+
+{% macro snowflake__interval_length_sec(start_timestamp, end_timestamp) %}
+   timediff(second, {{ start_timestamp }}, {{ end_timestamp }})
+{% endmacro %}
+
+{% macro redshift__interval_length_sec(start_timestamp, end_timestamp) %}
+   DATEDIFF(second, {{ start_timestamp }}, {{ end_timestamp }})
+{% endmacro %}
+
+{%- macro in_time_window(time_column) %}
+    {# /* If not time_filter is specified, we compute the metric over the entire table else we filter for the time frame */ #}
+    {% if time_column is none %}
+            true
+    {% else %}
+        {{ adapter.dispatch('in_time_window', 'f2dq')(time_column) }}
+    {% endif %}
+{% endmacro -%}
+
+{% macro default__in_time_window(time_column) %}
+    {{time_column}} >= {{ time_window_start() }} and
+    {{time_column}} < {{ time_window_end() }}
+{% endmacro %}
+
+{% macro bigquery__in_time_window(time_column) %}
+    cast({{time_column}} as timestamp) >= {{ time_window_start() }} and
+    cast({{time_column}} as timestamp) < {{ time_window_end() }}
+{% endmacro %}
+
+
+{% macro format_timestamp(column_name) %}
+    {{ adapter.dispatch('format_timestamp', 'f2dq')(column_name) }}
+{% endmacro %}
+
+{% macro default__format_timestamp(column_name) %}
+    to_char({{column_name}}, 'YYYY-MM-DD HH24:MI:SS')
+{% endmacro %}
+
+{% macro bigquery__format_timestamp(column_name) %}
+    FORMAT_TIMESTAMP('%Y-%m-%d %H:%I:%S', {{column_name}})
+{% endmacro %}
+
+/*
+provide a common way to compare time vs a range: start_date <= target <= end_date
+if start_date is none: target <= end_date
+if end_date is none: target >= start_date
+think none as infinity
+*/
+{%- macro in_date_window(target, start_date, end_date) %}
+  {{ adapter.dispatch('in_date_window','f2dq')(target, start_date, end_date) }}
+{% endmacro -%}
+
+{% macro default__in_date_window(target, start_date, end_date) %}
+  {% if start_date is not none and end_date is not none %}
+    date({{target}}) between '{{start_date}}' and '{{end_date}}'
+  {% elif start_date is none %}
+    date({{target}}) <= '{{end_date}}'
+  {% elif end_date is none %}
+    date({{target}}) >= '{{start_date}}'
+  {% endif %}
+{% endmacro %}
+

macros/utils/used_types.sql

@@ -0,0 +1,100 @@
+{% macro timestamp_type() %}
+    {{ adapter.dispatch('timestamp_type', 'f2dq')() }}
+{% endmacro %}
+
+{% macro default__timestamp_type() %}
+    timestamp without time zone
+{% endmacro %}
+
+{% macro redshift__timestamp_type() %}
+    TIMESTAMP
+{% endmacro %}
+
+{% macro bigquery__timestamp_type() %}
+    TIMESTAMP
+{% endmacro %}
+
+{% macro snowflake__timestamp_type() %}
+    TIMESTAMP_NTZ
+{% endmacro %}
+
+{% macro string_type() %}
+    {{ adapter.dispatch('string_type', 'f2dq')() }}
+{% endmacro %}
+
+{% macro default__string_type() %}
+    text
+{% endmacro %}
+
+{% macro redshift__string_type() %}
+    varchar(2047)
+{% endmacro %}
+
+{% macro bigquery__string_type() %}
+    STRING
+{% endmacro %}
+
+{% macro snowflake__string_type() %}
+    STRING
+{% endmacro %}
+
+{% macro long_string_type() %}
+    {{ adapter.dispatch('long_string_type', 'f2dq')() }}
+{% endmacro %}
+
+{% macro default__long_string_type() %}
+    {{ f2dq.string_type() }}
+{% endmacro %}
+
+{% macro redshift__long_string_type() %}
+    varchar(65535)
+{% endmacro %}
+
+{% macro integer_type() %}
+    INTEGER
+{% endmacro %}
+
+
+{% macro boolean_type() %}
+    {{ adapter.dispatch('boolean_type', 'f2dq')() }}
+{% endmacro %}
+
+{% macro default__boolean_type() %}
+    BOOLEAN
+{% endmacro %}
+
+{% macro redshift__boolean_type() %}
+    boolean
+{% endmacro %}
+
+{% macro bigquery__boolean_type() %}
+    BOOLEAN
+{% endmacro %}
+
+{% macro snowflake__boolean_type() %}
+    BOOLEAN
+{% endmacro %}
+
+
+{% macro numeric_type() %}
+    {{ adapter.dispatch('numeric_type', 'f2dq')() }}
+{% endmacro %}
+
+{% macro default__numeric_type() %}
+    double precision
+{% endmacro %}
+
+{% macro redshift__numeric_type() %}
+    DOUBLE PRECISION
+{% endmacro %}
+
+{% macro bigquery__numeric_type() %}
+    FLOAT64
+{% endmacro %}
+
+{% macro snowflake__numeric_type() %}
+    FLOAT
+{% endmacro %}
+
+
+