{ "datasets": [ { "name": "0edc79ec", "displayName": "all_records_latest_eval", "queryLines": [ "-- DATA QUALITY INCIDENT TABLES (ALL STATUSES)\n", "\n", "WITH recent_records AS (\n", " SELECT *\n", " FROM system.data_quality_monitoring.table_results\n", " WHERE event_time >= DATEADD(day, -7, CURRENT_DATE)\n", "),\n", "\n", "-- Training error codes (model not yet trained or insufficient data)\n", "training_error_codes AS (\n", " SELECT ARRAY(\n", " 'NOT_ENOUGH_DISTINCT_DAYS',\n", " 'FAILED_TO_FIT_MODEL',\n", " 'FAILED_TO_FIT_MODEL_WITH_HIGH_RESIDUAL',\n", " 'NO_UPDATES_IN_TABLE_HISTORY',\n", " 'NOT_ENOUGH_UPDATE_OP_BACKTESTING',\n", " 'NOT_ENOUGH_UPDATE_OP',\n", " 'NOT_ENOUGH_TABLE_HISTORY',\n", " 'NOT_ENOUGH_TABLE_HISTORY_CUSTOM_SQL',\n", " 'USER_CONFIGURED_SKIP'\n", " ) AS codes\n", "),\n", "\n", "-- Service/User error codes (actual errors requiring attention)\n", "service_error_codes AS (\n", " SELECT ARRAY(\n", " 'ROW_COUNT_COLLECTION_ERROR',\n", " 'INTERNAL_ERROR',\n", " 'FAILED_TO_PREDICT',\n", " 'FAILED_TO_FETCH_HISTORY',\n", " 'PERMISSION_DENIED',\n", " 'USER_ERROR',\n", " 'MATERIALIZED_VIEW_HISTORY_ERROR',\n", " 'STREAMING_TABLE_HISTORY_ERROR',\n", " 'TOO_FREQUENTLY_UPDATING',\n", " 'LEGACY_MV_NOT_SUPPORTED',\n", " 'STORAGE_ACCESS_ERROR',\n", " 'TABLE_NOT_FOUND',\n", " 'BLAST_RADIUS_COMPUTATION_ERROR',\n", " 'JOB_RCA_COMPUTATION_ERROR'\n", " ) AS codes\n", "),\n", "\n", "-- Number of scans per table\n", "daily_counts AS (\n", " SELECT\n", " table_id,\n", " DATE(event_time) AS event_day,\n", " COUNT(*) AS row_count\n", " FROM recent_records\n", " WHERE event_time >= DATEADD(day, -7, CURRENT_DATE)\n", " GROUP BY\n", " table_id,\n", " DATE(event_time)\n", "),\n", "scan_freq AS (\n", " SELECT\n", " table_id,\n", " SUM(row_count) AS num_scans_in_last_week,\n", " ROUND(SUM(row_count) / 7) AS num_scans_in_last_one_day\n", " FROM daily_counts\n", " GROUP BY\n", " table_id\n", "),\n", "\n", "latest_event_times AS (\n", " SELECT table_id,\n", " MAX(event_time) AS latest_event_time\n", " FROM recent_records\n", " GROUP BY table_id\n", "),\n", "\n", "-- Get the latest non-null RCA for each table\n", "latest_rca AS (\n", " SELECT \n", " table_id,\n", " root_cause_analysis AS rca,\n", " ROW_NUMBER() OVER (PARTITION BY table_id ORDER BY event_time DESC) AS rn\n", " FROM recent_records\n", " WHERE root_cause_analysis IS NOT NULL\n", " AND root_cause_analysis.upstream_jobs IS NOT NULL\n", " AND SIZE(root_cause_analysis.upstream_jobs) > 0\n", "),\n", "latest_rca_per_table AS (\n", " SELECT table_id, rca\n", " FROM latest_rca\n", " WHERE rn = 1\n", "),\n", "\n", "latest_row AS (\n", " SELECT r.table_id,\n", " r.catalog_name,\n", " r.schema_name,\n", " r.table_name,\n", " r.status AS raw_status,\n", " r._internal_debug_info.status_since AS status_since,\n", " -- Collect all error codes from completeness, event_freshness, and commit_freshness\n", " ARRAY_DISTINCT(\n", " ARRAY_COMPACT(\n", " ARRAY(\n", " r.completeness.daily_row_count.error_code,\n", " r._internal_debug_info.event_freshness.error_code,\n", " r.freshness.commit_freshness.error_code\n", " )\n", " )\n", " ) AS error_codes,\n", " -- Unhealthy reason\n", " CASE\n", " WHEN LOWER(r.freshness.commit_freshness.status) = 'unhealthy' THEN 'Stale'\n", " WHEN LOWER(r._internal_debug_info.event_freshness.status) = 'unhealthy' THEN 'Stale'\n", " WHEN LOWER(r.completeness.daily_row_count.status) = 'unhealthy' THEN 'Incomplete'\n", " WHEN r._internal_debug_info.custom_sql IS NOT NULL \n", " AND SIZE(FILTER(r._internal_debug_info.custom_sql, x -> LOWER(x.status) = 'unhealthy')) > 0 THEN 'Invalid'\n", " ELSE NULL\n", " END AS unhealthy_reason,\n", " CASE\n", " WHEN LOWER(r.freshness.commit_freshness.status) = 'unhealthy' THEN 'COMMIT_FRESHNESS_REASON'\n", " WHEN LOWER(r._internal_debug_info.event_freshness.status) = 'unhealthy' THEN 'EVENT_FRESHNESS_REASON'\n", " WHEN LOWER(r.completeness.daily_row_count.status) = 'unhealthy' THEN 'COMPLETENESS_REASON'\n", " WHEN r._internal_debug_info.custom_sql IS NOT NULL \n", " AND SIZE(FILTER(r._internal_debug_info.custom_sql, x -> LOWER(x.status) = 'unhealthy')) > 0 THEN 'CUSTOM_SQL_REASON'\n", " ELSE NULL\n", " END AS reason_tooltip,\n", " CASE\n", " WHEN r.downstream_impact.impact_level >= 3 THEN 'High'\n", " WHEN r.downstream_impact.impact_level = 2 THEN 'Medium'\n", " WHEN r.downstream_impact.impact_level = 1 THEN 'Low'\n", " ELSE NULL\n", " END AS impact_level,\n", " r.downstream_impact.impact_level AS impact_level_raw,\n", " r.completeness.daily_row_count AS completeness,\n", " r._internal_debug_info.event_freshness AS event_freshness,\n", " r.freshness.commit_freshness AS commit_freshness,\n", " l.latest_event_time AS latest_event_time\n", " FROM recent_records r\n", " JOIN latest_event_times l\n", " ON r.table_id = l.table_id\n", " AND r.event_time = l.latest_event_time\n", "),\n", "\n", "parsed_status AS (\n", " SELECT \n", " lr.*,\n", " -- Status classification\n", " CASE\n", " WHEN LOWER(lr.raw_status) = 'healthy' THEN 'Healthy'\n", " WHEN LOWER(lr.raw_status) = 'unhealthy' THEN 'Unhealthy'\n", " -- Check training error codes first\n", " WHEN LOWER(lr.raw_status) = 'unknown' \n", " AND SIZE(ARRAY_INTERSECT(lr.error_codes, (SELECT codes FROM training_error_codes))) > 0 \n", " THEN 'Training'\n", " -- Check service/user error codes\n", " WHEN LOWER(lr.raw_status) = 'unknown' \n", " AND SIZE(ARRAY_INTERSECT(lr.error_codes, (SELECT codes FROM service_error_codes))) > 0 \n", " THEN 'Error'\n", " -- No error codes from completeness/freshness, likely from custom_sql (percent_null)\n", " WHEN LOWER(lr.raw_status) = 'unknown' THEN 'Training'\n", " ELSE lr.raw_status\n", " END AS status,\n", " -- Error code (first one from the list)\n", " CASE\n", " WHEN LOWER(lr.raw_status) = 'unknown' AND SIZE(lr.error_codes) > 0 \n", " THEN lr.error_codes[0]\n", " ELSE NULL\n", " END AS error_code,\n", " -- Reason: based on status provide different reasons\n", " CASE\n", " -- Unhealthy: Stale, Incomplete, or Invalid (custom_sql)\n", " WHEN LOWER(lr.raw_status) = 'unhealthy' THEN lr.unhealthy_reason\n", " -- Training: show friendly training reason\n", " WHEN LOWER(lr.raw_status) = 'unknown' \n", " AND SIZE(ARRAY_INTERSECT(lr.error_codes, (SELECT codes FROM training_error_codes))) > 0 \n", " THEN CASE lr.error_codes[0]\n", " WHEN 'NOT_ENOUGH_DISTINCT_DAYS' THEN 'Not enough data days'\n", " WHEN 'FAILED_TO_FIT_MODEL' THEN 'Model training failed'\n", " WHEN 'FAILED_TO_FIT_MODEL_WITH_HIGH_RESIDUAL' THEN 'Model has high residual'\n", " WHEN 'NO_UPDATES_IN_TABLE_HISTORY' THEN 'No table updates found'\n", " WHEN 'NOT_ENOUGH_UPDATE_OP_BACKTESTING' THEN 'Not enough updates for backtesting'\n", " WHEN 'NOT_ENOUGH_UPDATE_OP' THEN 'Not enough update operations'\n", " WHEN 'NOT_ENOUGH_TABLE_HISTORY' THEN 'Not enough table history'\n", " WHEN 'NOT_ENOUGH_TABLE_HISTORY_CUSTOM_SQL' THEN 'Not enough history for custom SQL'\n", " WHEN 'USER_CONFIGURED_SKIP' THEN 'User configured to skip'\n", " ELSE lr.error_codes[0]\n", " END\n", " -- Error: show friendly error reason\n", " WHEN LOWER(lr.raw_status) = 'unknown' \n", " AND SIZE(ARRAY_INTERSECT(lr.error_codes, (SELECT codes FROM service_error_codes))) > 0 \n", " THEN CASE lr.error_codes[0]\n", " WHEN 'ROW_COUNT_COLLECTION_ERROR' THEN 'Row count collection failed'\n", " WHEN 'INTERNAL_ERROR' THEN 'Internal error'\n", " WHEN 'FAILED_TO_PREDICT' THEN 'Prediction failed'\n", " WHEN 'FAILED_TO_FETCH_HISTORY' THEN 'Failed to fetch history'\n", " WHEN 'PERMISSION_DENIED' THEN 'Permission denied'\n", " WHEN 'USER_ERROR' THEN 'User configuration error'\n", " WHEN 'MATERIALIZED_VIEW_HISTORY_ERROR' THEN 'MV history error'\n", " WHEN 'STREAMING_TABLE_HISTORY_ERROR' THEN 'Streaming table history error'\n", " WHEN 'TOO_FREQUENTLY_UPDATING' THEN 'Table updating too frequently'\n", " WHEN 'LEGACY_MV_NOT_SUPPORTED' THEN 'Legacy MV not supported'\n", " WHEN 'STORAGE_ACCESS_ERROR' THEN 'Storage access error'\n", " WHEN 'TABLE_NOT_FOUND' THEN 'Table not found'\n", " WHEN 'BLAST_RADIUS_COMPUTATION_ERROR' THEN 'Blast radius computation failed'\n", " WHEN 'JOB_RCA_COMPUTATION_ERROR' THEN 'Job RCA computation failed'\n", " ELSE lr.error_codes[0]\n", " END\n", " -- Healthy: no reason needed\n", " WHEN LOWER(lr.raw_status) = 'healthy' THEN NULL\n", " -- Unknown with no categorized error codes: likely from custom_sql (percent_null)\n", " WHEN LOWER(lr.raw_status) = 'unknown' \n", " THEN '(perhaps) coming from percent_null, please check Quality tab for details'\n", " ELSE NULL\n", " END AS reason\n", " FROM latest_row lr\n", ")\n", "\n", "SELECT\n", " -- Combined table name: catalog.schema.table\n", " CONCAT(ps.catalog_name, '.', ps.schema_name, '.', ps.table_name) AS table_name,\n", " \n", " -- Table link\n", " CONCAT(\n", " '/explore/data/', ps.catalog_name, '/', ps.schema_name, '/', ps.table_name,\n", " '?activeTab=quality&activeSection=anomaly-detection'\n", " ) AS table_link,\n", " \n", " ps.status,\n", " ps.raw_status,\n", " ps.reason,\n", " ps.error_code,\n", " ps.error_codes,\n", " ps.status_since,\n", " ps.reason_tooltip,\n", " ps.impact_level,\n", " \n", " -- RCA from latest_rca_per_table\n", " CASE\n", " WHEN rca_tbl.rca IS NOT NULL \n", " AND SIZE(rca_tbl.rca.upstream_jobs) > 0\n", " THEN CONCAT(\n", " 'Failing job: ',\n", " rca_tbl.rca.upstream_jobs[0].job_name,\n", " ' (', rca_tbl.rca.upstream_jobs[0].last_run_status, ')',\n", " CASE \n", " WHEN SIZE(rca_tbl.rca.upstream_jobs) > 1 \n", " THEN CONCAT(' +', SIZE(rca_tbl.rca.upstream_jobs) - 1, ' more')\n", " ELSE ''\n", " END\n", " )\n", " ELSE ''\n", " END AS root_cause_analysis,\n", " \n", " -- RCA job link\n", " CASE\n", " WHEN rca_tbl.rca IS NOT NULL \n", " AND SIZE(rca_tbl.rca.upstream_jobs) > 0\n", " THEN rca_tbl.rca.upstream_jobs[0].run_page_url\n", " ELSE NULL\n", " END AS rca_job_link,\n", " \n", " ps.completeness,\n", " ps.event_freshness,\n", " ps.commit_freshness,\n", " COALESCE(sf.num_scans_in_last_week, 0) AS num_scans_in_last_week,\n", " COALESCE(sf.num_scans_in_last_one_day, 0) AS num_scans_in_last_one_day,\n", " \n", " -- Scan frequency label\n", " CASE\n", " WHEN COALESCE(sf.num_scans_in_last_week, 0) <= 0 AND COALESCE(sf.num_scans_in_last_one_day, 0) <= 0 \n", " THEN 'None'\n", " ELSE CONCAT('Every ', \n", " CASE\n", " WHEN COALESCE(sf.num_scans_in_last_one_day, 0) > 0 \n", " AND (24.0 / sf.num_scans_in_last_one_day) < 20 \n", " THEN CONCAT(CEIL(24.0 / sf.num_scans_in_last_one_day), ' hour(s)')\n", " WHEN COALESCE(sf.num_scans_in_last_week, 0) > 0 \n", " AND (7.0 / sf.num_scans_in_last_week) < 5 \n", " THEN CONCAT(CEIL(7.0 / sf.num_scans_in_last_week), ' day(s)')\n", " WHEN COALESCE(sf.num_scans_in_last_week, 0) > 0 \n", " THEN CONCAT(CEIL(7.0 / sf.num_scans_in_last_week / 7), ' week(s)')\n", " ELSE '1 week'\n", " END\n", " )\n", " END AS scan_frequency_label,\n", " \n", " -- Last scanned (human-readable)\n", " CASE\n", " WHEN TIMESTAMPDIFF(MINUTE, ps.latest_event_time, CURRENT_TIMESTAMP()) < 1 \n", " THEN 'Just now'\n", " WHEN TIMESTAMPDIFF(MINUTE, ps.latest_event_time, CURRENT_TIMESTAMP()) < 60 \n", " THEN CONCAT(TIMESTAMPDIFF(MINUTE, ps.latest_event_time, CURRENT_TIMESTAMP()), ' minute(s) ago')\n", " WHEN TIMESTAMPDIFF(HOUR, ps.latest_event_time, CURRENT_TIMESTAMP()) < 24 \n", " THEN CONCAT(TIMESTAMPDIFF(HOUR, ps.latest_event_time, CURRENT_TIMESTAMP()), ' hour(s) ago')\n", " WHEN TIMESTAMPDIFF(DAY, ps.latest_event_time, CURRENT_TIMESTAMP()) < 7 \n", " THEN CONCAT(TIMESTAMPDIFF(DAY, ps.latest_event_time, CURRENT_TIMESTAMP()), ' day(s) ago')\n", " ELSE CONCAT(TIMESTAMPDIFF(WEEK, ps.latest_event_time, CURRENT_TIMESTAMP()), ' week(s) ago')\n", " END AS last_scanned,\n", " \n", " ps.latest_event_time AS last_scanned_timestamp\n", "\n", "FROM parsed_status ps\n", "LEFT JOIN scan_freq sf ON sf.table_id = ps.table_id\n", "LEFT JOIN latest_rca_per_table rca_tbl ON rca_tbl.table_id = ps.table_id\n", "ORDER BY\n", " CASE ps.status\n", " WHEN 'Unhealthy' THEN 1\n", " WHEN 'Error' THEN 2\n", " WHEN 'Training' THEN 3\n", " WHEN 'Healthy' THEN 4\n", " ELSE 5\n", " END ASC,\n", " COALESCE(ps.impact_level_raw, 0) DESC,\n", " ps.status_since ASC" ] }, { "name": "0883446b", "displayName": "percent_healthy_over_time", "queryLines": [ "WITH AllEvaluationDates AS (\n", " SELECT DISTINCT CAST(event_time AS DATE) AS evaluation_date\n", " FROM system.data_quality_monitoring.table_results\n", "),\n", "AllTables AS (\n", " SELECT DISTINCT table_id\n", " FROM system.data_quality_monitoring.table_results\n", "),\n", "LatestStatusAsOfDate AS (\n", " SELECT\n", " t.table_id,\n", " d.evaluation_date,\n", " /* latest row as-of that day */\n", " CASE UPPER(s.status)\n", " WHEN 'HEALTHY' THEN 'Healthy'\n", " WHEN 'UNHEALTHY' THEN 'Unhealthy'\n", " ELSE 'Unknown'\n", " END AS final_status,\n", " s.event_time,\n", " ROW_NUMBER() OVER (\n", " PARTITION BY t.table_id, d.evaluation_date\n", " ORDER BY s.event_time DESC\n", " ) AS rn\n", " FROM AllTables t\n", " CROSS JOIN AllEvaluationDates d\n", " LEFT JOIN (\n", " SELECT\n", " table_id,\n", " status,\n", " event_time,\n", " CAST(event_time AS DATE) AS eval_date\n", " FROM system.data_quality_monitoring.table_results\n", " ) s\n", " ON t.table_id = s.table_id\n", " AND s.eval_date <= d.evaluation_date\n", "),\n", "DailyAggregates AS (\n", " SELECT\n", " evaluation_date,\n", " COUNT(DISTINCT CASE WHEN final_status IN ('Healthy','Unhealthy') THEN table_id END) AS total_tables,\n", " COUNT(DISTINCT CASE WHEN final_status = 'Healthy' THEN table_id END) AS healthy_tables\n", " FROM LatestStatusAsOfDate\n", " WHERE rn = 1 -- keep the latest-as-of record per table/day\n", " AND final_status IS NOT NULL\n", " GROUP BY evaluation_date\n", ")\n", "SELECT\n", " evaluation_date,\n", " healthy_tables,\n", " total_tables,\n", " ROUND(100.0 * healthy_tables / NULLIF(total_tables, 0), 2) AS percent_healthy\n", "FROM DailyAggregates\n", "WHERE total_tables > 0\n", "ORDER BY evaluation_date DESC;" ] }, { "name": "6a0f2ffc", "displayName": "schema_latest_eval_global", "queryLines": [ "WITH latest AS (\n", " SELECT *\n", " FROM (\n", " SELECT\n", " ROW_NUMBER() OVER (PARTITION BY table_id ORDER BY event_time DESC) AS rn,\n", " CASE UPPER(status)\n", " WHEN 'HEALTHY' THEN 'Healthy'\n", " WHEN 'UNHEALTHY' THEN 'Unhealthy'\n", " ELSE 'Unknown'\n", " END AS status_mapped\n", " FROM system.data_quality_monitoring.table_results\n", " ) t\n", " WHERE rn = 1\n", "),\n", "counts AS (\n", " SELECT\n", " status_mapped AS status,\n", " COUNT(*) AS cnt\n", " FROM latest\n", " WHERE status_mapped IN ('Healthy','Unhealthy','Unknown')\n", " GROUP BY status_mapped\n", "),\n", "tot AS (\n", " SELECT COUNT(*) AS total\n", " FROM latest\n", " WHERE status_mapped IN ('Healthy','Unhealthy','Unknown')\n", "),\n", "wanted AS (\n", " SELECT * FROM VALUES ('Healthy'), ('Unhealthy'), ('Unknown') AS s(status)\n", ")\n", "SELECT\n", " w.status,\n", " COALESCE(c.cnt, 0) AS count,\n", " ROUND(COALESCE(c.cnt, 0) / NULLIF(CAST(t.total AS DOUBLE), 0), 4) AS percentage\n", "FROM wanted w\n", "LEFT JOIN counts c ON c.status = w.status\n", "CROSS JOIN tot t;" ] }, { "name": "67d6a2ab", "displayName": "static_table_cleanup", "queryLines": [ "WITH LatestChecks AS (\n", " SELECT\n", " CONCAT(catalog_name, \".\", schema_name, \".\", table_name) AS full_table_name,\n", " CONCAT(\n", " \"/explore/data/\", catalog_name, \"/\", schema_name, \"/\", table_name,\n", " \"?activeTab=quality&activeSection=anomaly-detection\"\n", " ) AS table_link,\n", "\n", " event_time,\n", " _internal_debug_info.is_static AS is_static,\n", " freshness.commit_freshness.last_value AS last_data_update_timestamp,\n", "\n", " ROW_NUMBER() OVER (\n", " PARTITION BY table_id\n", " ORDER BY event_time DESC\n", " ) AS rn\n", " FROM system.data_quality_monitoring.table_results\n", " WHERE (:table_name = 'All tables' OR table_name = :table_name)\n", " AND (:catalog = 'All catalogs' OR catalog_name = :catalog)\n", " AND (:schema = 'All schemas' OR schema_name = :schema)\n", ")\n", "SELECT\n", " event_time AS evaluated_at,\n", " full_table_name AS table_name,\n", " table_link,\n", " last_data_update_timestamp\n", "FROM LatestChecks\n", "WHERE rn = 1\n", " AND is_static = TRUE\n", "ORDER BY\n", " last_data_update_timestamp ASC,\n", " evaluated_at DESC;\n" ], "parameters": [ { "displayName": "table_name", "keyword": "table_name", "dataType": "STRING", "defaultSelection": { "values": { "dataType": "STRING", "values": [ { "value": "All tables" } ] } } }, { "displayName": "catalog", "keyword": "catalog", "dataType": "STRING", "defaultSelection": { "values": { "dataType": "STRING", "values": [ { "value": "All catalogs" } ] } } }, { "displayName": "schema", "keyword": "schema", "dataType": "STRING", "defaultSelection": { "values": { "dataType": "STRING", "values": [ { "value": "All schemas" } ] } } } ] }, { "name": "b53c4849", "displayName": "warning", "queryLines": ["SELECT \"Logging table for your schema(s) is required\" AS warning\n"] }, { "name": "e8e644bb", "displayName": "impact_values", "queryLines": [ "SELECT impact FROM (\n", " VALUES ('All'), ('High'), ('Medium'), ('Low'), ('Unknown')\n", ") AS impact_levels(impact);\n" ] }, { "name": "dcb2c77a", "displayName": "status", "queryLines": [ "SELECT status FROM (\n", " VALUES ('All'), ('Healthy'), ('Unhealthy'), ('Error'), ('Training'),('Unknown')\n", ") AS status(status);\n" ] } ], "pages": [ { "name": "quality-overview", "displayName": "Quality Overview", "layout": [ { "widget": { "name": "5a7d2707", "multilineTextboxSpec": { "lines": [ "# Data Quality Monitoring \n", "\n", "Last updated: Jan 26, 2026, v1.0\n", "\n", "Data Quality Monitoring analyzes the historical data patterns of every table to intelligently detect anomalies. All datasets are scanned once and then only important tables are rescanned based on their update patterns. " ] } }, "position": { "x": 0, "y": 0, "width": 6, "height": 3 } }, { "widget": { "name": "d00b15dd", "queries": [ { "name": "main_query", "query": { "datasetName": "0edc79ec", "fields": [ { "name": "table_name", "expression": "`table_name`" }, { "name": "table_link", "expression": "`table_link`" }, { "name": "status", "expression": "`status`" }, { "name": "last_scanned", "expression": "`last_scanned`" }, { "name": "impact_level", "expression": "`impact_level`" }, { "name": "status_since", "expression": "`status_since`" }, { "name": "scan_frequency_label", "expression": "`scan_frequency_label`" }, { "name": "reason", "expression": "`reason`" }, { "name": "rca_job_link", "expression": "`rca_job_link`" }, { "name": "root_cause_analysis", "expression": "`root_cause_analysis`" } ], "disaggregated": true } } ], "spec": { "version": 1, "widgetType": "table", "encodings": { "columns": [ { "fieldName": "table_name", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": false, "order": 0, "title": "table_name", "allowSearch": true, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "defaultColumnWidth": 500 }, { "fieldName": "table_link", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ table_name }}", "linkTitleTemplate": "{{ table_name }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "link", "visible": true, "order": 1, "title": "Table", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false }, { "fieldName": "status", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 2, "title": "Status", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "cellFormat": { "default": { "foregroundColor": null }, "rules": [ { "if": { "column": "status", "fn": "=", "literal": "Unhealthy" }, "value": { "foregroundColor": "#FF3621" } }, { "if": { "column": "status", "fn": "=", "literal": "Healthy" }, "value": { "foregroundColor": "#00A972" } }, { "if": { "column": "status", "fn": "=", "literal": "Unknown" }, "value": { "foregroundColor": "#FFAB00" } } ] } }, { "fieldName": "last_scanned", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 3, "title": "Last Scanned", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "cellFormat": { "default": { "foregroundColor": { "themeColorType": "fontColor" } }, "rules": [] } }, { "fieldName": "impact_level", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 7, "title": "Impact", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "description": "Based on historical downstream query volume" }, { "fieldName": "status_since", "dateTimeFormat": "DD/MM/YYYY HH:mm", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "datetime", "displayAs": "datetime", "visible": true, "order": 8, "title": "Status Since", "allowSearch": false, "alignContent": "right", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false }, { "fieldName": "scan_frequency_label", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 14, "title": "Scan Frequency", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "description": "How often this table was typically scanned over the past week." }, { "fieldName": "reason", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "string", "visible": true, "order": 17, "title": "Reason", "allowSearch": false, "alignContent": "left", "allowHTML": true, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "cellFormat": { "default": { "foregroundColor": null }, "rules": [] }, "defaultColumnWidth": 250 }, { "fieldName": "rca_job_link", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ root_cause_analysis }}", "linkTitleTemplate": "{{ root_cause_analysis }}", "linkOpenInNewTab": true, "type": "string", "displayAs": "link", "visible": true, "order": 19, "title": "Root Cause", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "description": "" } ] }, "invisibleColumns": [ { "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "raw_status", "type": "string", "displayAs": "string", "order": 4, "title": "raw_status", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false }, { "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "error_code", "type": "string", "displayAs": "string", "order": 5, "title": "error_code", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false }, { "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "error_codes", "type": "complex", "displayAs": "json", "order": 6, "title": "error_codes", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false }, { "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "completeness", "type": "complex", "displayAs": "json", "order": 9, "title": "completeness", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false }, { "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "event_freshness", "type": "complex", "displayAs": "json", "order": 10, "title": "event_freshness", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false }, { "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "commit_freshness", "type": "complex", "displayAs": "json", "order": 11, "title": "commit_freshness", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false }, { "numberFormat": "0", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "num_scans_in_last_week", "type": "integer", "displayAs": "number", "order": 12, "title": "num_scans_in_last_week", "allowSearch": false, "alignContent": "right", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false }, { "numberFormat": "0.00", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "num_scans_in_last_one_day", "type": "float", "displayAs": "number", "order": 13, "title": "num_scans_in_last_one_day", "allowSearch": false, "alignContent": "right", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false }, { "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "reason_tooltip", "type": "string", "displayAs": "string", "order": 15, "title": "reason_tooltip", "allowSearch": false, "alignContent": "left", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false }, { "dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS", "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "last_scanned_timestamp", "type": "datetime", "displayAs": "datetime", "order": 16, "title": "last_scanned_timestamp", "allowSearch": false, "alignContent": "right", "allowHTML": false, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false }, { "booleanValues": ["false", "true"], "imageUrlTemplate": "{{ @ }}", "imageTitleTemplate": "{{ @ }}", "imageWidth": "", "imageHeight": "", "linkUrlTemplate": "{{ @ }}", "linkTextTemplate": "{{ @ }}", "linkTitleTemplate": "{{ @ }}", "linkOpenInNewTab": true, "name": "root_cause_analysis", "type": "string", "displayAs": "string", "order": 18, "title": "root_cause_analysis", "allowSearch": false, "alignContent": "left", "allowHTML": true, "highlightLinks": false, "useMonospaceFont": false, "preserveWhitespace": false, "defaultColumnWidth": 200 } ], "allowHTMLByDefault": false, "itemsPerPage": 25, "paginationSize": "default", "condensed": true, "withRowNumber": false, "firstNFrozenColumns": 0, "frame": { "showTitle": true, "title": "Unhealthy tables", "showDescription": false, "description": "" } } }, "position": { "x": 0, "y": 9, "width": 6, "height": 9 } }, { "widget": { "name": "abf760b7", "queries": [ { "name": "main_query", "query": { "datasetName": "0883446b", "fields": [ { "name": "evaluation_date", "expression": "`evaluation_date`" }, { "name": "percent_healthy", "expression": "`percent_healthy`" } ], "disaggregated": true } } ], "spec": { "version": 3, "widgetType": "line", "encodings": { "x": { "fieldName": "evaluation_date", "scale": { "type": "temporal" }, "displayName": "evaluation date " }, "y": { "fieldName": "percent_healthy", "scale": { "type": "quantitative", "domain": { "min": 0, "max": 100 } }, "displayName": "% healthy table" } }, "frame": { "showTitle": true, "title": "Healthy tables over time" } } }, "position": { "x": 3, "y": 3, "width": 3, "height": 3 } }, { "widget": { "name": "a0d6f9d2", "queries": [ { "name": "main_query", "query": { "datasetName": "6a0f2ffc", "fields": [ { "name": "sum(count)", "expression": "SUM(`count`)" } ], "filters": [ { "expression": "`status` IN ('Healthy')" } ], "disaggregated": false } } ], "spec": { "version": 2, "widgetType": "counter", "encodings": { "value": { "fieldName": "sum(count)", "style": { "color": "#00A972" }, "displayName": "Sum of count" } }, "frame": { "showTitle": true, "title": "# Healthy tables" } } }, "position": { "x": 1, "y": 3, "width": 1, "height": 3 } }, { "widget": { "name": "54be92e0", "queries": [ { "name": "main_query", "query": { "datasetName": "6a0f2ffc", "fields": [ { "name": "sum(count)", "expression": "SUM(`count`)" } ], "filters": [ { "expression": "`status` IN ('Unhealthy')" } ], "disaggregated": false } } ], "spec": { "version": 2, "widgetType": "counter", "encodings": { "value": { "fieldName": "sum(count)", "format": { "type": "number-plain", "abbreviation": "compact", "decimalPlaces": { "type": "max", "places": 2 } }, "style": { "color": "#FF3621" }, "displayName": "Sum of count" } }, "frame": { "showTitle": true, "title": "# Unhealthy tables" } } }, "position": { "x": 2, "y": 3, "width": 1, "height": 3 } }, { "widget": { "name": "1b7f5bde", "queries": [ { "name": "main_query", "query": { "datasetName": "6a0f2ffc", "fields": [ { "name": "percentage", "expression": "`percentage`" } ], "filters": [ { "expression": "`status` IN ('Healthy')" } ], "disaggregated": true } } ], "spec": { "version": 2, "widgetType": "counter", "encodings": { "value": { "fieldName": "percentage", "style": { "color": "#00A972" }, "format": { "type": "number-percent", "decimalPlaces": { "type": "max", "places": 2 } }, "displayName": "percentage" } }, "frame": { "title": "% Healthy tables", "showTitle": true, "showDescription": false, "description": " A table is healthy if is both fresh and complete" } } }, "position": { "x": 0, "y": 3, "width": 1, "height": 3 } }, { "widget": { "name": "2d3b45af", "multilineTextboxSpec": { "lines": [ "## Quality incidents by impact\n", "\n", "Investigate unhealthy tables below. Impact is based on downstream lineage and query volume." ] } }, "position": { "x": 0, "y": 6, "width": 6, "height": 2 } }, { "widget": { "name": "f425ede9", "queries": [ { "name": "dashboards/01f0faed3627103abcbe4fffeb94cdb4/datasets/01f0faed3627179f873430d76ec7d757_impact_level", "query": { "datasetName": "0edc79ec", "fields": [ { "name": "impact_level", "expression": "`impact_level`" }, { "name": "impact_level_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)" } ], "disaggregated": false } } ], "spec": { "version": 2, "widgetType": "filter-single-select", "encodings": { "fields": [ { "fieldName": "impact_level", "queryName": "dashboards/01f0faed3627103abcbe4fffeb94cdb4/datasets/01f0faed3627179f873430d76ec7d757_impact_level" } ] }, "scale": { "type": "categorical", "sort": { "by": "custom-order", "orderedValues": ["All", "High", "Medium", "Low", "Unknown", "n/a (healthy)"] } }, "frame": { "showTitle": true, "title": "impact" } } }, "position": { "x": 3, "y": 8, "width": 3, "height": 1 } }, { "widget": { "name": "5681fb7f", "queries": [ { "name": "dashboards/01f0faed3627103abcbe4fffeb94cdb4/datasets/01f0faed3627179f873430d76ec7d757_status", "query": { "datasetName": "0edc79ec", "fields": [ { "name": "status", "expression": "`status`" }, { "name": "status_associativity", "expression": "COUNT_IF(`associative_filter_predicate_group`)" } ], "disaggregated": false } } ], "spec": { "version": 2, "widgetType": "filter-single-select", "encodings": { "fields": [ { "fieldName": "status", "queryName": "dashboards/01f0faed3627103abcbe4fffeb94cdb4/datasets/01f0faed3627179f873430d76ec7d757_status" } ] }, "disallowAll": false, "frame": { "showTitle": true, "title": "status" } } }, "position": { "x": 0, "y": 8, "width": 3, "height": 1 } } ], "pageType": "PAGE_TYPE_CANVAS" } ], "uiSettings": { "theme": { "widgetHeaderAlignment": "ALIGNMENT_UNSPECIFIED" }, "applyModeEnabled": false } }