From a8a90c12ec3834f7f49a726c8e9fa107972ce3f1 Mon Sep 17 00:00:00 2001 From: Ashwani Srivastav Date: Mon, 17 Aug 2026 07:09:02 +0000 Subject: [PATCH 1/7] updated ref_area_code --- .../golden_data/golden_observations.csv | 47 +++++++++++++++++++ .../golden_data/golden_summary_report.csv | 3 ++ .../oecd/quarterly_gdp/manifest.json | 3 +- statvar_imports/oecd/quarterly_gdp/pvmap.csv | 2 + .../oecd/quarterly_gdp/validation_config.json | 28 +++++++++++ 5 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 statvar_imports/oecd/quarterly_gdp/golden_data/golden_observations.csv create mode 100644 statvar_imports/oecd/quarterly_gdp/golden_data/golden_summary_report.csv create mode 100644 statvar_imports/oecd/quarterly_gdp/validation_config.json diff --git a/statvar_imports/oecd/quarterly_gdp/golden_data/golden_observations.csv b/statvar_imports/oecd/quarterly_gdp/golden_data/golden_observations.csv new file mode 100644 index 0000000000..34672df2ef --- /dev/null +++ b/statvar_imports/oecd/quarterly_gdp/golden_data/golden_observations.csv @@ -0,0 +1,47 @@ +"observationAbout" +"country/SAU" +"country/RUS" +"country/SVK" +"country/ITA" +"country/ISL" +"country/ESP" +"country/PRT" +"country/BEL" +"country/TUR" +"country/LTU" +"country/FIN" +"country/CHN" +"country/NOR" +"country/CAN" +"country/GBR" +"country/USA" +"country/IRL" +"country/DEU" +"country/LUX" +"country/KOR" +"country/HUN" +"country/CZE" +"country/NLD" +"country/GRC" +"country/MEX" +"country/DNK" +"country/SVN" +"country/LVA" +"country/NZL" +"country/CHL" +"country/IDN" +"country/COL" +"country/AUS" +"country/IND" +"country/ARG" +"country/BRA" +"country/ISR" +"country/POL" +"country/EST" +"country/AUT" +"country/CHE" +"country/JPN" +"country/ZAF" +"country/SWE" +"country/CRI" +"country/FRA" diff --git a/statvar_imports/oecd/quarterly_gdp/golden_data/golden_summary_report.csv b/statvar_imports/oecd/quarterly_gdp/golden_data/golden_summary_report.csv new file mode 100644 index 0000000000..c6d5a46d9a --- /dev/null +++ b/statvar_imports/oecd/quarterly_gdp/golden_data/golden_summary_report.csv @@ -0,0 +1,3 @@ +"Units","ScalingFactors","observationPeriods","MinDate","MeasurementMethods","NumPlaces","StatVar" +"[Percent]","[]","[P3M]","2020-03","[ChainLinkedVolume]","46","Adjusted_GrowthRate_GrossDomesticProduct_EconomicActivity_QuarterOnChange" +"[Percent]","[]","[P3M]","2020-03","[ChainLinkedVolume]","46","Adjusted_GrowthRate_GrossDomesticProduct_EconomicActivity_YearOnChange" diff --git a/statvar_imports/oecd/quarterly_gdp/manifest.json b/statvar_imports/oecd/quarterly_gdp/manifest.json index 08512a3ef5..3a8c932076 100644 --- a/statvar_imports/oecd/quarterly_gdp/manifest.json +++ b/statvar_imports/oecd/quarterly_gdp/manifest.json @@ -23,7 +23,8 @@ "cleaned_csv": "output/oecd_quarterly_gdp.csv" } ], - "cron_schedule": "0 8 1 */3 *" + "cron_schedule": "0 8 1 */3 *", + "validation_config_file": "validation_config.json" } ] } diff --git a/statvar_imports/oecd/quarterly_gdp/pvmap.csv b/statvar_imports/oecd/quarterly_gdp/pvmap.csv index d8032f170d..3038da4d3a 100644 --- a/statvar_imports/oecd/quarterly_gdp/pvmap.csv +++ b/statvar_imports/oecd/quarterly_gdp/pvmap.csv @@ -4,6 +4,8 @@ key,property,value,property2,value2,property3,value3 # Dimensions,,,,,, FREQ:Q,observationPeriod,P3M,,,, REF_AREA,observationAbout,country/{Data},,,, +REF_AREA:EU,#ignore,reason: not a country,,,, +REF_AREA:EA,#ignore,reason: not a country,,,, REF_AREA:EU27_2020,#ignore,reason: not a country,,,, REF_AREA:EA20,#ignore,reason: not a country,,,, REF_AREA:OECD,#ignore,reason: not a country,,,, diff --git a/statvar_imports/oecd/quarterly_gdp/validation_config.json b/statvar_imports/oecd/quarterly_gdp/validation_config.json new file mode 100644 index 0000000000..95a2ef72d3 --- /dev/null +++ b/statvar_imports/oecd/quarterly_gdp/validation_config.json @@ -0,0 +1,28 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted points is within the threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 0.1 + } + }, + { + "rule_id": "check_goldens_summary_report", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_summary_report.csv" } + }, + { + "rule_id": "Check_goldens_output_csv", + "description": "Verifies the generated output CSV data matches established critical golden records", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_observations.csv", + "input_files": "../../../../output/oecd_quarterly_gdp.csv" + } + } + ] +} From a084a2ab4b465ceb54c4b42ae6490260b5145f42 Mon Sep 17 00:00:00 2001 From: Ashwani725 Date: Mon, 17 Aug 2026 13:40:11 +0530 Subject: [PATCH 2/7] Fix rule_id casing in validation_config.json --- statvar_imports/oecd/quarterly_gdp/validation_config.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/statvar_imports/oecd/quarterly_gdp/validation_config.json b/statvar_imports/oecd/quarterly_gdp/validation_config.json index 95a2ef72d3..47eaf0417c 100644 --- a/statvar_imports/oecd/quarterly_gdp/validation_config.json +++ b/statvar_imports/oecd/quarterly_gdp/validation_config.json @@ -13,10 +13,11 @@ "rule_id": "check_goldens_summary_report", "validator": "GOLDENS_CHECK", "params": { - "golden_files": "../../../../golden_data/golden_summary_report.csv" } + "golden_files": "../../../../golden_data/golden_summary_report.csv" + } }, { - "rule_id": "Check_goldens_output_csv", + "rule_id": "check_goldens_output_csv", "description": "Verifies the generated output CSV data matches established critical golden records", "validator": "GOLDENS_CHECK", "params": { From 2e9ae6265f216955ca3f24a7610330dfc3fa55b3 Mon Sep 17 00:00:00 2001 From: Ashwani725 Date: Mon, 31 Aug 2026 17:06:20 +0530 Subject: [PATCH 3/7] goldens deleted as no longer required --- .../golden_data/golden_observations.csv | 47 ------------------- 1 file changed, 47 deletions(-) delete mode 100644 statvar_imports/oecd/quarterly_gdp/golden_data/golden_observations.csv diff --git a/statvar_imports/oecd/quarterly_gdp/golden_data/golden_observations.csv b/statvar_imports/oecd/quarterly_gdp/golden_data/golden_observations.csv deleted file mode 100644 index 34672df2ef..0000000000 --- a/statvar_imports/oecd/quarterly_gdp/golden_data/golden_observations.csv +++ /dev/null @@ -1,47 +0,0 @@ -"observationAbout" -"country/SAU" -"country/RUS" -"country/SVK" -"country/ITA" -"country/ISL" -"country/ESP" -"country/PRT" -"country/BEL" -"country/TUR" -"country/LTU" -"country/FIN" -"country/CHN" -"country/NOR" -"country/CAN" -"country/GBR" -"country/USA" -"country/IRL" -"country/DEU" -"country/LUX" -"country/KOR" -"country/HUN" -"country/CZE" -"country/NLD" -"country/GRC" -"country/MEX" -"country/DNK" -"country/SVN" -"country/LVA" -"country/NZL" -"country/CHL" -"country/IDN" -"country/COL" -"country/AUS" -"country/IND" -"country/ARG" -"country/BRA" -"country/ISR" -"country/POL" -"country/EST" -"country/AUT" -"country/CHE" -"country/JPN" -"country/ZAF" -"country/SWE" -"country/CRI" -"country/FRA" From a7eb602de69d3442e201fbfee7401baa28b2de22 Mon Sep 17 00:00:00 2001 From: Ashwani725 Date: Mon, 31 Aug 2026 17:06:42 +0530 Subject: [PATCH 4/7] goldens deleted as no longer required --- .../oecd/quarterly_gdp/golden_data/golden_summary_report.csv | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 statvar_imports/oecd/quarterly_gdp/golden_data/golden_summary_report.csv diff --git a/statvar_imports/oecd/quarterly_gdp/golden_data/golden_summary_report.csv b/statvar_imports/oecd/quarterly_gdp/golden_data/golden_summary_report.csv deleted file mode 100644 index c6d5a46d9a..0000000000 --- a/statvar_imports/oecd/quarterly_gdp/golden_data/golden_summary_report.csv +++ /dev/null @@ -1,3 +0,0 @@ -"Units","ScalingFactors","observationPeriods","MinDate","MeasurementMethods","NumPlaces","StatVar" -"[Percent]","[]","[P3M]","2020-03","[ChainLinkedVolume]","46","Adjusted_GrowthRate_GrossDomesticProduct_EconomicActivity_QuarterOnChange" -"[Percent]","[]","[P3M]","2020-03","[ChainLinkedVolume]","46","Adjusted_GrowthRate_GrossDomesticProduct_EconomicActivity_YearOnChange" From 329863895ffc364b8740536ac0301a95e9faa873 Mon Sep 17 00:00:00 2001 From: Ashwani725 Date: Tue, 1 Sep 2026 15:20:16 +0530 Subject: [PATCH 5/7] Remove golden record validation rules from config Removed validation rules for golden summary and output CSV checks. --- .../oecd/quarterly_gdp/validation_config.json | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/statvar_imports/oecd/quarterly_gdp/validation_config.json b/statvar_imports/oecd/quarterly_gdp/validation_config.json index 47eaf0417c..66014b3369 100644 --- a/statvar_imports/oecd/quarterly_gdp/validation_config.json +++ b/statvar_imports/oecd/quarterly_gdp/validation_config.json @@ -8,22 +8,6 @@ "params": { "threshold": 0.1 } - }, - { - "rule_id": "check_goldens_summary_report", - "validator": "GOLDENS_CHECK", - "params": { - "golden_files": "../../../../golden_data/golden_summary_report.csv" - } - }, - { - "rule_id": "check_goldens_output_csv", - "description": "Verifies the generated output CSV data matches established critical golden records", - "validator": "GOLDENS_CHECK", - "params": { - "golden_files": "../../../../golden_data/golden_observations.csv", - "input_files": "../../../../output/oecd_quarterly_gdp.csv" - } } ] } From 9fdeb367c25f158183400f168507847f369551a3 Mon Sep 17 00:00:00 2001 From: Ashwani Srivastav Date: Tue, 1 Sep 2026 11:13:15 +0000 Subject: [PATCH 6/7] appendinf cra suggestion --- statvar_imports/oecd/quarterly_gdp/manifest.json | 3 ++- statvar_imports/oecd/quarterly_gdp/process.sh | 1 + .../oecd/quarterly_gdp/validation_config.json | 12 ++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/statvar_imports/oecd/quarterly_gdp/manifest.json b/statvar_imports/oecd/quarterly_gdp/manifest.json index 3a8c932076..7b507797ee 100644 --- a/statvar_imports/oecd/quarterly_gdp/manifest.json +++ b/statvar_imports/oecd/quarterly_gdp/manifest.json @@ -14,7 +14,8 @@ "process.sh" ], "source_files": [ - "input/*" + "input/*", + "counters/*" ], "import_inputs": [ { diff --git a/statvar_imports/oecd/quarterly_gdp/process.sh b/statvar_imports/oecd/quarterly_gdp/process.sh index 892f59c18d..ed150bf724 100755 --- a/statvar_imports/oecd/quarterly_gdp/process.sh +++ b/statvar_imports/oecd/quarterly_gdp/process.sh @@ -28,5 +28,6 @@ python3 "$PROJECT_ROOT/tools/statvar_importer/stat_var_processor.py" \ --input_data="$SCRIPT_DIR/input/oecd_gdp_data.csv" \ --pv_map="$SCRIPT_DIR/pvmap.csv" \ --config_file="$SCRIPT_DIR/metadata.csv" \ + --existing_statvar_mcf="gs://unresolved_mcf/scripts/statvar/stat_vars.mcf" \ --output_path="$SCRIPT_DIR/output/oecd_quarterly_gdp" \ --output_counters="$SCRIPT_DIR/oecd_quarterly_gdp/validation/statvar_counters.csv" diff --git a/statvar_imports/oecd/quarterly_gdp/validation_config.json b/statvar_imports/oecd/quarterly_gdp/validation_config.json index 66014b3369..88e63dec4f 100644 --- a/statvar_imports/oecd/quarterly_gdp/validation_config.json +++ b/statvar_imports/oecd/quarterly_gdp/validation_config.json @@ -8,6 +8,18 @@ "params": { "threshold": 0.1 } + }, + { + "rule_id": "check_max_date_consistent", + "description": "Checks that the max date is consistent across all statvars.", + "validator": "MAX_DATE_CONSISTENT", + "params": {} + }, + { + "rule_id": "check_max_date_latest", + "description": "Checks that data is fresh and up to date.", + "validator": "MAX_DATE_LATEST", + "params": {} } ] } From 095205e76d81508a7cef744272476b661075bdff Mon Sep 17 00:00:00 2001 From: Ashwani725 Date: Tue, 1 Sep 2026 17:04:00 +0530 Subject: [PATCH 7/7] Update source_files in manifest.json --- statvar_imports/oecd/quarterly_gdp/manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statvar_imports/oecd/quarterly_gdp/manifest.json b/statvar_imports/oecd/quarterly_gdp/manifest.json index 7b507797ee..4dda88b9de 100644 --- a/statvar_imports/oecd/quarterly_gdp/manifest.json +++ b/statvar_imports/oecd/quarterly_gdp/manifest.json @@ -15,7 +15,7 @@ ], "source_files": [ "input/*", - "counters/*" + "oecd_quarterly_gdp/validation/statvar_counters.csv" ], "import_inputs": [ {