diff --git a/statvar_imports/oecd/quarterly_gdp/manifest.json b/statvar_imports/oecd/quarterly_gdp/manifest.json index 08512a3ef5..4dda88b9de 100644 --- a/statvar_imports/oecd/quarterly_gdp/manifest.json +++ b/statvar_imports/oecd/quarterly_gdp/manifest.json @@ -14,7 +14,8 @@ "process.sh" ], "source_files": [ - "input/*" + "input/*", + "oecd_quarterly_gdp/validation/statvar_counters.csv" ], "import_inputs": [ { @@ -23,7 +24,8 @@ "cleaned_csv": "output/oecd_quarterly_gdp.csv" } ], - "cron_schedule": "0 8 1 */3 *" + "cron_schedule": "0 8 1 */3 *", + "validation_config_file": "validation_config.json" } ] } diff --git a/statvar_imports/oecd/quarterly_gdp/process.sh b/statvar_imports/oecd/quarterly_gdp/process.sh index 892f59c18d..ed150bf724 100755 --- a/statvar_imports/oecd/quarterly_gdp/process.sh +++ b/statvar_imports/oecd/quarterly_gdp/process.sh @@ -28,5 +28,6 @@ python3 "$PROJECT_ROOT/tools/statvar_importer/stat_var_processor.py" \ --input_data="$SCRIPT_DIR/input/oecd_gdp_data.csv" \ --pv_map="$SCRIPT_DIR/pvmap.csv" \ --config_file="$SCRIPT_DIR/metadata.csv" \ + --existing_statvar_mcf="gs://unresolved_mcf/scripts/statvar/stat_vars.mcf" \ --output_path="$SCRIPT_DIR/output/oecd_quarterly_gdp" \ --output_counters="$SCRIPT_DIR/oecd_quarterly_gdp/validation/statvar_counters.csv" diff --git a/statvar_imports/oecd/quarterly_gdp/pvmap.csv b/statvar_imports/oecd/quarterly_gdp/pvmap.csv index d8032f170d..3038da4d3a 100644 --- a/statvar_imports/oecd/quarterly_gdp/pvmap.csv +++ b/statvar_imports/oecd/quarterly_gdp/pvmap.csv @@ -4,6 +4,8 @@ key,property,value,property2,value2,property3,value3 # Dimensions,,,,,, FREQ:Q,observationPeriod,P3M,,,, REF_AREA,observationAbout,country/{Data},,,, +REF_AREA:EU,#ignore,reason: not a country,,,, +REF_AREA:EA,#ignore,reason: not a country,,,, REF_AREA:EU27_2020,#ignore,reason: not a country,,,, REF_AREA:EA20,#ignore,reason: not a country,,,, REF_AREA:OECD,#ignore,reason: not a country,,,, diff --git a/statvar_imports/oecd/quarterly_gdp/validation_config.json b/statvar_imports/oecd/quarterly_gdp/validation_config.json new file mode 100644 index 0000000000..88e63dec4f --- /dev/null +++ b/statvar_imports/oecd/quarterly_gdp/validation_config.json @@ -0,0 +1,25 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted points is within the threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 0.1 + } + }, + { + "rule_id": "check_max_date_consistent", + "description": "Checks that the max date is consistent across all statvars.", + "validator": "MAX_DATE_CONSISTENT", + "params": {} + }, + { + "rule_id": "check_max_date_latest", + "description": "Checks that data is fresh and up to date.", + "validator": "MAX_DATE_LATEST", + "params": {} + } + ] +}