From aee0e300711eef9450d67c147fc09109093ba2b5 Mon Sep 17 00:00:00 2001 From: shvngisingh Date: Fri, 7 Aug 2026 06:36:04 +0000 Subject: [PATCH 1/5] Remediate CDC Mortality Underlying Cause Single Race import pipeline --- statvar_imports/us_cdc/single_race/README.md | 4 ++-- statvar_imports/us_cdc/single_race/manifest.json | 11 +++++++---- .../us_cdc/single_race/single_race_metadata.csv | 6 ++++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/statvar_imports/us_cdc/single_race/README.md b/statvar_imports/us_cdc/single_race/README.md index 03574c73cb..65e8dd3cac 100644 --- a/statvar_imports/us_cdc/single_race/README.md +++ b/statvar_imports/us_cdc/single_race/README.md @@ -22,7 +22,7 @@ To obtain the raw input files, data must be manually downloaded from the source. *Single Race (6 categories) *ICD-10-113 Cause List -For each download, a specific state must be selected. The desired year range is from 2018 to 2023. After making the selections, click the "Send" button at the bottom to initiate the download. +For each download, a specific state must be selected. After making the selections, click the "Send" button at the bottom to initiate the download. ### Data Processing @@ -38,7 +38,7 @@ After the files are downloaded, the data is processed using the stat_var_process ```bash - python3 ../../../tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=input_files/*.csv --pv_map=single_race_pvmap.csv --config_file=single_race_metadata.csv --output_path=output/underlyingcauseofdeath2018_2023singlerace + python3 ../../../tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=input_files/*.csv --pv_map=single_race_pvmap.csv --config_file=single_race_metadata.csv --output_path=output/underlyingcauseofdeath_singlerace ``` ### Automation diff --git a/statvar_imports/us_cdc/single_race/manifest.json b/statvar_imports/us_cdc/single_race/manifest.json index 9808a6c1c5..1fc2256149 100644 --- a/statvar_imports/us_cdc/single_race/manifest.json +++ b/statvar_imports/us_cdc/single_race/manifest.json @@ -9,16 +9,19 @@ "provenance_description": "Mortality statistics, categorized by demographic factors and specific causes of death, location, race at state level.", "scripts": [ "download.sh", - "../../../tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=input_files/*.csv --pv_map=single_race_pvmap.csv --config_file=single_race_metadata.csv --output_path=output/underlyingcauseofdeath2018_2023singlerace" + "../../../tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=input_files/*.csv --pv_map=single_race_pvmap.csv --config_file=single_race_metadata.csv --output_path=output/underlyingcauseofdeath_singlerace --output_counters=counters/underlyingcauseofdeath_singlerace.csv" ], "import_inputs": [ { - "template_mcf": "output/underlyingcauseofdeath2018_2023singlerace.tmcf", - "cleaned_csv": "output/underlyingcauseofdeath2018_2023singlerace.csv" + "template_mcf": "output/underlyingcauseofdeath_singlerace.tmcf", + "node_mcf": "output/underlyingcauseofdeath_singlerace_stat_vars.mcf", + "cleaned_csv": "output/underlyingcauseofdeath_singlerace.csv" } ], + "validation_config_file": "validation_config.json", "source_files": [ - "input_files/*.csv" + "input_files/*.csv", + "counters/*.csv" ], "cron_schedule": "30 08 8-14 * 6" } diff --git a/statvar_imports/us_cdc/single_race/single_race_metadata.csv b/statvar_imports/us_cdc/single_race/single_race_metadata.csv index bf9c492068..54b5caf044 100644 --- a/statvar_imports/us_cdc/single_race/single_race_metadata.csv +++ b/statvar_imports/us_cdc/single_race/single_race_metadata.csv @@ -3,8 +3,8 @@ url,https://wonder.cdc.gov/controller/datarequest/D158 description,"Mortality statistics, categorized by demographic factors and specific causes of death, location, race at state level" #place_type,state level #places_within, -start_date,2018 -end_date,2023 +#start_date,2018 +#end_date,2024 release_frequency,P1Y process, comments, @@ -13,3 +13,5 @@ mapped_rows,1 mapped_columns,11 header_rows,1 debug,0 +output_only_new_statvars,False + From 173c1d40b5b438ff8af9b4086c77e950c152c19b Mon Sep 17 00:00:00 2001 From: shvngisingh Date: Fri, 7 Aug 2026 12:20:55 +0530 Subject: [PATCH 2/5] Update README.md --- statvar_imports/us_cdc/single_race/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statvar_imports/us_cdc/single_race/README.md b/statvar_imports/us_cdc/single_race/README.md index 65e8dd3cac..3bae57a8e7 100644 --- a/statvar_imports/us_cdc/single_race/README.md +++ b/statvar_imports/us_cdc/single_race/README.md @@ -38,7 +38,7 @@ After the files are downloaded, the data is processed using the stat_var_process ```bash - python3 ../../../tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=input_files/*.csv --pv_map=single_race_pvmap.csv --config_file=single_race_metadata.csv --output_path=output/underlyingcauseofdeath_singlerace + python3 ../../../tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=input_files/*.csv --pv_map=single_race_pvmap.csv --config_file=single_race_metadata.csv --output_path=output/underlyingcauseofdeath_singlerace --output_counters=counters/underlyingcauseofdeath_singlerace.csv ``` ### Automation From 95fea4dd55d9af496d3b400d42827222ca01416a Mon Sep 17 00:00:00 2001 From: shvngisingh Date: Wed, 19 Aug 2026 05:42:40 +0000 Subject: [PATCH 3/5] Remove node_mcf from manifest.json --- statvar_imports/us_cdc/single_race/manifest.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/statvar_imports/us_cdc/single_race/manifest.json b/statvar_imports/us_cdc/single_race/manifest.json index 1fc2256149..ed162ffa53 100644 --- a/statvar_imports/us_cdc/single_race/manifest.json +++ b/statvar_imports/us_cdc/single_race/manifest.json @@ -14,11 +14,9 @@ "import_inputs": [ { "template_mcf": "output/underlyingcauseofdeath_singlerace.tmcf", - "node_mcf": "output/underlyingcauseofdeath_singlerace_stat_vars.mcf", "cleaned_csv": "output/underlyingcauseofdeath_singlerace.csv" } ], - "validation_config_file": "validation_config.json", "source_files": [ "input_files/*.csv", "counters/*.csv" From 678d7fc1ace01e5e9fdded500e64e355785a0e30 Mon Sep 17 00:00:00 2001 From: shvngisingh Date: Thu, 27 Aug 2026 06:32:15 +0000 Subject: [PATCH 4/5] Restore node_mcf in manifest and remediate metadata and README --- statvar_imports/us_cdc/single_race/README.md | 4 ++-- statvar_imports/us_cdc/single_race/manifest.json | 3 ++- statvar_imports/us_cdc/single_race/single_race_metadata.csv | 5 ++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/statvar_imports/us_cdc/single_race/README.md b/statvar_imports/us_cdc/single_race/README.md index 3bae57a8e7..e40e27124b 100644 --- a/statvar_imports/us_cdc/single_race/README.md +++ b/statvar_imports/us_cdc/single_race/README.md @@ -1,12 +1,12 @@ ### This import process handles data from wonder.cdc platform. -- Description: Mortality statistics, categorized by demographic factors and specific causes of death, location, race at state level. +- Description: Mortality statistics, categorized by demographic factors and specific causes of death, location, race at county level. - Source URL: https://wonder.cdc.gov/ucd-icd10-expanded.html - Import Type: Semi-Automated -- Data Availability: 2018 to 2023 +- Data Availability: 2018 onwards - Release Frequency: P1Y, which means every Year. diff --git a/statvar_imports/us_cdc/single_race/manifest.json b/statvar_imports/us_cdc/single_race/manifest.json index ed162ffa53..24e7bc74b8 100644 --- a/statvar_imports/us_cdc/single_race/manifest.json +++ b/statvar_imports/us_cdc/single_race/manifest.json @@ -6,7 +6,7 @@ "support@datacommons.org" ], "provenance_url": "https://wonder.cdc.gov/ucd-icd10-expanded.html", - "provenance_description": "Mortality statistics, categorized by demographic factors and specific causes of death, location, race at state level.", + "provenance_description": "Mortality statistics, categorized by demographic factors and specific causes of death, location, race at county level.", "scripts": [ "download.sh", "../../../tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=input_files/*.csv --pv_map=single_race_pvmap.csv --config_file=single_race_metadata.csv --output_path=output/underlyingcauseofdeath_singlerace --output_counters=counters/underlyingcauseofdeath_singlerace.csv" @@ -14,6 +14,7 @@ "import_inputs": [ { "template_mcf": "output/underlyingcauseofdeath_singlerace.tmcf", + "node_mcf": "output/underlyingcauseofdeath_singlerace_stat_vars.mcf", "cleaned_csv": "output/underlyingcauseofdeath_singlerace.csv" } ], diff --git a/statvar_imports/us_cdc/single_race/single_race_metadata.csv b/statvar_imports/us_cdc/single_race/single_race_metadata.csv index 54b5caf044..5ef942a0d7 100644 --- a/statvar_imports/us_cdc/single_race/single_race_metadata.csv +++ b/statvar_imports/us_cdc/single_race/single_race_metadata.csv @@ -1,6 +1,6 @@ parameter,value url,https://wonder.cdc.gov/controller/datarequest/D158 -description,"Mortality statistics, categorized by demographic factors and specific causes of death, location, race at state level" +description,"Mortality statistics, categorized by demographic factors and specific causes of death, location, race at county level" #place_type,state level #places_within, #start_date,2018 @@ -8,10 +8,9 @@ description,"Mortality statistics, categorized by demographic factors and specif release_frequency,P1Y process, comments, -output_columns,"observationAbout,observationDate,variableMeasured,observationPeriod,value,unit,measurementMethod" +output_columns,"observationAbout,observationDate,variableMeasured,observationPeriod,value" mapped_rows,1 mapped_columns,11 header_rows,1 debug,0 output_only_new_statvars,False - From 15f8de3894e392ecf220dbfb5aaebaedf0582c91 Mon Sep 17 00:00:00 2001 From: shvngisingh Date: Tue, 1 Sep 2026 12:28:16 +0530 Subject: [PATCH 5/5] Update manifest.json --- statvar_imports/us_cdc/single_race/manifest.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/statvar_imports/us_cdc/single_race/manifest.json b/statvar_imports/us_cdc/single_race/manifest.json index 24e7bc74b8..19582fdaba 100644 --- a/statvar_imports/us_cdc/single_race/manifest.json +++ b/statvar_imports/us_cdc/single_race/manifest.json @@ -14,7 +14,7 @@ "import_inputs": [ { "template_mcf": "output/underlyingcauseofdeath_singlerace.tmcf", - "node_mcf": "output/underlyingcauseofdeath_singlerace_stat_vars.mcf", + "node_mcf": "output/underlyingcauseofdeath_singlerace*.mcf", "cleaned_csv": "output/underlyingcauseofdeath_singlerace.csv" } ], @@ -25,4 +25,4 @@ "cron_schedule": "30 08 8-14 * 6" } ] -} \ No newline at end of file +}