Remediate CDC_Mortality_UnderlyingCause_SingleRace import

statvar_imports/us_cdc/single_race/README.md

@@ -1,12 +1,12 @@
 ### This import process handles data from wonder.cdc platform.
 
-- Description: Mortality statistics, categorized by demographic factors and specific causes of death, location, race at state level.
+- Description: Mortality statistics, categorized by demographic factors and specific causes of death, location, race at county level.
 
 - Source URL: https://wonder.cdc.gov/ucd-icd10-expanded.html
 
 - Import Type: Semi-Automated
 
-- Data Availability: 2018 to 2023
+- Data Availability: 2018 onwards
 
 - Release Frequency: P1Y, which means every Year.
 
@@ -22,7 +22,7 @@ To obtain the raw input files, data must be manually downloaded from the source.
 	*Single Race (6 categories)
 	*ICD-10-113 Cause List
 
-For each download, a specific state must be selected. The desired year range is from 2018 to 2023. After making the selections, click the "Send" button at the bottom to initiate the download.
+For each download, a specific state must be selected. After making the selections, click the "Send" button at the bottom to initiate the download.
 
 
 ### Data Processing
@@ -38,7 +38,7 @@ After the files are downloaded, the data is processed using the stat_var_process
 
 ```bash
 
-	python3 ../../../tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=input_files/*.csv --pv_map=single_race_pvmap.csv --config_file=single_race_metadata.csv --output_path=output/underlyingcauseofdeath2018_2023singlerace
+	python3 ../../../tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=input_files/*.csv --pv_map=single_race_pvmap.csv --config_file=single_race_metadata.csv --output_path=output/underlyingcauseofdeath_singlerace --output_counters=counters/underlyingcauseofdeath_singlerace.csv
 ```
 
 ### Automation

statvar_imports/us_cdc/single_race/manifest.json

@@ -6,21 +6,23 @@
                 "support@datacommons.org"
             ],
             "provenance_url": "https://wonder.cdc.gov/ucd-icd10-expanded.html",
-            "provenance_description": "Mortality statistics, categorized by demographic factors and specific causes of death, location, race at state level.",
+            "provenance_description": "Mortality statistics, categorized by demographic factors and specific causes of death, location, race at county level.",
             "scripts": [
                 "download.sh",
-                "../../../tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=input_files/*.csv --pv_map=single_race_pvmap.csv --config_file=single_race_metadata.csv --output_path=output/underlyingcauseofdeath2018_2023singlerace"
+                "../../../tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=input_files/*.csv --pv_map=single_race_pvmap.csv --config_file=single_race_metadata.csv --output_path=output/underlyingcauseofdeath_singlerace --output_counters=counters/underlyingcauseofdeath_singlerace.csv"
             ],
             "import_inputs": [
                 {
-                    "template_mcf": "output/underlyingcauseofdeath2018_2023singlerace.tmcf",
-                    "cleaned_csv": "output/underlyingcauseofdeath2018_2023singlerace.csv"
+                    "template_mcf": "output/underlyingcauseofdeath_singlerace.tmcf",
+                    "node_mcf": "output/underlyingcauseofdeath_singlerace*.mcf",
+                    "cleaned_csv": "output/underlyingcauseofdeath_singlerace.csv"
                 }
             ],
             "source_files": [
-                "input_files/*.csv"
+                "input_files/*.csv",
+                "counters/*.csv"
             ],
             "cron_schedule": "30 08 8-14 * 6"
         }
     ]
-}
+}

statvar_imports/us_cdc/single_race/single_race_metadata.csv

@@ -1,15 +1,16 @@
 parameter,value
 url,https://wonder.cdc.gov/controller/datarequest/D158
-description,"Mortality statistics, categorized by demographic factors and specific causes of death, location, race at state level"
+description,"Mortality statistics, categorized by demographic factors and specific causes of death, location, race at county level"
 #place_type,state level
 #places_within,
-start_date,2018
-end_date,2023
+#start_date,2018
+#end_date,2024
 release_frequency,P1Y
 process,
 comments,
-output_columns,"observationAbout,observationDate,variableMeasured,observationPeriod,value,unit,measurementMethod"
+output_columns,"observationAbout,observationDate,variableMeasured,observationPeriod,value"
 mapped_rows,1
 mapped_columns,11
 header_rows,1
 debug,0
+output_only_new_statvars,False