From c641f9fb405298ad06bcb71f3d7b0e9cc82b0600 Mon Sep 17 00:00:00 2001 From: Ajai Tirumali Date: Tue, 30 Dec 2025 19:53:11 +0530 Subject: [PATCH 1/6] Support 0 eval results --- tools/statvar_importer/property_value_mapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/statvar_importer/property_value_mapper.py b/tools/statvar_importer/property_value_mapper.py index 76d5209216..8c60f481a3 100644 --- a/tools/statvar_importer/property_value_mapper.py +++ b/tools/statvar_importer/property_value_mapper.py @@ -347,7 +347,7 @@ def _process_eval(self, pvs: dict, data_key: str) -> bool: self._log_every_n) if not eval_prop: eval_prop = data_key - if eval_data and eval_data != eval_str: + if eval_data is not None and eval_data != eval_str: pvs[eval_prop] = eval_data self._counters.add_counter('processed-eval', 1, eval_str) pvs.pop(eval_key) From f30e89550743d537d0aed8f59e3031cd2452cf19 Mon Sep 17 00:00:00 2001 From: Ajai Tirumali Date: Wed, 25 Mar 2026 16:32:16 +0530 Subject: [PATCH 2/6] Use environment variable for DC API root --- util/dc_api_wrapper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/dc_api_wrapper.py b/util/dc_api_wrapper.py index f1a1fee247..98f7209d0e 100644 --- a/util/dc_api_wrapper.py +++ b/util/dc_api_wrapper.py @@ -15,7 +15,7 @@ It uses the DataCommonsClient library module for DC APIs and adds support for batched requests, retries and HTTP caching. -DC API requires an environment variable set for DC_API_KEY. +DC API requires an environment variable set for DC_API_KEY and DC_API_ROOT. Please refer to https://docs.datacommons.org/api/python/v2 for more details. """ @@ -265,7 +265,7 @@ def get_datacommons_client(config: dict = None) -> DataCommonsClient: """Returns a DataCommonsClient object initialized using config.""" config = _validate_v2_config(config) api_key = get_dc_api_key(config) - dc_instance = config.get('dc_api_root') + dc_instance = config.get('dc_api_root', os.environ.get('DC_API_ROOT')) url = None # Check if API root is a host or url endpoint. if dc_instance: From 2a0e93d8f73aeef6ff2b9b48bf9b5b819ed99182 Mon Sep 17 00:00:00 2001 From: Ajai Tirumali Date: Wed, 25 Mar 2026 22:17:08 +0530 Subject: [PATCH 3/6] cleanup dc_api_root configs --- scripts/earthengine/utils.py | 2 -- scripts/earthengine/utils_test.py | 1 - .../common/datacommons_api_wrappers/datacommons_wrappers.py | 3 --- .../datacommons_api_wrappers/datacommons_wrappers_test.py | 2 -- .../india_rbi_state_statistics/environment_sdg_metadata.csv | 1 - .../india_rbi_state_statistics/infrastructure_metadata.csv | 1 - .../india_rbi_state_statistics/rbi_metadata.csv | 5 ----- .../denmark_demographics/denmark_demographics_metadata.csv | 1 - .../fema/flood_insurance_claims/us_flood_nfip_config.py | 3 --- .../state_domestic_product_metadata.csv | 1 - .../statistics_poland/StatisticsPoland_metadata.csv | 3 --- .../ap_ib_gt_enrollment/config/common_metadata.csv | 1 - .../state/config/SATorACT_Participation_metadata.csv | 1 - tools/statvar_importer/config_flags.py | 2 +- util/dc_api_wrapper.py | 3 ++- 15 files changed, 3 insertions(+), 27 deletions(-) diff --git a/scripts/earthengine/utils.py b/scripts/earthengine/utils.py index 44dbd71535..7e59202eaa 100644 --- a/scripts/earthengine/utils.py +++ b/scripts/earthengine/utils.py @@ -46,7 +46,6 @@ # Constants _MAX_LATITUDE = 90.0 _MAX_LONGITUDE = 180.0 -_DC_API_ROOT = 'https://api.datacommons.org' # Utilities for dicts. @@ -372,7 +371,6 @@ def place_id_to_lat_lng(placeid: str, { 'dc_api_version': 'V2', 'dc_api_use_cache': True, - 'dc_api_root': _DC_API_ROOT, }, ) node_props = resp.get(placeid) if resp else None diff --git a/scripts/earthengine/utils_test.py b/scripts/earthengine/utils_test.py index bfdd347661..f0f8e567cd 100644 --- a/scripts/earthengine/utils_test.py +++ b/scripts/earthengine/utils_test.py @@ -394,5 +394,4 @@ def test_place_id_to_lat_lng_dc_api(self): [placeid], ['latitude', 'longitude'], { 'dc_api_version': 'V2', 'dc_api_use_cache': True, - 'dc_api_root': utils._DC_API_ROOT, }) diff --git a/scripts/us_census/acs5yr/subject_tables/common/datacommons_api_wrappers/datacommons_wrappers.py b/scripts/us_census/acs5yr/subject_tables/common/datacommons_api_wrappers/datacommons_wrappers.py index 39c93bc599..70e0936ee5 100644 --- a/scripts/us_census/acs5yr/subject_tables/common/datacommons_api_wrappers/datacommons_wrappers.py +++ b/scripts/us_census/acs5yr/subject_tables/common/datacommons_api_wrappers/datacommons_wrappers.py @@ -64,9 +64,6 @@ def dc_check_existence(dcid_list: list, wrapper_config = { 'dc_api_batch_size': max_items, - 'dc_api_root': - 'https://autopush.api.datacommons.org' - if use_autopush else 'https://api.datacommons.org' } return dc_api_is_defined_dcid(dcid_list, wrapper_config) diff --git a/scripts/us_census/acs5yr/subject_tables/common/datacommons_api_wrappers/datacommons_wrappers_test.py b/scripts/us_census/acs5yr/subject_tables/common/datacommons_api_wrappers/datacommons_wrappers_test.py index 5eb9d2a497..e0374c7010 100644 --- a/scripts/us_census/acs5yr/subject_tables/common/datacommons_api_wrappers/datacommons_wrappers_test.py +++ b/scripts/us_census/acs5yr/subject_tables/common/datacommons_api_wrappers/datacommons_wrappers_test.py @@ -37,14 +37,12 @@ def test_dc_check_existence_mock(self, mock_is_defined): mock_is_defined.assert_called_with( ['node1'], { 'dc_api_batch_size': 450, - 'dc_api_root': 'https://autopush.api.datacommons.org' }) # Test 2: use_autopush=False dc_check_existence(['node2'], use_autopush=False, max_items=10) mock_is_defined.assert_called_with(['node2'], { 'dc_api_batch_size': 10, - 'dc_api_root': 'https://api.datacommons.org' }) @mock.patch('datacommons_wrappers.request_post_json') diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_sdg_metadata.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_sdg_metadata.csv index 782aa4c330..f11ac0b3c4 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_sdg_metadata.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_sdg_metadata.csv @@ -2,4 +2,3 @@ parameter,value header_rows,3 output_columns,"observationAbout,observationDate,variableMeasured,value,unit,observationPeriod" mapped_rows,3 -dc_api_root,https://api.datacommons.org diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_metadata.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_metadata.csv index 475c900919..f5c45d8c4e 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_metadata.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/infrastructure_metadata.csv @@ -2,4 +2,3 @@ parameter,value header_rows,5 output_columns,"observationAbout,observationDate,variableMeasured,value,unit,observationPeriod" mapped_rows,5 -dc_api_root,https://api.datacommons.org diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv index ad0d50f768..c2042f4fd4 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_metadata.csv @@ -2,8 +2,3 @@ parameter,value output_columns,"observationAbout,observationDate,variableMeasured,value,unit,observationPeriod" header_rows,4 mapped_rows,4 -dc_api_root,https://api.datacommons.org - - - - diff --git a/statvar_imports/denmark_demographics/denmark_demographics_metadata.csv b/statvar_imports/denmark_demographics/denmark_demographics_metadata.csv index 41f8f31e37..95d252a541 100644 --- a/statvar_imports/denmark_demographics/denmark_demographics_metadata.csv +++ b/statvar_imports/denmark_demographics/denmark_demographics_metadata.csv @@ -1,3 +1,2 @@ parameter,value output_columns,"observationDate,value,observationAbout,variableMeasured" -dc_api_root,https://api.datacommons.org diff --git a/statvar_imports/fema/flood_insurance_claims/us_flood_nfip_config.py b/statvar_imports/fema/flood_insurance_claims/us_flood_nfip_config.py index 90e53db883..082ce8b7e3 100644 --- a/statvar_imports/fema/flood_insurance_claims/us_flood_nfip_config.py +++ b/statvar_imports/fema/flood_insurance_claims/us_flood_nfip_config.py @@ -68,7 +68,4 @@ 5, 'dc_api_use_cache': True, - #'dc_api_root': 'http://autopush.api.datacommons.org', - 'dc_api_root': - 'http://api.datacommons.org', } diff --git a/statvar_imports/india_rbistatedomesticproduct/state_domestic_product_metadata.csv b/statvar_imports/india_rbistatedomesticproduct/state_domestic_product_metadata.csv index ee630bdff4..0c90bd2702 100644 --- a/statvar_imports/india_rbistatedomesticproduct/state_domestic_product_metadata.csv +++ b/statvar_imports/india_rbistatedomesticproduct/state_domestic_product_metadata.csv @@ -11,4 +11,3 @@ comments, output_columns,"observationAbout,observationDate,variableMeasured,value,unit,measurementMethod,observationPeriod" #header_rows,6 #mapped_rows,5 -dc_api_root,https://api.datacommons.org diff --git a/statvar_imports/statistics_poland/StatisticsPoland_metadata.csv b/statvar_imports/statistics_poland/StatisticsPoland_metadata.csv index b909a13a08..a3a30ec1c2 100644 --- a/statvar_imports/statistics_poland/StatisticsPoland_metadata.csv +++ b/statvar_imports/statistics_poland/StatisticsPoland_metadata.csv @@ -9,6 +9,3 @@ places_within,country/POL #skip_rows,1 header_rows,5 mapped_columns,2 -dc_api_root,https://api.datacommons.org - - diff --git a/statvar_imports/us_urban_school/ap_ib_gt_enrollment/config/common_metadata.csv b/statvar_imports/us_urban_school/ap_ib_gt_enrollment/config/common_metadata.csv index 41a321a836..2c8f80a15c 100644 --- a/statvar_imports/us_urban_school/ap_ib_gt_enrollment/config/common_metadata.csv +++ b/statvar_imports/us_urban_school/ap_ib_gt_enrollment/config/common_metadata.csv @@ -3,4 +3,3 @@ mapped_rows,1 output_columns,"observationDate,observationAbout,variableMeasured,value" #input_rows,10 mapped_columns,2 -dc_api_root,https://api.datacommons.org diff --git a/statvar_imports/us_urban_school/sat_act_participation/state/config/SATorACT_Participation_metadata.csv b/statvar_imports/us_urban_school/sat_act_participation/state/config/SATorACT_Participation_metadata.csv index 4909fa4a53..75997951a4 100644 --- a/statvar_imports/us_urban_school/sat_act_participation/state/config/SATorACT_Participation_metadata.csv +++ b/statvar_imports/us_urban_school/sat_act_participation/state/config/SATorACT_Participation_metadata.csv @@ -1,3 +1,2 @@ parameter,value output_columns,"observationAbout,observationDate,value,variableMeasured,unit,scalingFactor" -dc_api_root,https://api.datacommons.org diff --git a/tools/statvar_importer/config_flags.py b/tools/statvar_importer/config_flags.py index d5214a3510..94a162f33c 100644 --- a/tools/statvar_importer/config_flags.py +++ b/tools/statvar_importer/config_flags.py @@ -370,7 +370,7 @@ def get_default_config() -> dict: True, # Settings for DC API. 'dc_api_root': - 'http://api.datacommons.org', + os.environ.get('DC_API_ROOT', 'http://api.datacommons.org'), 'dc_api_use_cache': False, 'dc_api_batch_size': diff --git a/util/dc_api_wrapper.py b/util/dc_api_wrapper.py index 98f7209d0e..682d4aeeaf 100644 --- a/util/dc_api_wrapper.py +++ b/util/dc_api_wrapper.py @@ -520,7 +520,8 @@ def dc_api_resolve_latlng(lat_lngs: list, dictionary containing the resolved place information. """ config = _validate_v2_config(config) - api_root = config.get('dc_api_root', _DEFAULT_API_ROOT) + api_root = config.get('dc_api_root', + os.environ.get('DC_API_ROOT', _DEFAULT_API_ROOT)) v1_data = {} v1_data['coordinates'] = lat_lngs num_ids = len(lat_lngs) From dfad7506200cbf30ee076a9e258e975460c965a2 Mon Sep 17 00:00:00 2001 From: Ajai Tirumali Date: Wed, 25 Mar 2026 23:12:48 +0530 Subject: [PATCH 4/6] lint fix --- scripts/earthengine/utils.py | 12 +-- scripts/earthengine/utils_test.py | 10 +- .../datacommons_wrappers.py | 3 +- .../datacommons_wrappers_test.py | 7 +- tools/statvar_importer/config_flags.py | 2 +- util/dc_api_wrapper.py | 100 +++++++++++------- 6 files changed, 77 insertions(+), 57 deletions(-) diff --git a/scripts/earthengine/utils.py b/scripts/earthengine/utils.py index 7e59202eaa..aeb94045ac 100644 --- a/scripts/earthengine/utils.py +++ b/scripts/earthengine/utils.py @@ -19,11 +19,11 @@ from datetime import datetime import glob import os +from pathlib import Path import pickle import re import sys import tempfile -from pathlib import Path from typing import Union from absl import logging @@ -305,8 +305,8 @@ def grid_get_neighbor_ids(grid_id: str) -> list: if lat_offset != 0 or lng_offset != 0: neighbour_lat = lat + lat_offset * deg neighbour_lng = lng + lng_offset * deg - if abs(neighbour_lat) < _MAX_LATITUDE and abs( - neighbour_lng) < _MAX_LONGITUDE: + if (abs(neighbour_lat) < _MAX_LATITUDE and + abs(neighbour_lng) < _MAX_LONGITUDE): neighbours.append( grid_id_from_lat_lng( deg, @@ -433,7 +433,7 @@ def add_namespace(dcid: str, prefix: str = 'dcid:') -> str: def str_get_numeric_value( - value: Union[str, list, int, float]) -> Union[int, float, None]: + value: Union[str, list, int, float],) -> Union[int, float, None]: """Returns the numeric value from input string or None.""" if isinstance(value, list): value = value[0] @@ -528,7 +528,7 @@ def date_advance_by_period(date_str: str, if not date_str: return '' dt = datetime.strptime(date_str, date_format) - (delta, unit) = date_parse_time_period(time_period) + delta, unit = date_parse_time_period(time_period) if not delta or not unit: logging.error( f'Unable to parse time period: {time_period} for date: {date_str}') @@ -545,7 +545,7 @@ def date_format_by_time_period(date_str: str, time_period: str) -> str: """ if not time_period: return date_str - (delta, unit) = date_parse_time_period(time_period) + delta, unit = date_parse_time_period(time_period) date_parts = date_str.split('-') if unit == 'years': return date_parts[0] diff --git a/scripts/earthengine/utils_test.py b/scripts/earthengine/utils_test.py index f0f8e567cd..e93be53fea 100644 --- a/scripts/earthengine/utils_test.py +++ b/scripts/earthengine/utils_test.py @@ -390,8 +390,8 @@ def test_place_id_to_lat_lng_dc_api(self): lat, lng = utils.place_id_to_lat_lng(placeid, dc_api_lookup=True) self.assertAlmostEqual(37.221614, lat) self.assertAlmostEqual(-121.68954, lng) - mock_get.assert_called_once_with( - [placeid], ['latitude', 'longitude'], { - 'dc_api_version': 'V2', - 'dc_api_use_cache': True, - }) + mock_get.assert_called_once_with([placeid], + ['latitude', 'longitude'], { + 'dc_api_version': 'V2', + 'dc_api_use_cache': True, + }) diff --git a/scripts/us_census/acs5yr/subject_tables/common/datacommons_api_wrappers/datacommons_wrappers.py b/scripts/us_census/acs5yr/subject_tables/common/datacommons_api_wrappers/datacommons_wrappers.py index 70e0936ee5..eb0e487f2a 100644 --- a/scripts/us_census/acs5yr/subject_tables/common/datacommons_api_wrappers/datacommons_wrappers.py +++ b/scripts/us_census/acs5yr/subject_tables/common/datacommons_api_wrappers/datacommons_wrappers.py @@ -62,8 +62,7 @@ def dc_check_existence(dcid_list: list, Dict object with dcids as key values and boolean values signifying existence as values. """ wrapper_config = { - 'dc_api_batch_size': - max_items, + 'dc_api_batch_size': max_items, } return dc_api_is_defined_dcid(dcid_list, wrapper_config) diff --git a/scripts/us_census/acs5yr/subject_tables/common/datacommons_api_wrappers/datacommons_wrappers_test.py b/scripts/us_census/acs5yr/subject_tables/common/datacommons_api_wrappers/datacommons_wrappers_test.py index e0374c7010..e605afb5a9 100644 --- a/scripts/us_census/acs5yr/subject_tables/common/datacommons_api_wrappers/datacommons_wrappers_test.py +++ b/scripts/us_census/acs5yr/subject_tables/common/datacommons_api_wrappers/datacommons_wrappers_test.py @@ -34,10 +34,9 @@ def test_dc_check_existence_mock(self, mock_is_defined): # Test 1: Default (use_autopush=True by default in function signature) mock_is_defined.return_value = {'node1': True} dc_check_existence(['node1']) - mock_is_defined.assert_called_with( - ['node1'], { - 'dc_api_batch_size': 450, - }) + mock_is_defined.assert_called_with(['node1'], { + 'dc_api_batch_size': 450, + }) # Test 2: use_autopush=False dc_check_existence(['node2'], use_autopush=False, max_items=10) diff --git a/tools/statvar_importer/config_flags.py b/tools/statvar_importer/config_flags.py index 94a162f33c..a7449ca032 100644 --- a/tools/statvar_importer/config_flags.py +++ b/tools/statvar_importer/config_flags.py @@ -370,7 +370,7 @@ def get_default_config() -> dict: True, # Settings for DC API. 'dc_api_root': - os.environ.get('DC_API_ROOT', 'http://api.datacommons.org'), + os.environ.get('DC_API_ROOT', 'https://api.datacommons.org'), 'dc_api_use_cache': False, 'dc_api_batch_size': diff --git a/util/dc_api_wrapper.py b/util/dc_api_wrapper.py index 682d4aeeaf..1265452f8c 100644 --- a/util/dc_api_wrapper.py +++ b/util/dc_api_wrapper.py @@ -21,16 +21,21 @@ import os import sys -import urllib -import requests from typing import Union +import urllib from absl import logging from datacommons_client.client import DataCommonsClient from datacommons_client.utils.error_handling import APIError, DCConnectionError, DCStatusError +import requests import requests_cache -from tenacity import (RetryCallState, Retrying, retry_if_exception, - stop_after_attempt, wait_fixed) +from tenacity import ( + RetryCallState, + Retrying, + retry_if_exception, + stop_after_attempt, + wait_fixed, +) _SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) sys.path.append(_SCRIPT_DIR) @@ -64,8 +69,14 @@ def _get_exception_status_code(exception): def _should_retry_exception(exception: Exception) -> bool: - if isinstance(exception, (DCConnectionError, requests.exceptions.Timeout, - requests.exceptions.ChunkedEncodingError)): + if isinstance( + exception, + ( + DCConnectionError, + requests.exceptions.Timeout, + requests.exceptions.ChunkedEncodingError, + ), + ): return True if isinstance(exception, (urllib.error.HTTPError, DCStatusError, APIError)): status_code = _get_exception_status_code(exception) @@ -105,6 +116,7 @@ def dc_api_wrapper( retries: Maximum number of attempts (including the first attempt). retry_sec: Interval in seconds between retries for which caller is blocked. use_cache: If True, uses request cache for faster response. + Returns: The response from the DataCommons API call. """ @@ -147,8 +159,9 @@ def dc_api_wrapper( logging.error(f'Got exception for api: {function}, {e}') return None except Exception as e: - e.add_note(f'DC API call failed for {function} with max attempts ' - f'{max_attempts}.') + e.add_note( + f'DC API call failed for {function} with max attempts {max_attempts}.' + ) raise @@ -252,11 +265,13 @@ def get_dc_api_key(config: dict = None) -> str: api_key = config.get('dc_api_key', os.environ.get('DC_API_KEY')) if not api_key: logging.log_first_n( - logging.WARNING, f'Using default DC API key with limited quota. ' - 'Please set an API key in the environment variable: DC_API_KEY.' - 'Refer https://docs.datacommons.org/api/python/v2/#authentication ' - 'for more details.', - n=1) + logging.WARNING, + f'Using default DC API key with limited quota. ' + f'Please set an API key in the environment variable: DC_API_KEY.' + f'Refer https://docs.datacommons.org/api/python/v2/#authentication ' + f'for more details.', + n=1, + ) api_key = _DEFAULT_DC_API_KEY return api_key @@ -286,6 +301,7 @@ def get_datacommons_client(config: dict = None) -> DataCommonsClient: def dc_api_is_defined_dcid(dcids: list, config: dict = {}) -> dict: """Returns a dictionary with dcids mapped to True/False based on whether + the dcid is defined in the API and has a 'typeOf' property. Uses the property_value() DC API to lookup 'typeOf' for each dcid. dcids not defined in KG get a value of False. @@ -301,11 +317,13 @@ def dc_api_is_defined_dcid(dcids: list, config: dict = {}) -> dict: # Set parameters for node API. client = get_datacommons_client(config) api_function = client.node.fetch_property_values - api_result = dc_api_batched_wrapper(function=api_function, - dcids=dcids, - args={'properties': 'typeOf'}, - dcid_arg_kw='node_dcids', - config=config) + api_result = dc_api_batched_wrapper( + function=api_function, + dcids=dcids, + args={'properties': 'typeOf'}, + dcid_arg_kw='node_dcids', + config=config, + ) response = {} for dcid in dcids: dcid_stripped = _strip_namespace(dcid) @@ -348,11 +366,13 @@ def _dc_api_get_node_property_v2(dcids: list, api_function = client.node.fetch_property_values args = {'properties': prop} dcid_arg_kw = 'node_dcids' - api_result = dc_api_batched_wrapper(function=api_function, - dcids=dcids, - args=args, - dcid_arg_kw=dcid_arg_kw, - config=config) + api_result = dc_api_batched_wrapper( + function=api_function, + dcids=dcids, + args=args, + dcid_arg_kw=dcid_arg_kw, + config=config, + ) response = {} for dcid in dcids: dcid_stripped = _strip_namespace(dcid) @@ -398,11 +418,13 @@ def dc_api_get_node_property_values(dcids: list, config: dict = {}) -> dict: api_function = client.node.fetch args = {'expression': '->*'} dcid_arg_kw = 'node_dcids' - api_result = dc_api_batched_wrapper(function=api_function, - dcids=dcids, - args=args, - dcid_arg_kw=dcid_arg_kw, - config=config) + api_result = dc_api_batched_wrapper( + function=api_function, + dcids=dcids, + args=args, + dcid_arg_kw=dcid_arg_kw, + config=config, + ) response = {} for dcid, arcs in api_result.items(): pvs = {} @@ -446,11 +468,13 @@ def dc_api_resolve_placeid(dcids: list, api_function = client.resolve.fetch args = {'expression': f'<-{in_prop}->dcid'} dcid_arg_kw = 'node_ids' - api_result = dc_api_batched_wrapper(function=api_function, - dcids=dcids, - args=args, - dcid_arg_kw=dcid_arg_kw, - config=config) + api_result = dc_api_batched_wrapper( + function=api_function, + dcids=dcids, + args=args, + dcid_arg_kw=dcid_arg_kw, + config=config, + ) results = {} if api_result: for node in api_result.get('entities', []): @@ -478,7 +502,7 @@ def dc_api_resolve_latlng(lat_lngs: list, } if return_v1_response is True, a v1 response of this form is returned: - + { "placeCoordinates": [ { @@ -552,8 +576,7 @@ def dc_api_resolve_latlng(lat_lngs: list, def _convert_v2_to_v1_coordinate_response(v2_response: dict) -> dict: - """Converts a v2 coordinate resolution response to a v1 response. - """ + """Converts a v2 coordinate resolution response to a v1 response.""" v1_response = {'placeCoordinates': []} for entity in v2_response.get('entities', []): node = entity.get('node', '') @@ -573,15 +596,14 @@ def _convert_v2_to_v1_coordinate_response(v2_response: dict) -> dict: candidate.get('dcid') for candidate in entity.get('candidates', []) ], - 'places': entity.get('candidates', []) + 'places': entity.get('candidates', []), } v1_response['placeCoordinates'].append(place_coordinate) return v1_response def _convert_v1_to_v2_coordinate_request(v1_request: dict) -> dict: - """Converts a v1 coordinate resolution request to a v2 request. - """ + """Converts a v1 coordinate resolution request to a v2 request.""" v2_request = {'nodes': [], 'property': '<-geoCoordinate->dcid'} for coordinate in v1_request.get('coordinates', []): lat = coordinate.get('latitude') From 32bd938d3df0bb3333a74532777737725e614f09 Mon Sep 17 00:00:00 2001 From: Ajai Tirumali Date: Sat, 25 Jul 2026 00:09:18 +0530 Subject: [PATCH 5/6] Switch dc-import to prod --- import-automation/executor/app/configs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/import-automation/executor/app/configs.py b/import-automation/executor/app/configs.py index de49bc80e4..d2dcb5e37b 100644 --- a/import-automation/executor/app/configs.py +++ b/import-automation/executor/app/configs.py @@ -140,7 +140,7 @@ class ExecutorConfig: "gs://unresolved_mcf/scripts/statvar/stat_vars.mcf" }) # API Environment used by the import tool - use_autopush_dc_api: bool = True + use_autopush_dc_api: bool = False # Invoke import tool genmcf. invoke_import_tool: bool = True # Invoke differ tool. From d1ff8f12a0b612a2624404c10c4b388ec3343ceb Mon Sep 17 00:00:00 2001 From: Ajai Tirumali Date: Wed, 26 Aug 2026 12:00:34 +0000 Subject: [PATCH 6/6] Migrate compatible imports to use download_util_script.py --- scripts/us_bea/states_gdp/manifest.json | 58 +++++++++---------- statvar_imports/ccd/enrollment/manifest.json | 47 +++++++-------- .../uae_bayanat/uae_population/manifest.json | 44 +++++++------- statvar_imports/undata/manifest.json | 48 +++++++-------- .../us_monthly_retail_sales/manifest.json | 44 +++++++------- 5 files changed, 121 insertions(+), 120 deletions(-) diff --git a/scripts/us_bea/states_gdp/manifest.json b/scripts/us_bea/states_gdp/manifest.json index f9248ef063..9d43781a7d 100644 --- a/scripts/us_bea/states_gdp/manifest.json +++ b/scripts/us_bea/states_gdp/manifest.json @@ -1,33 +1,33 @@ { - "import_specifications": [ + "import_specifications": [ + { + "import_name": "BEA_USStatesQuarterlyGDP", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://www.bea.gov/data/gdp/gdp-state", + "provenance_description": "Quarterly data on inflation-adjusted GDP at a state level in the United States from the US Bureau of Economic Analysis.", + "scripts": [ + "../../../util/download_util_script.py --download_url=https://apps.bea.gov/regional/zip/SQGDP.zip --output_folder=input_files --unzip=True", + "preprocess_csv.py", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/preprocessed_data.csv --pv_map=bea_gdp_pv_map.csv --config_file=bea_gdp_metadata.csv --places_resolved_csv=bea_gdp_place_map.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/BEA_GDP_output" + ], + "source_files": [ + "input_data/*.csv" + ], + "import_inputs": [ { - "import_name": "BEA_USStatesQuarterlyGDP", - "curator_emails": [ - "support@datacommons.org" - ], - "provenance_url": "https://www.bea.gov/data/gdp/gdp-state", - "provenance_description": "Quarterly data on inflation-adjusted GDP at a state level in the United States from the US Bureau of Economic Analysis.", - "scripts": [ - "common_download.py", - "import_data.py", - "import_industry_data_and_gen_mcf.py" - ], - "source_files": [ - "input_data/*.csv" - ], - "import_inputs": [ - { - "cleaned_csv": "states_gdp.csv", - "template_mcf": "states_gdp.tmcf", - "node_mcf": "states_gdp_statvars.mcf" - }, - { - "cleaned_csv": "states_industry_gdp.csv", - "node_mcf": "states_gdp_industry_statvars.mcf", - "template_mcf": "states_industry_gdp.tmcf" - } - ], - "cron_schedule": "30 10 30 1,4,7,10,12 *" + "cleaned_csv": "states_gdp.csv", + "template_mcf": "states_gdp.tmcf", + "node_mcf": "states_gdp_statvars.mcf" + }, + { + "cleaned_csv": "states_industry_gdp.csv", + "node_mcf": "states_gdp_industry_statvars.mcf", + "template_mcf": "states_industry_gdp.tmcf" } - ] + ], + "cron_schedule": "30 10 30 1,4,7,10,12 *" + } + ] } diff --git a/statvar_imports/ccd/enrollment/manifest.json b/statvar_imports/ccd/enrollment/manifest.json index c8cce2d201..09fd2a711c 100644 --- a/statvar_imports/ccd/enrollment/manifest.json +++ b/statvar_imports/ccd/enrollment/manifest.json @@ -1,26 +1,27 @@ { - "import_specifications": [ + "import_specifications": [ + { + "import_name": "US_CCD_Enrollment", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://nces.ed.gov/programs/digest/d24/tables/dt24_203.65.asp", + "provenance_description": "Enrollment in public elementary and secondary schools, by level, grade, and race/ethnicity.", + "scripts": [ + "../../../util/download_util_script.py --download_url=https://nces.ed.gov/programs/digest/d24/tables/xls/tabn203.65.xlsx --output_folder=source_files", + "preprocess.py", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.csv --pv_map=pvmap.csv --config_file=metadata.csv --places_resolved_csv=place_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/enrollment_output" + ], + "source_files": [ + "input_files/ccd_enrollment_input.csv" + ], + "import_inputs": [ { - "import_name": "US_CCD_Enrollment", - "curator_emails": [ - "support@datacommons.org" - ], - "provenance_url": "https://nces.ed.gov/programs/digest/d24/tables/dt24_203.65.asp", - "provenance_description": "Enrollment in public elementary and secondary schools, by level, grade, and race/ethnicity.", - "scripts": [ - "download.py", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ccd_enrollment_input.csv --pv_map=CCD_Enrollment_pvmap.csv --config_file=CCD_Enrollment_metadata.csv --output_path=output/CCD_Enrollment_output --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf" - ], - "source_files": [ - "input_files/ccd_enrollment_input.csv" - ], - "import_inputs": [ - { - "template_mcf": "output/CCD_Enrollment_output.tmcf", - "cleaned_csv": "output/CCD_Enrollment_output.csv" - } - ], - "cron_schedule": "0 06 * * 5" + "template_mcf": "output/CCD_Enrollment_output.tmcf", + "cleaned_csv": "output/CCD_Enrollment_output.csv" } - ] -} \ No newline at end of file + ], + "cron_schedule": "0 06 * * 5" + } + ] +} diff --git a/statvar_imports/uae_bayanat/uae_population/manifest.json b/statvar_imports/uae_bayanat/uae_population/manifest.json index e109c14730..2c45c50a2c 100644 --- a/statvar_imports/uae_bayanat/uae_population/manifest.json +++ b/statvar_imports/uae_bayanat/uae_population/manifest.json @@ -1,26 +1,26 @@ { - "import_specifications": [ + "import_specifications": [ + { + "import_name": "UAE_Population", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://bayanat.ae/api/DatasetResources/DownloadSingle?resourceID=FT1CGOdcLYt6KdUJnFHWwOdTWpeao-LX539QkMLeAKA&fileName=UAE%20Population%20By%20Emirates%20Nationality%20and%20gender", + "provenance_description": "UAE_Population statical data", + "scripts": [ + "../../../util/download_util_script.py --download_url=https://bayanat.ae/api/DatasetResources/DownloadSingle?resourceID=FT1CGOdcLYt6KdUJnFHWwOdTWpeao-LX539QkMLeAKA&fileName=UAE%20Population%20By%20Emirates%20Nationality%20and%20gender --output_folder=input", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input/uae_populationbyemiratesnationalityandgender.xlsx --pv_map=uae_population_pvmap.csv --config_file=uae_population_metadata.csv --places_resolved_csv=uae_population_places_resolved_csv.csv --output_path=output/uae_population_output" + ], + "source_files": [ + "input/uae_populationbyemiratesnationalityandgender.xlsx" + ], + "import_inputs": [ { - "import_name": "UAE_Population", - "curator_emails": [ - "support@datacommons.org" - ], - "provenance_url": "https://bayanat.ae/api/DatasetResources/DownloadSingle?resourceID=FT1CGOdcLYt6KdUJnFHWwOdTWpeao-LX539QkMLeAKA&fileName=UAE%20Population%20By%20Emirates%20Nationality%20and%20gender", - "provenance_description": "UAE_Population statical data", - "scripts": [ - "uae_download.py", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=input/uae_populationbyemiratesnationalityandgender.xlsx --pv_map=uae_population_pvmap.csv --config_file=uae_population_metadata.csv --places_resolved_csv=uae_population_places_resolved_csv.csv --output_path=output/uae_population_output" - ], - "source_files": [ - "input/uae_populationbyemiratesnationalityandgender.xlsx" - ], - "import_inputs": [ - { - "template_mcf": "output/uae_population_output.tmcf", - "cleaned_csv": "output/uae_population_output.csv" - } - ], - "cron_schedule": "0 05 * * 1" + "template_mcf": "output/uae_population_output.tmcf", + "cleaned_csv": "output/uae_population_output.csv" } - ] + ], + "cron_schedule": "0 05 * * 1" + } + ] } diff --git a/statvar_imports/undata/manifest.json b/statvar_imports/undata/manifest.json index 055dd20efd..5f3cef4af8 100644 --- a/statvar_imports/undata/manifest.json +++ b/statvar_imports/undata/manifest.json @@ -1,27 +1,27 @@ { - "import_specifications": [ + "import_specifications": [ + { + "import_name": "UNData", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://data.un.org/Data.aspx?q=city+population&d=POP&f=tableCode%3a240", + "provenance_description": "Population data for countries, capital cities, urban and rural areas not covered by other sources. Outliers were removed from the output", + "scripts": [ + "../../util/download_util_script.py --download_url=https://data.un.org/Handlers/DownloadHandler.ashx?DataFilter=tableCode:240&DataMartId=POP&Format=csv&c=2,3,6,8,10,12,14,16,17,18&s=_countryEnglishNameOrderBy:asc,refYear:desc,areaCode:asc --output_folder=input_file --unzip=True", + "../../tools/statvar_importer/stat_var_processor.py --input_data=input_file/*.csv --pv_map=UNData_pvmap.csv --config_file=UNData_metadata.csv --places_resolved_csv=UNData_places_resolved_csv.csv --output_path=output/UNData", + "../../tools/statvar_importer/filter_data_outliers.py --filter_data_input=output/UNData.csv --filter_data_output=output/filter_undata.csv --filter_data_min_value=2 --filter_data_max_yearly_change_ratio=0.5 --filter_data_max_change_ratio=1" + ], + "source_files": [ + "input_file/*.csv" + ], + "import_inputs": [ { - "import_name": "UNData", - "curator_emails": [ - "support@datacommons.org" - ], - "provenance_url": "https://data.un.org/Data.aspx?q=city+population&d=POP&f=tableCode%3a240", - "provenance_description": "Population data for countries, capital cities, urban and rural areas not covered by other sources. Outliers were removed from the output", - "scripts": [ - "download.py", - "../../tools/statvar_importer/stat_var_processor.py --input_data=input_file/*.csv --pv_map=UNData_pvmap.csv --config_file=UNData_metadata.csv --places_resolved_csv=UNData_places_resolved_csv.csv --output_path=output/UNData", - "../../tools/statvar_importer/filter_data_outliers.py --filter_data_input=output/UNData.csv --filter_data_output=output/filter_undata.csv --filter_data_min_value=2 --filter_data_max_yearly_change_ratio=0.5 --filter_data_max_change_ratio=1" - ], - "source_files": [ - "input_file/*.csv" - ], - "import_inputs": [ - { - "template_mcf": "output/UNData.tmcf", - "cleaned_csv": "output/filter_undata.csv" - } - ], - "cron_schedule": "0 05 * * 3" + "template_mcf": "output/UNData.tmcf", + "cleaned_csv": "output/filter_undata.csv" } - ] -} \ No newline at end of file + ], + "cron_schedule": "0 05 * * 3" + } + ] +} diff --git a/statvar_imports/us_census/us_monthly_retail_sales/manifest.json b/statvar_imports/us_census/us_monthly_retail_sales/manifest.json index 71a273e9a9..1ce7a88c08 100644 --- a/statvar_imports/us_census/us_monthly_retail_sales/manifest.json +++ b/statvar_imports/us_census/us_monthly_retail_sales/manifest.json @@ -1,26 +1,26 @@ { - "import_specifications": [ + "import_specifications": [ + { + "import_name": "USMonthlyRetailSales", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://www.census.gov/retail/", + "provenance_description": "The Advance Monthly and Monthly Retail Trade Surveys (MARTS and MRTS), the Annual Retail Trade Survey (ARTS), and the Quarterly E-Commerce Report work together to produce the most comprehensive data available on retail economic activity in the United States.", + "scripts": [ + "../../../util/download_util_script.py --download_url=https://www.census.gov/retail/mrts/www/mrtssales92-present.xlsx --output_folder=input_file", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_file/mrtssales92-present.xlsx --pv_map=monthly_retail_pvmap.csv --statvar_dcid_remap_csv=monthly_retail_remap.csv --config_file=monthly_retail_metadata.csv --output_path=output/monthly_retail" + ], + "import_inputs": [ { - "import_name": "USMonthlyRetailSales", - "curator_emails": [ - "support@datacommons.org" - ], - "provenance_url": "https://www.census.gov/retail/", - "provenance_description": "The Advance Monthly and Monthly Retail Trade Surveys (MARTS and MRTS), the Annual Retail Trade Survey (ARTS), and the Quarterly E-Commerce Report work together to produce the most comprehensive data available on retail economic activity in the United States.", - "scripts": [ - "download.py", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_file/monthly_retail.xlsx --pv_map=monthly_retail_pvmap.csv --statvar_dcid_remap_csv=monthly_retail_remap.csv --config_file=monthly_retail_metadata.csv --output_path=output/monthly_retail" - ], - "import_inputs": [ - { - "template_mcf": "output/monthly_retail.tmcf", - "cleaned_csv": "output/monthly_retail.csv" - } - ], - "source_files": [ - "input_file/monthly_retail.xlsx" - ], - "cron_schedule": "30 05 * * 1" + "template_mcf": "output/monthly_retail.tmcf", + "cleaned_csv": "output/monthly_retail.csv" } - ] + ], + "source_files": [ + "input_file/mrtssales92-present.xlsx" + ], + "cron_schedule": "30 05 * * 1" + } + ] }