diff --git a/import-automation/executor/app/configs.py b/import-automation/executor/app/configs.py index 179e6c93f9..6049a7f927 100644 --- a/import-automation/executor/app/configs.py +++ b/import-automation/executor/app/configs.py @@ -140,7 +140,7 @@ class ExecutorConfig: "gs://unresolved_mcf/scripts/statvar/stat_vars.mcf" }) # API Environment used by the import tool - use_autopush_dc_api: bool = True + use_autopush_dc_api: bool = False # Invoke import tool genmcf. invoke_import_tool: bool = True # Invoke differ tool. diff --git a/scripts/us_bea/states_gdp/manifest.json b/scripts/us_bea/states_gdp/manifest.json index f9248ef063..9d43781a7d 100644 --- a/scripts/us_bea/states_gdp/manifest.json +++ b/scripts/us_bea/states_gdp/manifest.json @@ -1,33 +1,33 @@ { - "import_specifications": [ + "import_specifications": [ + { + "import_name": "BEA_USStatesQuarterlyGDP", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://www.bea.gov/data/gdp/gdp-state", + "provenance_description": "Quarterly data on inflation-adjusted GDP at a state level in the United States from the US Bureau of Economic Analysis.", + "scripts": [ + "../../../util/download_util_script.py --download_url=https://apps.bea.gov/regional/zip/SQGDP.zip --output_folder=input_files --unzip=True", + "preprocess_csv.py", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/preprocessed_data.csv --pv_map=bea_gdp_pv_map.csv --config_file=bea_gdp_metadata.csv --places_resolved_csv=bea_gdp_place_map.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/BEA_GDP_output" + ], + "source_files": [ + "input_data/*.csv" + ], + "import_inputs": [ { - "import_name": "BEA_USStatesQuarterlyGDP", - "curator_emails": [ - "support@datacommons.org" - ], - "provenance_url": "https://www.bea.gov/data/gdp/gdp-state", - "provenance_description": "Quarterly data on inflation-adjusted GDP at a state level in the United States from the US Bureau of Economic Analysis.", - "scripts": [ - "common_download.py", - "import_data.py", - "import_industry_data_and_gen_mcf.py" - ], - "source_files": [ - "input_data/*.csv" - ], - "import_inputs": [ - { - "cleaned_csv": "states_gdp.csv", - "template_mcf": "states_gdp.tmcf", - "node_mcf": "states_gdp_statvars.mcf" - }, - { - "cleaned_csv": "states_industry_gdp.csv", - "node_mcf": "states_gdp_industry_statvars.mcf", - "template_mcf": "states_industry_gdp.tmcf" - } - ], - "cron_schedule": "30 10 30 1,4,7,10,12 *" + "cleaned_csv": "states_gdp.csv", + "template_mcf": "states_gdp.tmcf", + "node_mcf": "states_gdp_statvars.mcf" + }, + { + "cleaned_csv": "states_industry_gdp.csv", + "node_mcf": "states_gdp_industry_statvars.mcf", + "template_mcf": "states_industry_gdp.tmcf" } - ] + ], + "cron_schedule": "30 10 30 1,4,7,10,12 *" + } + ] } diff --git a/statvar_imports/ccd/enrollment/manifest.json b/statvar_imports/ccd/enrollment/manifest.json index c8cce2d201..09fd2a711c 100644 --- a/statvar_imports/ccd/enrollment/manifest.json +++ b/statvar_imports/ccd/enrollment/manifest.json @@ -1,26 +1,27 @@ { - "import_specifications": [ + "import_specifications": [ + { + "import_name": "US_CCD_Enrollment", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://nces.ed.gov/programs/digest/d24/tables/dt24_203.65.asp", + "provenance_description": "Enrollment in public elementary and secondary schools, by level, grade, and race/ethnicity.", + "scripts": [ + "../../../util/download_util_script.py --download_url=https://nces.ed.gov/programs/digest/d24/tables/xls/tabn203.65.xlsx --output_folder=source_files", + "preprocess.py", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.csv --pv_map=pvmap.csv --config_file=metadata.csv --places_resolved_csv=place_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/enrollment_output" + ], + "source_files": [ + "input_files/ccd_enrollment_input.csv" + ], + "import_inputs": [ { - "import_name": "US_CCD_Enrollment", - "curator_emails": [ - "support@datacommons.org" - ], - "provenance_url": "https://nces.ed.gov/programs/digest/d24/tables/dt24_203.65.asp", - "provenance_description": "Enrollment in public elementary and secondary schools, by level, grade, and race/ethnicity.", - "scripts": [ - "download.py", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ccd_enrollment_input.csv --pv_map=CCD_Enrollment_pvmap.csv --config_file=CCD_Enrollment_metadata.csv --output_path=output/CCD_Enrollment_output --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf" - ], - "source_files": [ - "input_files/ccd_enrollment_input.csv" - ], - "import_inputs": [ - { - "template_mcf": "output/CCD_Enrollment_output.tmcf", - "cleaned_csv": "output/CCD_Enrollment_output.csv" - } - ], - "cron_schedule": "0 06 * * 5" + "template_mcf": "output/CCD_Enrollment_output.tmcf", + "cleaned_csv": "output/CCD_Enrollment_output.csv" } - ] -} \ No newline at end of file + ], + "cron_schedule": "0 06 * * 5" + } + ] +} diff --git a/statvar_imports/uae_bayanat/uae_population/manifest.json b/statvar_imports/uae_bayanat/uae_population/manifest.json index e109c14730..2c45c50a2c 100644 --- a/statvar_imports/uae_bayanat/uae_population/manifest.json +++ b/statvar_imports/uae_bayanat/uae_population/manifest.json @@ -1,26 +1,26 @@ { - "import_specifications": [ + "import_specifications": [ + { + "import_name": "UAE_Population", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://bayanat.ae/api/DatasetResources/DownloadSingle?resourceID=FT1CGOdcLYt6KdUJnFHWwOdTWpeao-LX539QkMLeAKA&fileName=UAE%20Population%20By%20Emirates%20Nationality%20and%20gender", + "provenance_description": "UAE_Population statical data", + "scripts": [ + "../../../util/download_util_script.py --download_url=https://bayanat.ae/api/DatasetResources/DownloadSingle?resourceID=FT1CGOdcLYt6KdUJnFHWwOdTWpeao-LX539QkMLeAKA&fileName=UAE%20Population%20By%20Emirates%20Nationality%20and%20gender --output_folder=input", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input/uae_populationbyemiratesnationalityandgender.xlsx --pv_map=uae_population_pvmap.csv --config_file=uae_population_metadata.csv --places_resolved_csv=uae_population_places_resolved_csv.csv --output_path=output/uae_population_output" + ], + "source_files": [ + "input/uae_populationbyemiratesnationalityandgender.xlsx" + ], + "import_inputs": [ { - "import_name": "UAE_Population", - "curator_emails": [ - "support@datacommons.org" - ], - "provenance_url": "https://bayanat.ae/api/DatasetResources/DownloadSingle?resourceID=FT1CGOdcLYt6KdUJnFHWwOdTWpeao-LX539QkMLeAKA&fileName=UAE%20Population%20By%20Emirates%20Nationality%20and%20gender", - "provenance_description": "UAE_Population statical data", - "scripts": [ - "uae_download.py", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=input/uae_populationbyemiratesnationalityandgender.xlsx --pv_map=uae_population_pvmap.csv --config_file=uae_population_metadata.csv --places_resolved_csv=uae_population_places_resolved_csv.csv --output_path=output/uae_population_output" - ], - "source_files": [ - "input/uae_populationbyemiratesnationalityandgender.xlsx" - ], - "import_inputs": [ - { - "template_mcf": "output/uae_population_output.tmcf", - "cleaned_csv": "output/uae_population_output.csv" - } - ], - "cron_schedule": "0 05 * * 1" + "template_mcf": "output/uae_population_output.tmcf", + "cleaned_csv": "output/uae_population_output.csv" } - ] + ], + "cron_schedule": "0 05 * * 1" + } + ] } diff --git a/statvar_imports/undata/manifest.json b/statvar_imports/undata/manifest.json index 055dd20efd..5f3cef4af8 100644 --- a/statvar_imports/undata/manifest.json +++ b/statvar_imports/undata/manifest.json @@ -1,27 +1,27 @@ { - "import_specifications": [ + "import_specifications": [ + { + "import_name": "UNData", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://data.un.org/Data.aspx?q=city+population&d=POP&f=tableCode%3a240", + "provenance_description": "Population data for countries, capital cities, urban and rural areas not covered by other sources. Outliers were removed from the output", + "scripts": [ + "../../util/download_util_script.py --download_url=https://data.un.org/Handlers/DownloadHandler.ashx?DataFilter=tableCode:240&DataMartId=POP&Format=csv&c=2,3,6,8,10,12,14,16,17,18&s=_countryEnglishNameOrderBy:asc,refYear:desc,areaCode:asc --output_folder=input_file --unzip=True", + "../../tools/statvar_importer/stat_var_processor.py --input_data=input_file/*.csv --pv_map=UNData_pvmap.csv --config_file=UNData_metadata.csv --places_resolved_csv=UNData_places_resolved_csv.csv --output_path=output/UNData", + "../../tools/statvar_importer/filter_data_outliers.py --filter_data_input=output/UNData.csv --filter_data_output=output/filter_undata.csv --filter_data_min_value=2 --filter_data_max_yearly_change_ratio=0.5 --filter_data_max_change_ratio=1" + ], + "source_files": [ + "input_file/*.csv" + ], + "import_inputs": [ { - "import_name": "UNData", - "curator_emails": [ - "support@datacommons.org" - ], - "provenance_url": "https://data.un.org/Data.aspx?q=city+population&d=POP&f=tableCode%3a240", - "provenance_description": "Population data for countries, capital cities, urban and rural areas not covered by other sources. Outliers were removed from the output", - "scripts": [ - "download.py", - "../../tools/statvar_importer/stat_var_processor.py --input_data=input_file/*.csv --pv_map=UNData_pvmap.csv --config_file=UNData_metadata.csv --places_resolved_csv=UNData_places_resolved_csv.csv --output_path=output/UNData", - "../../tools/statvar_importer/filter_data_outliers.py --filter_data_input=output/UNData.csv --filter_data_output=output/filter_undata.csv --filter_data_min_value=2 --filter_data_max_yearly_change_ratio=0.5 --filter_data_max_change_ratio=1" - ], - "source_files": [ - "input_file/*.csv" - ], - "import_inputs": [ - { - "template_mcf": "output/UNData.tmcf", - "cleaned_csv": "output/filter_undata.csv" - } - ], - "cron_schedule": "0 05 * * 3" + "template_mcf": "output/UNData.tmcf", + "cleaned_csv": "output/filter_undata.csv" } - ] -} \ No newline at end of file + ], + "cron_schedule": "0 05 * * 3" + } + ] +} diff --git a/statvar_imports/us_census/us_monthly_retail_sales/manifest.json b/statvar_imports/us_census/us_monthly_retail_sales/manifest.json index 71a273e9a9..1ce7a88c08 100644 --- a/statvar_imports/us_census/us_monthly_retail_sales/manifest.json +++ b/statvar_imports/us_census/us_monthly_retail_sales/manifest.json @@ -1,26 +1,26 @@ { - "import_specifications": [ + "import_specifications": [ + { + "import_name": "USMonthlyRetailSales", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://www.census.gov/retail/", + "provenance_description": "The Advance Monthly and Monthly Retail Trade Surveys (MARTS and MRTS), the Annual Retail Trade Survey (ARTS), and the Quarterly E-Commerce Report work together to produce the most comprehensive data available on retail economic activity in the United States.", + "scripts": [ + "../../../util/download_util_script.py --download_url=https://www.census.gov/retail/mrts/www/mrtssales92-present.xlsx --output_folder=input_file", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_file/mrtssales92-present.xlsx --pv_map=monthly_retail_pvmap.csv --statvar_dcid_remap_csv=monthly_retail_remap.csv --config_file=monthly_retail_metadata.csv --output_path=output/monthly_retail" + ], + "import_inputs": [ { - "import_name": "USMonthlyRetailSales", - "curator_emails": [ - "support@datacommons.org" - ], - "provenance_url": "https://www.census.gov/retail/", - "provenance_description": "The Advance Monthly and Monthly Retail Trade Surveys (MARTS and MRTS), the Annual Retail Trade Survey (ARTS), and the Quarterly E-Commerce Report work together to produce the most comprehensive data available on retail economic activity in the United States.", - "scripts": [ - "download.py", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_file/monthly_retail.xlsx --pv_map=monthly_retail_pvmap.csv --statvar_dcid_remap_csv=monthly_retail_remap.csv --config_file=monthly_retail_metadata.csv --output_path=output/monthly_retail" - ], - "import_inputs": [ - { - "template_mcf": "output/monthly_retail.tmcf", - "cleaned_csv": "output/monthly_retail.csv" - } - ], - "source_files": [ - "input_file/monthly_retail.xlsx" - ], - "cron_schedule": "30 05 * * 1" + "template_mcf": "output/monthly_retail.tmcf", + "cleaned_csv": "output/monthly_retail.csv" } - ] + ], + "source_files": [ + "input_file/mrtssales92-present.xlsx" + ], + "cron_schedule": "30 05 * * 1" + } + ] }