Migrate compatible imports to use download_util_script.py #2188
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| @@ -1,33 +1,33 @@ | ||||||||||||||
| { | ||||||||||||||
| "import_specifications": [ | ||||||||||||||
| { | ||||||||||||||
| "import_name": "BEA_USStatesQuarterlyGDP", | ||||||||||||||
| "curator_emails": [ | ||||||||||||||
| "support@datacommons.org" | ||||||||||||||
| ], | ||||||||||||||
| "provenance_url": "https://www.bea.gov/data/gdp/gdp-state", | ||||||||||||||
| "provenance_description": "Quarterly data on inflation-adjusted GDP at a state level in the United States from the US Bureau of Economic Analysis.", | ||||||||||||||
| "scripts": [ | ||||||||||||||
| "../../../util/download_util_script.py --download_url=https://apps.bea.gov/regional/zip/SQGDP.zip --output_folder=input_files --unzip=True", | ||||||||||||||
| "preprocess_csv.py", | ||||||||||||||
| "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/preprocessed_data.csv --pv_map=bea_gdp_pv_map.csv --config_file=bea_gdp_metadata.csv --places_resolved_csv=bea_gdp_place_map.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/BEA_GDP_output" | ||||||||||||||
| ], | ||||||||||||||
| "source_files": [ | ||||||||||||||
| "input_data/*.csv" | ||||||||||||||
| ], | ||||||||||||||
|
Comment on lines
+15
to
+17
Contributor
There was a problem hiding this comment. The
Suggested change
|
||||||||||||||
| "import_inputs": [ | ||||||||||||||
| { | ||||||||||||||
| "cleaned_csv": "states_gdp.csv", | ||||||||||||||
| "template_mcf": "states_gdp.tmcf", | ||||||||||||||
| "node_mcf": "states_gdp_statvars.mcf" | ||||||||||||||
| }, | ||||||||||||||
| { | ||||||||||||||
| "cleaned_csv": "states_industry_gdp.csv", | ||||||||||||||
| "node_mcf": "states_gdp_industry_statvars.mcf", | ||||||||||||||
| "template_mcf": "states_industry_gdp.tmcf" | ||||||||||||||
| } | ||||||||||||||
| ], | ||||||||||||||
|
Comment on lines
+18
to
+29
Contributor
There was a problem hiding this comment. There is a critical mismatch between the "import_inputs": [
{
"cleaned_csv": "output_files/BEA_GDP_output.csv",
"template_mcf": "output_files/BEA_GDP_output.tmcf",
"node_mcf": "output_files/BEA_GDP_output_stat_vars.mcf"
}
], |
||||||||||||||
| "cron_schedule": "30 10 30 1,4,7,10,12 *" | ||||||||||||||
| } | ||||||||||||||
| ] | ||||||||||||||
| } | ||||||||||||||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| @@ -1,26 +1,27 @@ | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| { | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| "import_specifications": [ | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| { | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| "import_name": "US_CCD_Enrollment", | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| "curator_emails": [ | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| "support@datacommons.org" | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| ], | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| "provenance_url": "https://nces.ed.gov/programs/digest/d24/tables/dt24_203.65.asp", | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| "provenance_description": "Enrollment in public elementary and secondary schools, by level, grade, and race/ethnicity.", | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| "scripts": [ | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| "../../../util/download_util_script.py --download_url=https://nces.ed.gov/programs/digest/d24/tables/xls/tabn203.65.xlsx --output_folder=source_files", | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| "preprocess.py", | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.csv --pv_map=pvmap.csv --config_file=metadata.csv --places_resolved_csv=place_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/enrollment_output" | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| ], | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| "source_files": [ | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| "input_files/ccd_enrollment_input.csv" | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| ], | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
Comment on lines
+15
to
+17
Contributor
There was a problem hiding this comment. The
Suggested change
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| "import_inputs": [ | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| { | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| "template_mcf": "output/CCD_Enrollment_output.tmcf", | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| "cleaned_csv": "output/CCD_Enrollment_output.csv" | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| ], | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
Comment on lines
+18
to
+23
Contributor
There was a problem hiding this comment. There is a critical mismatch between the
Suggested change
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| "cron_schedule": "0 06 * * 5" | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| ] | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,26 +1,26 @@ | ||
| { | ||
| "import_specifications": [ | ||
| { | ||
| "import_name": "UAE_Population", | ||
| "curator_emails": [ | ||
| "support@datacommons.org" | ||
| ], | ||
| "provenance_url": "https://bayanat.ae/api/DatasetResources/DownloadSingle?resourceID=FT1CGOdcLYt6KdUJnFHWwOdTWpeao-LX539QkMLeAKA&fileName=UAE%20Population%20By%20Emirates%20Nationality%20and%20gender", | ||
| "provenance_description": "UAE_Population statical data", | ||
| "scripts": [ | ||
| "../../../util/download_util_script.py --download_url=https://bayanat.ae/api/DatasetResources/DownloadSingle?resourceID=FT1CGOdcLYt6KdUJnFHWwOdTWpeao-LX539QkMLeAKA&fileName=UAE%20Population%20By%20Emirates%20Nationality%20and%20gender --output_folder=input", | ||
| "../../../tools/statvar_importer/stat_var_processor.py --input_data=input/uae_populationbyemiratesnationalityandgender.xlsx --pv_map=uae_population_pvmap.csv --config_file=uae_population_metadata.csv --places_resolved_csv=uae_population_places_resolved_csv.csv --output_path=output/uae_population_output" | ||
| ], | ||
| "source_files": [ | ||
| "input/uae_populationbyemiratesnationalityandgender.xlsx" | ||
| ], | ||
| "import_inputs": [ | ||
| { | ||
| "template_mcf": "output/uae_population_output.tmcf", | ||
| "cleaned_csv": "output/uae_population_output.csv" | ||
| } | ||
| ], | ||
| "cron_schedule": "0 05 * * 1" | ||
| } | ||
| ] | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,27 +1,27 @@ | ||
| { | ||
| "import_specifications": [ | ||
| { | ||
| "import_name": "UNData", | ||
| "curator_emails": [ | ||
| "support@datacommons.org" | ||
| ], | ||
| "provenance_url": "https://data.un.org/Data.aspx?q=city+population&d=POP&f=tableCode%3a240", | ||
| "provenance_description": "Population data for countries, capital cities, urban and rural areas not covered by other sources. Outliers were removed from the output", | ||
| "scripts": [ | ||
| "../../util/download_util_script.py --download_url=https://data.un.org/Handlers/DownloadHandler.ashx?DataFilter=tableCode:240&DataMartId=POP&Format=csv&c=2,3,6,8,10,12,14,16,17,18&s=_countryEnglishNameOrderBy:asc,refYear:desc,areaCode:asc --output_folder=input_file --unzip=True", | ||
| "../../tools/statvar_importer/stat_var_processor.py --input_data=input_file/*.csv --pv_map=UNData_pvmap.csv --config_file=UNData_metadata.csv --places_resolved_csv=UNData_places_resolved_csv.csv --output_path=output/UNData", | ||
| "../../tools/statvar_importer/filter_data_outliers.py --filter_data_input=output/UNData.csv --filter_data_output=output/filter_undata.csv --filter_data_min_value=2 --filter_data_max_yearly_change_ratio=0.5 --filter_data_max_change_ratio=1" | ||
| ], | ||
| "source_files": [ | ||
| "input_file/*.csv" | ||
| ], | ||
| "import_inputs": [ | ||
| { | ||
| "template_mcf": "output/UNData.tmcf", | ||
| "cleaned_csv": "output/filter_undata.csv" | ||
| } | ||
| ], | ||
| "cron_schedule": "0 05 * * 3" | ||
| } | ||
| ] | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,26 +1,26 @@ | ||
| { | ||
| "import_specifications": [ | ||
| { | ||
| "import_name": "USMonthlyRetailSales", | ||
| "curator_emails": [ | ||
| "support@datacommons.org" | ||
| ], | ||
| "provenance_url": "https://www.census.gov/retail/", | ||
| "provenance_description": "The Advance Monthly and Monthly Retail Trade Surveys (MARTS and MRTS), the Annual Retail Trade Survey (ARTS), and the Quarterly E-Commerce Report work together to produce the most comprehensive data available on retail economic activity in the United States.", | ||
| "scripts": [ | ||
| "../../../util/download_util_script.py --download_url=https://www.census.gov/retail/mrts/www/mrtssales92-present.xlsx --output_folder=input_file", | ||
| "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_file/mrtssales92-present.xlsx --pv_map=monthly_retail_pvmap.csv --statvar_dcid_remap_csv=monthly_retail_remap.csv --config_file=monthly_retail_metadata.csv --output_path=output/monthly_retail" | ||
| ], | ||
| "import_inputs": [ | ||
| { | ||
| "template_mcf": "output/monthly_retail.tmcf", | ||
| "cleaned_csv": "output/monthly_retail.csv" | ||
| } | ||
| ], | ||
| "source_files": [ | ||
| "input_file/mrtssales92-present.xlsx" | ||
| ], | ||
| "cron_schedule": "30 05 * * 1" | ||
| } | ||
| ] | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This configuration change (
use_autopush_dc_api: bool = False) is unrelated to the PR's objective of migrating compatible imports to usedownload_util_script.py. Unrelated changes should be kept out of this PR to keep it focused and avoid accidental environment switches.References