diff --git a/statvar_imports/india_rbistatedomesticproduct/manifest.json b/statvar_imports/india_rbistatedomesticproduct/manifest.json index 50323af5b7..138e7eba7f 100644 --- a/statvar_imports/india_rbistatedomesticproduct/manifest.json +++ b/statvar_imports/india_rbistatedomesticproduct/manifest.json @@ -9,18 +9,22 @@ "provenance_description": "RBI State Domestic Product data", "scripts": [ "rbi_sdp_download.py", - "../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/*.xlsx --pv_map=state_domestic_product_pv_map.csv --config_file=state_domestic_product_metadata.csv --places_resolved_csv=state_domestic_product_places_resolved.csv --output_path=output/state_domestic_product_output" + "../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/*.xlsx --pv_map=state_domestic_product_pv_map.csv --config_file=state_domestic_product_metadata.csv --places_resolved_csv=state_domestic_product_places_resolved.csv --output_path=output/state_domestic_product_output --output_counters=counters/state_domestic_product_counters.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --statvar_dcid_remap_csv=statvar_remap.csv" ], "source_files": [ - "source_files/*.xlsx" + "source_files/*.xlsx", + "counters/*.csv", + "statvar_remap.csv" ], "import_inputs": [ { "template_mcf": "output/state_domestic_product_output.tmcf", - "cleaned_csv": "output/state_domestic_product_output.csv" + "cleaned_csv": "output/state_domestic_product_output.csv", + "node_mcf": "output/*.mcf" } ], - "cron_schedule": "0 09 4,18 * *" + "cron_schedule": "0 09 4,18 * *", + "validation_config_file": "validation_config.json" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/india_rbistatedomesticproduct/statvar_remap.csv b/statvar_imports/india_rbistatedomesticproduct/statvar_remap.csv new file mode 100644 index 0000000000..f49d4a157f --- /dev/null +++ b/statvar_imports/india_rbistatedomesticproduct/statvar_remap.csv @@ -0,0 +1,2 @@ +statvar,dcid +RealValue_Amount_EconomicActivity_NetDomesticProduction_PerCapita,RealValue_Amount_EconomicActivity_NetDomesticProduction_AsAFractionOf_Count_Person diff --git a/statvar_imports/india_rbistatedomesticproduct/validation_config.json b/statvar_imports/india_rbistatedomesticproduct/validation_config.json new file mode 100644 index 0000000000..e1cc8b9e8d --- /dev/null +++ b/statvar_imports/india_rbistatedomesticproduct/validation_config.json @@ -0,0 +1,28 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted records for the entire import is within threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 0.1 + } + }, + { + "rule_id": "check_max_date_consistent", + "description": "Checks that the latest date is the same for all StatVars.", + "validator": "MAX_DATE_CONSISTENT", + "params": {} + }, + { + "rule_id": "check_max_date_freshness", + "description": "Checks that the latest date does not regress below 2025-03.", + "validator": "SQL_VALIDATOR", + "params": { + "query": "SELECT StatVar, MaxDate FROM stats", + "condition": "MaxDate >= '2025-03'" + } + } + ] +}