From 6e018af6a0c426b9e6f4ff1dc321c0d9d6515ca3 Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Mon, 24 Aug 2026 11:28:08 +0000 Subject: [PATCH 1/4] Made chnages in the manifest file --- .../india_rbistatedomesticproduct/manifest.json | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/statvar_imports/india_rbistatedomesticproduct/manifest.json b/statvar_imports/india_rbistatedomesticproduct/manifest.json index 50323af5b7..1858760552 100644 --- a/statvar_imports/india_rbistatedomesticproduct/manifest.json +++ b/statvar_imports/india_rbistatedomesticproduct/manifest.json @@ -9,15 +9,17 @@ "provenance_description": "RBI State Domestic Product data", "scripts": [ "rbi_sdp_download.py", - "../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/*.xlsx --pv_map=state_domestic_product_pv_map.csv --config_file=state_domestic_product_metadata.csv --places_resolved_csv=state_domestic_product_places_resolved.csv --output_path=output/state_domestic_product_output" + "../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/*.xlsx --pv_map=state_domestic_product_pv_map.csv --config_file=state_domestic_product_metadata.csv --places_resolved_csv=state_domestic_product_places_resolved.csv --output_path=output/state_domestic_product_output --output_counters=counters/state_domestic_product_counters.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf" ], "source_files": [ - "source_files/*.xlsx" + "source_files/*.xlsx", + "counters/*.csv" ], "import_inputs": [ { "template_mcf": "output/state_domestic_product_output.tmcf", - "cleaned_csv": "output/state_domestic_product_output.csv" + "cleaned_csv": "output/state_domestic_product_output.csv", + "node_mcf": "output/*.mcf" } ], "cron_schedule": "0 09 4,18 * *" From f2af5de53ac66bd953b997064b52fdd28a00dfe9 Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Wed, 2 Sep 2026 08:47:41 +0000 Subject: [PATCH 2/4] Added threshold for deletions --- .../india_rbistatedomesticproduct/manifest.json | 4 +++- .../validation_config.json | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 statvar_imports/india_rbistatedomesticproduct/validation_config.json diff --git a/statvar_imports/india_rbistatedomesticproduct/manifest.json b/statvar_imports/india_rbistatedomesticproduct/manifest.json index 1858760552..2e9495f5c2 100644 --- a/statvar_imports/india_rbistatedomesticproduct/manifest.json +++ b/statvar_imports/india_rbistatedomesticproduct/manifest.json @@ -22,7 +22,9 @@ "node_mcf": "output/*.mcf" } ], - "cron_schedule": "0 09 4,18 * *" + "cron_schedule": "0 09 4,18 * *", + "validation_config_file": "validation_config.json" + } ] } \ No newline at end of file diff --git a/statvar_imports/india_rbistatedomesticproduct/validation_config.json b/statvar_imports/india_rbistatedomesticproduct/validation_config.json new file mode 100644 index 0000000000..7249b48bfe --- /dev/null +++ b/statvar_imports/india_rbistatedomesticproduct/validation_config.json @@ -0,0 +1,13 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted records for the entire import is within threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 0.1 + } + } + ] +} \ No newline at end of file From 65ee4575a1ac48f85e16ff26e60de47c6b7c5295 Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Wed, 2 Sep 2026 09:18:08 +0000 Subject: [PATCH 3/4] Chnages as per CRA --- .../india_rbistatedomesticproduct/manifest.json | 3 +-- .../india_rbistatedomesticproduct/validation_config.json | 8 +++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/statvar_imports/india_rbistatedomesticproduct/manifest.json b/statvar_imports/india_rbistatedomesticproduct/manifest.json index 2e9495f5c2..a979cece7b 100644 --- a/statvar_imports/india_rbistatedomesticproduct/manifest.json +++ b/statvar_imports/india_rbistatedomesticproduct/manifest.json @@ -24,7 +24,6 @@ ], "cron_schedule": "0 09 4,18 * *", "validation_config_file": "validation_config.json" - } ] -} \ No newline at end of file +} diff --git a/statvar_imports/india_rbistatedomesticproduct/validation_config.json b/statvar_imports/india_rbistatedomesticproduct/validation_config.json index 7249b48bfe..84fe7f0edc 100644 --- a/statvar_imports/india_rbistatedomesticproduct/validation_config.json +++ b/statvar_imports/india_rbistatedomesticproduct/validation_config.json @@ -8,6 +8,12 @@ "params": { "threshold": 0.1 } + }, + { + "rule_id": "check_max_date_consistent", + "description": "Checks that the latest date is the same for all StatVars.", + "validator": "MAX_DATE_CONSISTENT", + "params": {} } ] -} \ No newline at end of file +} From ce2f4cca8e9d3a59eb788d522c050469c5c50cbf Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Thu, 3 Sep 2026 16:01:11 +0000 Subject: [PATCH 4/4] Added statvar_remap.csv --- .../india_rbistatedomesticproduct/manifest.json | 5 +++-- .../india_rbistatedomesticproduct/statvar_remap.csv | 2 ++ .../india_rbistatedomesticproduct/validation_config.json | 9 +++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 statvar_imports/india_rbistatedomesticproduct/statvar_remap.csv diff --git a/statvar_imports/india_rbistatedomesticproduct/manifest.json b/statvar_imports/india_rbistatedomesticproduct/manifest.json index a979cece7b..138e7eba7f 100644 --- a/statvar_imports/india_rbistatedomesticproduct/manifest.json +++ b/statvar_imports/india_rbistatedomesticproduct/manifest.json @@ -9,11 +9,12 @@ "provenance_description": "RBI State Domestic Product data", "scripts": [ "rbi_sdp_download.py", - "../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/*.xlsx --pv_map=state_domestic_product_pv_map.csv --config_file=state_domestic_product_metadata.csv --places_resolved_csv=state_domestic_product_places_resolved.csv --output_path=output/state_domestic_product_output --output_counters=counters/state_domestic_product_counters.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf" + "../../tools/statvar_importer/stat_var_processor.py --input_data=source_files/*.xlsx --pv_map=state_domestic_product_pv_map.csv --config_file=state_domestic_product_metadata.csv --places_resolved_csv=state_domestic_product_places_resolved.csv --output_path=output/state_domestic_product_output --output_counters=counters/state_domestic_product_counters.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --statvar_dcid_remap_csv=statvar_remap.csv" ], "source_files": [ "source_files/*.xlsx", - "counters/*.csv" + "counters/*.csv", + "statvar_remap.csv" ], "import_inputs": [ { diff --git a/statvar_imports/india_rbistatedomesticproduct/statvar_remap.csv b/statvar_imports/india_rbistatedomesticproduct/statvar_remap.csv new file mode 100644 index 0000000000..f49d4a157f --- /dev/null +++ b/statvar_imports/india_rbistatedomesticproduct/statvar_remap.csv @@ -0,0 +1,2 @@ +statvar,dcid +RealValue_Amount_EconomicActivity_NetDomesticProduction_PerCapita,RealValue_Amount_EconomicActivity_NetDomesticProduction_AsAFractionOf_Count_Person diff --git a/statvar_imports/india_rbistatedomesticproduct/validation_config.json b/statvar_imports/india_rbistatedomesticproduct/validation_config.json index 84fe7f0edc..e1cc8b9e8d 100644 --- a/statvar_imports/india_rbistatedomesticproduct/validation_config.json +++ b/statvar_imports/india_rbistatedomesticproduct/validation_config.json @@ -14,6 +14,15 @@ "description": "Checks that the latest date is the same for all StatVars.", "validator": "MAX_DATE_CONSISTENT", "params": {} + }, + { + "rule_id": "check_max_date_freshness", + "description": "Checks that the latest date does not regress below 2025-03.", + "validator": "SQL_VALIDATOR", + "params": { + "query": "SELECT StatVar, MaxDate FROM stats", + "condition": "MaxDate >= '2025-03'" + } } ] }