Fix download script, add validation config and golden data for NYU TennesseDiabetes

statvar_imports/nyu_diabetes/tennessee/download.py

@@ -15,6 +15,9 @@
 
 from datetime import date
 import os
+import requests
+from urllib.parse import urlparse
+from tqdm import tqdm  
 from pathlib import Path
 import re
 from urllib.parse import urlparse
@@ -42,14 +45,18 @@
 INPUT_DIR = os.path.join(script_dir, "input_files")
 Path(INPUT_DIR).mkdir(parents=True, exist_ok=True)
 
-@retry(tries=3, delay=5, backoff=2)
 def retry_method(url, headers=None):
-    response = requests.get(url, headers=headers, timeout=120)
+    if headers is None:
+        headers = {}
+    if "User-Agent" not in headers:
+        headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+    response = requests.get(url, headers=headers, timeout=120, stream=True)
     response.raise_for_status()
     return response
 
 def download_files(url_list, save_folder):
     os.makedirs(os.path.join(save_folder), exist_ok=True)
+    downloaded_count = 0
 
     for url in url_list:
         try:
@@ -74,11 +81,15 @@ def download_files(url_list, save_folder):
             logging.info(
                 f"Completed download: source={url}, destination={file_path}, size_bytes={file_size}"
             )
+            downloaded_count += 1
         except Exception as e:
             logging.error(
                 f"Download failed: source={url}, destination={file_path}, error={e}"
             )
 
+    if url_list and downloaded_count == 0:
+        raise RuntimeError("No files were successfully downloaded.")
+
 
 def download_files_from_gcs(url_list, save_folder):
     os.makedirs(save_folder, exist_ok=True)

statvar_imports/nyu_diabetes/tennessee/golden_data/golden_observations.csv

@@ -0,0 +1,71 @@
+"observationAbout"
+"geoId/47"
+"geoId/47001"
+"geoId/47009"
+"geoId/47011"
+"geoId/47013"
+"geoId/47019"
+"geoId/47021"
+"geoId/47023"
+"geoId/47029"
+"geoId/47031"
+"geoId/47035"
+"geoId/47037"
+"geoId/47041"
+"geoId/47045"
+"geoId/47047"
+"geoId/47049"
+"geoId/47051"
+"geoId/47053"
+"geoId/47055"
+"geoId/47059"
+"geoId/47063"
+"geoId/47065"
+"geoId/47069"
+"geoId/47071"
+"geoId/47073"
+"geoId/47075"
+"geoId/47077"
+"geoId/47079"
+"geoId/47089"
+"geoId/47093"
+"geoId/47097"
+"geoId/47099"
+"geoId/47103"
+"geoId/47105"
+"geoId/47107"
+"geoId/47109"
+"geoId/47113"
+"geoId/47115"
+"geoId/47119"
+"geoId/47123"
+"geoId/47125"
+"geoId/47129"
+"geoId/47131"
+"geoId/47141"
+"geoId/47143"
+"geoId/47145"
+"geoId/47147"
+"geoId/47149"
+"geoId/47151"
+"geoId/47155"
+"geoId/47157"
+"geoId/47159"
+"geoId/47163"
+"geoId/47165"
+"geoId/47167"
+"geoId/47179"
+"geoId/47183"
+"geoId/47187"
+"geoId/47189"
+"geoId/47003"
+"geoId/47043"
+"geoId/47133"
+"geoId/47139"
+"geoId/47177"
+"geoId/47181"
+"geoId/47185"
+"geoId/47081"
+"geoId/47117"
+"geoId/47173"
+"geoId/47057"

statvar_imports/nyu_diabetes/tennessee/manifest.json

@@ -6,18 +6,22 @@
       "provenance_url": "https://www.tn.gov/health/health-program-areas/statistics/health-data/death-statistics.html",
       "provenance_description": "This import utilizes official diabetes mortality data from the Tennessee Department of Health, detailing county-level death statistics.",
       "scripts": ["download.py",
-        "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.xlsx --pv_map=pvmap.csv --config_file=metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=place_resolver.csv --output_path=output_files/tennessee_output"
+        "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/*.xlsx --pv_map=pvmap.csv --config_file=metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=place_resolver.csv --output_path=output_files/tennessee_output --output_counters=counters/tennessee_output_counters.csv"
       ] ,
       "source_files": [
-        "input_files/*.xlsx"
+        "input_files/*.xlsx",
+        "golden_data/*.csv",
+        "counters/*.csv"
       ],
       "import_inputs": [
         {
           "template_mcf": "output_files/tennessee_output.tmcf",
-          "cleaned_csv": "output_files/tennessee_output.csv"
+          "cleaned_csv": "output_files/tennessee_output.csv",
+          "node_mcf": "output/tennessee_output.mcf"
         }
       ],
-      "cron_schedule": "0 05 * * 5"
+      "cron_schedule": "0 05 * * 5",
+      "validation_config_file": "validation_config.json"
     }
   ]
 }

statvar_imports/nyu_diabetes/tennessee/validation_config.json

@@ -0,0 +1,22 @@
+{
+    "schema_version": "1.0",
+    "rules": [
+        {
+            "rule_id": "check_deleted_records_percent",
+            "description": "Checks that the percentage of deleted points is within the threshold.",
+            "validator": "DELETED_RECORDS_PERCENT",
+            "params": {
+                "threshold": 0.1
+            }
+        },
+        {
+            "rule_id": "check_goldens_output_csv",
+            "description": "Verifies the generated output CSV data matches established critical golden records",
+            "validator": "GOLDENS_CHECK",
+            "params": {
+                "golden_files": "../../../../golden_data/golden_observations.csv",
+                "input_files": "../../../../output_files/tennessee_output.csv"
+            }
+        }
+    ]
+}