Fix Pandas compatibility, DCID collisions, and validation config for NCES_SchoolDistrict and NCES_PublicSchool

scripts/us_nces/common/prop_conf.py

@@ -123,17 +123,20 @@
     "School Administrative Support Staff": "Faculty",
     "Student Support Services Staff": "Faculty",
     "School Psychologist": "Faculty",
-    "Other Support Services Staff": "Faculty"
+    "Other Support Services Staff": "Faculty",
+    "Total Staff": "Faculty"
 }
 # One specific column comes under school grade property.
 _SCHOOL_GRADE_PROP = {"Ungraded Students": "NCESUngradedClasses"}
 # melting the columns based on sv_name column.
 MELT_VAR_COL = "sv_name"
 
+def _PV_FORMAT(pv):
+    """Formats property-value pairs for MCF nodes; modified based on column."""
+    t = tuple(pv)
+    return f'"{t[0]}": "dcs:{t[1]}"' if 'None' not in str(t[1]) else ""
+
 # pylint:disable=unnecessary-lambda-assignment
-# Creating property pattern and the pattern is modified if required based on column.
-_PV_FORMAT = lambda prop_val: f'"{prop_val[0]}": "dcs:{prop_val[1]}"' \
-                        if 'None' not in prop_val[1] else ""
 _UPDATE_MEASUREMENT_DENO = lambda prop: _DENOMINATOR_PROP.get(prop, prop)
 _UPDATE_POPULATION_TYPE = lambda prop: _POPULATION_PROP.get(prop, "Student")
 _UPDATE_GRADE_LEVEL = lambda prop: _SCHOOL_GRADE_PROP.get(prop, prop)
@@ -167,7 +170,7 @@
                  r")")
 
 _SCHOOL_GRADE_PATTERN = (r"("
-                         r"Grade \d{,2}"
+                         r"Grade \d{1,2}"
                          r"|"
                          r"Prekindergarten and Kindergarten"
                          r"|"
@@ -236,6 +239,8 @@
                             r"School Psychologist"
                             r"|"
                             r"Other Support Services Staff"
+                            r"|"
+                            r"Total Staff"
                             r")")
 
 _GENDER_PATTERN = (r"("

scripts/us_nces/common/replacement_functions.py

@@ -372,7 +372,7 @@
     "Phone Number": "PhoneNumber"
 }
 
-_GENDER = {"female": "Female", "male": "Male"}
+_GENDER = {r"\bfemale\b": "Female", r"\bmale\b": "Male"}
 
 _LOCALE = {
     '13-City: Small': "NCES_CitySmall",

scripts/us_nces/common/us_education.py

@@ -437,8 +437,7 @@ def add_leading_zero(text):
             'dc_api_batch_size': 200,
             'dc_api_retries': 3,
             'dc_api_retry_sec': 5,
-            'dc_api_use_cache': False,
-            'dc_api_root': None
+            'dc_api_use_cache': False
         }
         # Passing the list through API call for checking its existance.
         dcid_check_zip = dc_api_is_defined_dcid(zip_list, config)
@@ -582,8 +581,7 @@ def _transform_public_place(self):
             'dc_api_batch_size': 200,
             'dc_api_retries': 3,
             'dc_api_retry_sec': 5,
-            'dc_api_use_cache': False,
-            'dc_api_root': None
+            'dc_api_use_cache': False
         }
 
         # Only call API if lists are not empty
@@ -747,8 +745,7 @@ def _transform_district_place(self):
             'dc_api_batch_size': 200,
             'dc_api_retries': 3,
             'dc_api_retry_sec': 5,
-            'dc_api_use_cache': False,
-            'dc_api_root': None
+            'dc_api_use_cache': False
         }
 
         # Only call API if we actually have states to check
@@ -942,20 +939,12 @@ def _parse_file(self, raw_df: pd.DataFrame) -> pd.DataFrame:
                 if df_dist_tmp.shape[0] > 0:
                     #If current year data is already there in main df - merge the columns
 
-                    # Remove common columns excluding key columns so as to remove duplicates.
-                    rem_common_columns = list(
-                        set(df_place.columns.to_list()) -
-                        set(self._key_col_place))
-
-                    df_dist_tmp = df_dist_tmp.loc[:, ~df_dist_tmp.columns.
-                                                  isin(rem_common_columns)]
-
-                    # Merge the different files columns of same year with key columns
-
-                    df_dist_tmp = pd.merge(df_dist_tmp,
-                                           df_place,
-                                           how="outer",
-                                           on=self._key_col_place)
+                    # Combine place columns of the same year with key columns
+                    df_dist_tmp = (
+                        df_dist_tmp.set_index(self._key_col_place)
+                        .combine_first(df_place.set_index(self._key_col_place))
+                        .reset_index()
+                    )
 
                 else:
                     # The current year data not present in final df
@@ -1052,11 +1041,8 @@ def generate_csv(self) -> pd.DataFrame:
                                                     SV_PROP_ORDER, FORM_STATVAR)
                     df_parsed = self._generate_stat_var_and_mcf(
                         df_parsed, SV_PROP_ORDER)
-                    for col in df_parsed.columns.values.tolist():
-                        df_parsed[col] = df_parsed[col].astype(
-                            'str').str.replace("FeMale", "Female")
-                # Adding new columns scaling_factor:100 and unit:dcs:Percent
-                #  wherever the SV is Percent.
+                    # Adding new columns scaling_factor:100 and unit:dcs:Percent
+                    # wherever the SV is Percent.
                     df_parsed["scaling_factor"] = np.where(
                         df_parsed["sv_name"].str.contains("Percent"), '100', '')
                     df_parsed["unit"] = np.where(

scripts/us_nces/demographics/public_school/manifest.json

@@ -17,17 +17,19 @@
             "import_inputs": [
                 {
                     "template_mcf": "gcs_folder/output_files/us_nces_demographics_public_school.tmcf",
-                    "cleaned_csv": "gcs_folder/output_files/us_nces_demographics_public_school.csv"
+                    "cleaned_csv": "gcs_folder/output_files/us_nces_demographics_public_school.csv",
+                    "node_mcf": "gcs_folder/output_files/us_nces_demographics_public_school.mcf"
                 },
                 {
                     "template_mcf": "gcs_folder/output_place/us_nces_demographics_public_place.tmcf",
                     "cleaned_csv": "gcs_folder/output_place/us_nces_demographics_public_place.csv"
                 }
             ],
             "cron_schedule": "30 3 2 3,6,9,12 *",
+            "validation_config_file": "validation_config.json",
             "resource_limits": {
                 "cpu": 32,
-                "memory": 512,
+                "memory": 1024,
                 "disk": 300
             }
         }