diff --git a/CHANGELOG b/CHANGELOG index 898cf584..cea6a207 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,14 @@ +2026-08-22 v10.2.0 +- Fix lat/lon axis flip for shapefile sources with an explicit `srs` tag https://github.com/openaddresses/batch-machine/pull/113 +- Extract nested zip files regardless of the conform `file` filter, cap zip entry size and nesting depth as a zip bomb guard, and skip recursing into a nested zip once the filter is already satisfied https://github.com/openaddresses/batch-machine/pull/112 +- Add KML support to the conform pipeline https://github.com/openaddresses/batch-machine/pull/111 +- Linearize curve geometries (e.g. `MULTISURFACE`) before exporting to WKT, fixing GDB parcel/building sources that error out https://github.com/openaddresses/batch-machine/pull/110 +- Fix `regexp` function's `replace` mode returning the unmatched field unchanged instead of an empty string https://github.com/openaddresses/batch-machine/pull/109 +- Collect the union of feature property keys before writing `geojson_source_to_csv` output, fixing a crash on GeoJSON sources with non-uniform feature properties https://github.com/openaddresses/batch-machine/pull/108 + +2026-08-13 v10.1.0 +- Support custom HTTP request headers for a source via `request.headers`, including on the file-extension pre-flight request (needed for downloads gated on Referer/etc.) + 2026-03-27 v10.0.0 - Upgrade base Docker image from GDAL 3.7.1 to 3.11.0 https://github.com/openaddresses/batch-machine/pull/99 - Update SSL CA certificates to fix download failures https://github.com/openaddresses/batch-machine/pull/98 diff --git a/Dockerfile b/Dockerfile index 47f27544..b6abe3e4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/osgeo/gdal:alpine-normal-3.11.0 +FROM ghcr.io/osgeo/gdal:alpine-normal-3.11.0@sha256:edf2793e0f1ceb74ab12a1d85bd3404b541a113720fbc1e032613e7df2774f7c RUN apk add --no-cache nodejs yarn git python3-dev py3-pip \ make sqlite-dev zlib-dev geos-dev \ diff --git a/README.md b/README.md index 6dcc683e..73b281b4 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ Supported layer types are `addresses`, `parcels`, `buildings`, and `centerlines` Review https://github.com/openaddresses/openaddresses/blob/master/CONTRIBUTING.md for input json syntax. -Supported conform formats include `shapefile`, `geojson`, `csv`, `xml`, `gdb`, and `gpkg`. +Supported conform formats include `shapefile`, `geojson`, `csv`, `xml`, `gdb`, `gpkg`, and `kml` (2D Point placemarks with simple `ExtendedData` attributes are verified; other geometry types and `Schema`-typed attributes go through the same GDAL driver but are untested; altitude/3D coordinates and KMZ are not supported). ## Geocodio fork notes @@ -77,3 +77,9 @@ diverge from upstream and must be preserved across upstream merges: conformed output row, so named-but-unnumbered properties stay individually addressable. It is a geocodio-specific addition and is not part of the upstream OpenAddresses schema. +- **Non-finite geometry is never written out** — ESRI services report a null + point geometry as the string `"NaN"`, which reached the output as + `"coordinates": [NaN, NaN]`. Invalid GeoJSON per RFC 7946, and unparseable by + strict decoders. `openaddr/cache.py` now skips those features on download and + `openaddr/conform.py` treats any non-finite WKT coordinate as no geometry, + with `allow_nan=False` on the output writer as a backstop. diff --git a/openaddr/VERSION b/openaddr/VERSION index a13e7b9c..2bd6f7e3 100644 --- a/openaddr/VERSION +++ b/openaddr/VERSION @@ -1 +1 @@ -10.0.0 +10.2.0 diff --git a/openaddr/__init__.py b/openaddr/__init__.py index 0014f27e..bdb18099 100644 --- a/openaddr/__init__.py +++ b/openaddr/__init__.py @@ -75,8 +75,10 @@ def cache(source_config, destdir, extras): source_urls = [source_urls] protocol_string = source_config.data_source.get('protocol') + request_settings = source_config.data_source.get('request') or {} + source_headers = request_settings.get('headers') or {} - task = DownloadTask.from_protocol_string(protocol_string, source_config) + task = DownloadTask.from_protocol_string(protocol_string, source_config, headers=source_headers) downloaded_files = task.download(source_urls, workdir, source_config) # FIXME: I wrote the download stuff to assume multiple files because @@ -128,6 +130,10 @@ def conform(source_config, destdir, extras, disable_centroids=False): if not isinstance(source_urls, list): source_urls = [source_urls] + # source_config.data_source['request'] is intentionally not passed here: + # this re-downloads from the OA-owned cache artifact (S3), not the + # contributor's original host, so contributor-supplied headers don't + # apply. task1 = URLDownloadTask(source_config.data_source_name) downloaded_path = task1.download(source_urls, workdir, source_config) _L.info("Downloaded to %s", downloaded_path) diff --git a/openaddr/cache.py b/openaddr/cache.py index d1ce4f70..c577aadb 100644 --- a/openaddr/cache.py +++ b/openaddr/cache.py @@ -47,6 +47,13 @@ def traverse(item): else: yield item +def coordinate_is_usable(value): + "Test a single coordinate for a finite number; ESRI reports null geometry as the string \"NaN\"" + try: + return math.isfinite(float(value)) + except (TypeError, ValueError): + return False + def request(method, url, **kwargs): if urlparse(url).scheme == 'ftp': if method != 'GET': @@ -130,22 +137,23 @@ def __init__(self, source_prefix, params={}, headers={}): @classmethod - def from_protocol_string(clz, protocol_string, source_prefix=None): + def from_protocol_string(clz, protocol_string, source_prefix=None, headers=None): + headers = headers or {} if protocol_string.lower() == 'http': - return URLDownloadTask(source_prefix) + return URLDownloadTask(source_prefix, headers=headers) elif protocol_string.lower() == 'file': - return URLDownloadTask(source_prefix) + return URLDownloadTask(source_prefix, headers=headers) elif protocol_string.lower() == 'ftp': - return URLDownloadTask(source_prefix) + return URLDownloadTask(source_prefix, headers=headers) elif protocol_string.lower() == 'esri': - return EsriRestDownloadTask(source_prefix) + return EsriRestDownloadTask(source_prefix, headers=headers) else: raise KeyError("I don't know how to extract for protocol {}".format(protocol_string)) def download(self, source_urls, workdir, source_config): raise NotImplementedError() -def guess_url_file_extension(url): +def guess_url_file_extension(url, headers=None): ''' Get a filename extension for a URL using various hints. ''' scheme, _, path, _, query, _ = urlparse(url) @@ -172,7 +180,7 @@ def guess_url_file_extension(url): # Get a dictionary of headers and a few bytes of content from the URL. # if scheme in ('http', 'https'): - response = request('GET', url, stream=True) + response = request('GET', url, headers=headers or {}, stream=True) handle, file = mkstemp() for chunk in response.iter_content(chunk_size=8192): @@ -256,7 +264,7 @@ def get_file_path(self, url, dir_path): hash = sha1((host + path_base).encode('utf-8')) name_base = u'{}-{}'.format(self.source_prefix, hash.hexdigest()[:8]) - path_ext = guess_url_file_extension(url) + path_ext = guess_url_file_extension(url, self.headers) _L.debug(u'Guessed {}{} for {}'.format(name_base, path_ext, url)) return os.path.join(dir_path, name_base + path_ext) @@ -391,7 +399,7 @@ def download(self, source_urls, workdir, source_config): _L.debug("File exists %s", file_path) continue - downloader = EsriDumper(source_url, parent_logger=_L, timeout=300) + downloader = EsriDumper(source_url, parent_logger=_L, timeout=300, extra_headers=self.headers) metadata = downloader.get_metadata() @@ -439,8 +447,8 @@ def download(self, source_urls, workdir, source_config): if not geom: raise TypeError("No geometry parsed") - if any((isinstance(g, float) and math.isnan(g)) for g in traverse(geom)): - raise TypeError("Geometry has NaN coordinates") + if any(not coordinate_is_usable(c) for c in traverse(geom.get('coordinates'))): + raise TypeError("Geometry has non-finite coordinates") shp = shape(geom) row[GEOM_FIELDNAME] = shp.wkt diff --git a/openaddr/conform.py b/openaddr/conform.py index cb12885b..993834e7 100644 --- a/openaddr/conform.py +++ b/openaddr/conform.py @@ -46,6 +46,10 @@ def gdal_error_handler(err_class, err_num, err_msg): # We add columns to the extracted CSV with our own data with these names. GEOM_FIELDNAME = 'oa:geom' +# WKT coordinates that json.dumps would write as NaN or Infinity, which no strict +# GeoJSON parser will accept. ESRI services report a null point geometry this way. +NONFINITE_COORDINATE_PATTERN = re.compile(r'\b(nan|inf(inity)?)\b', re.IGNORECASE) + ADDRESSES_SCHEMA = [ 'hash', 'number', 'street', 'unit', 'building_name', 'city', 'district', 'region', 'postcode', 'id', 'accuracy' ] BUILDINGS_SCHEMA = [ 'hash', 'height', 'levels'] PARCELS_SCHEMA = [ 'hash', 'pid' ] @@ -203,26 +207,56 @@ def is_in(path, names): return False class ZipDecompressTask(DecompressionTask): + # Recursing into nested zips (see #35/#112) means a maliciously or + # accidentally crafted zip bomb could otherwise fill a job's disk before + # any per-entry file type filtering applies. Cap the declared + # (uncompressed) size of any single entry and how many nested zips deep + # we'll recurse, so worst case is bounded and the job fails loudly + # instead of exhausting disk. + MAX_ZIP_ENTRY_BYTES = 2 * 1024 ** 3 # 2GB + MAX_NESTED_ZIP_DEPTH = 10 + def decompress(self, source_paths, workdir, filenames): output_files = [] expand_path = os.path.join(workdir, UNZIPPED_DIRNAME) mkdirsp(expand_path) # Extract contents of zip file into expand_path directory. + found = set() for source_path in source_paths: - self._extract_zip(source_path, expand_path, filenames) + found |= self._extract_zip(source_path, expand_path, filenames) + + def fully_satisfied(): + # Only short-circuit when there's an explicit file filter AND + # we can prove every name it asked for has actually been + # extracted - if filenames is empty (caller wants everything) + # or matching is inexact (e.g. a directory-style entry in + # `filenames` that `is_in()` matched but doesn't literally + # equal), this stays False and we fall back to full recursion, + # same as before this optimization existed. + return bool(filenames) and set(filenames) <= found # Recursively extract nested zip files, but fail if more than one zip - # appears at the same directory level. + # appears at the same directory level. Skip recursing at all once the + # requested file(s) are already found - there's no reason to keep + # opening nested zips we don't need, which also limits exposure to a + # zip bomb hiding deeper in the chain than what was asked for. processed = set() - pending = list(self._find_single_zips(expand_path)) + pending = [] if fully_satisfied() else list(self._find_single_zips(expand_path)) while pending: + if len(processed) >= self.MAX_NESTED_ZIP_DEPTH: + raise DecompressionError( + "Refusing to recurse more than {} nested zip files deep - possible zip bomb" + .format(self.MAX_NESTED_ZIP_DEPTH) + ) zip_path = pending.pop() if zip_path in processed: continue processed.add(zip_path) - self._extract_zip(zip_path, os.path.dirname(zip_path), filenames) + found |= self._extract_zip(zip_path, os.path.dirname(zip_path), filenames) + if fully_satisfied(): + break pending.extend(self._find_single_zips(os.path.dirname(zip_path))) # Collect names of directories and files in expand_path directory. @@ -232,20 +266,54 @@ def decompress(self, source_paths, workdir, filenames): output_files.append(os.path.join(dirpath, dirname)) _L.debug("Expanded directory {}".format(output_files[-1])) for filename in filenames: + if filename.lower().endswith('.zip'): + # A zip left un-recursed-into (e.g. because the request + # was already satisfied without it, or filtered out at + # its own directory level) isn't a usable source file. + continue output_files.append(os.path.join(dirpath, filename)) _L.debug("Expanded file {}".format(output_files[-1])) return output_files def _extract_zip(self, source_path, expand_path, filenames): + ''' Extract matching entries, returning the subset of `filenames` + (lower-cased) that were found by exact name - used by + decompress() to tell whether it's safe to stop recursing into + further nested zips. + ''' + found = set() with ZipFile(source_path, 'r') as z: - for name in z.namelist(): + for zinfo in z.infolist(): + name = zinfo.filename + + # Check the declared (uncompressed) size from the zip's + # central directory before extracting anything - a bomb's + # compressed size can be tiny, but its declared size isn't. + if zinfo.file_size > self.MAX_ZIP_ENTRY_BYTES: + raise DecompressionError( + "Refusing to extract {} - declared size {} bytes exceeds {} byte limit, possible zip bomb" + .format(name, zinfo.file_size, self.MAX_ZIP_ENTRY_BYTES) + ) + + # Nested zip files are always extracted regardless of the + # filenames filter, since the requested file may be inside + # one of them. The filter is re-applied when that nested + # zip is itself extracted. + if name.lower().endswith('.zip'): + z.extract(zinfo, expand_path) + continue + if len(filenames) and not is_in(name, filenames): # Download only the named file, if any. _L.debug("Skipped file {}".format(name)) continue + if len(filenames) and name.lower() in filenames: + found.add(name.lower()) + z.extract(name, expand_path) + return found def _find_single_zips(self, root_path): zip_paths = [] @@ -427,6 +495,28 @@ def find_source_path(data_source, source_paths): return c _L.warning("Source names file %s but could not find it", source_file_name) return None + elif format_string == "kml": + candidates = [] + for fn in source_paths: + basename, ext = os.path.splitext(fn) + if ext.lower() == ".kml": + candidates.append(fn) + if len(candidates) == 0: + _L.warning("No KML found in %s", source_paths) + return None + elif len(candidates) == 1: + _L.debug("Selected %s for source", candidates[0]) + return candidates[0] + else: + if "file" not in conform: + _L.warning("Multiple KML files found, but source has no file attribute.") + return None + source_file_name = conform["file"] + for c in candidates: + if source_file_name == os.path.basename(c): + return c + _L.warning("Source names file %s but could not find it", source_file_name) + return None elif format_string == "xml": # Return file if it's specified, else return the first .gml file we find if "file" in conform: @@ -550,6 +640,10 @@ def ogr_source_to_csv(source_config, source_path, dest_path, disable_centroids=F _L.debug("SRS tag found specifying %s", srs) inSpatialRef = osr.SpatialReference() inSpatialRef.ImportFromEPSG(int(srs[5:])) + + if int(osgeo.__version__[0]) >= 3: + # GDAL 3 changes axis order: https://github.com/OSGeo/gdal/issues/1546 + inSpatialRef.SetAxisMappingStrategy(osgeo.osr.OAMS_TRADITIONAL_GIS_ORDER) else: # OGR is capable of doing more than EPSG, but so far we don't need it. raise Exception("Bad SRS. Can only handle EPSG, the SRS tag is %s", srs) @@ -606,6 +700,14 @@ def ogr_source_to_csv(source_config, source_path, dest_path, disable_centroids=F if geom is not None: geom.Transform(coordTransform) + if geom.HasCurveGeometry(): + # Some sources (notably file geodatabases with curved + # parcel/building boundaries) contain curve geometry types + # like MULTISURFACE or CURVEPOLYGON. Shapely/GEOS can't + # parse those WKT types, so linearize them first. + # https://github.com/openaddresses/batch-machine/issues/62 + geom = geom.GetLinearGeometry() + if source_config.layer == "addresses" and not disable_centroids: # For Addresses - Calculate the centroid on surface of the geometry and write it as X and Y columns try: @@ -716,18 +818,26 @@ def csv_source_to_csv(source_config, source_path, dest_path, disable_centroids=F def geojson_source_to_csv(source_config, source_path, dest_path, disable_centroids=False): ''' ''' + # Not every feature shares the same set of properties, so make a first + # pass to collect the union of every feature's property keys (in + # first-seen order) before opening the CSV writer. + out_fieldnames = [] + seen_fieldnames = set() + with open(source_path) as file: + for feature in stream_geojson(file): + for key in feature['properties'].keys(): + if key not in seen_fieldnames: + seen_fieldnames.add(key) + out_fieldnames.append(key) + out_fieldnames.append(GEOM_FIELDNAME) + # For every row in the source GeoJSON with open(source_path) as file: # Write the extracted CSV file with open(dest_path, 'w', encoding='utf-8') as dest_fp: - writer = None + writer = csv.DictWriter(dest_fp, out_fieldnames) + writer.writeheader() for (row_number, feature) in enumerate(stream_geojson(file)): - if writer is None: - out_fieldnames = list(feature['properties'].keys()) - out_fieldnames.append(GEOM_FIELDNAME) - writer = csv.DictWriter(dest_fp, out_fieldnames) - writer.writeheader() - try: row = feature['properties'] if feature['geometry'] is None: @@ -791,7 +901,7 @@ def row_extract_and_reproject(source_config, source_row, disable_centroids=False if source_row.get(GEOM_FIELDNAME.replace('GEOM', 'geom')) is not None: del out_row[GEOM_FIELDNAME.replace('GEOM', 'geom')] - if source_geom == "POINT (nan nan)": + if source_geom is not None and NONFINITE_COORDINATE_PATTERN.search(source_geom): out_row[GEOM_FIELDNAME] = None return out_row @@ -943,8 +1053,8 @@ def row_fxn_regexp(sc, row, key, fxn): pattern = re.compile(fxn.get("pattern", False)) replace = fxn.get('replace', False) if replace: - match = re.sub(pattern, convert_regexp_replace(replace), row[fxn["field"]]) - row["oa:{}".format(key)] = match + value = row[fxn["field"]] + row["oa:{}".format(key)] = re.sub(pattern, convert_regexp_replace(replace), value) if pattern.search(value) else '' else: match = pattern.search(row[fxn["field"]]) row["oa:{}".format(key)] = ''.join(filter(None, match.groups())) if match else '' @@ -1145,7 +1255,7 @@ def row_convert_to_out(source_config, row): "Convert a row from the source schema to OpenAddresses output schema" geom = row.get(GEOM_FIELDNAME, None) - if geom == "POINT EMPTY" or geom == '': + if geom == "POINT EMPTY" or geom == '' or (geom is not None and NONFINITE_COORDINATE_PATTERN.search(geom)): geom = None output = { @@ -1190,7 +1300,7 @@ def extract_to_source_csv(source_config, source_path, extract_path, disable_cent format_string = source_config.data_source["conform"]['format'] protocol_string = source_config.data_source['protocol'] - if format_string in ("shapefile", "xml", "gdb", "gpkg"): + if format_string in ("shapefile", "xml", "gdb", "gpkg", "kml"): ogr_source_path = normalize_ogr_filename_case(source_path) ogr_source_to_csv(source_config, ogr_source_path, extract_path, disable_centroids) elif format_string == "csv": @@ -1222,7 +1332,7 @@ def transform_to_out_geojson(source_config, extract_path, dest_path): # For every row in the extract for extract_row in reader: out_row = row_transform_and_convert(source_config, extract_row) - dest_fp.write(json.dumps(out_row) + '\n') + dest_fp.write(json.dumps(out_row, allow_nan=False) + '\n') def conform_cli(source_config, source_path, dest_path, disable_centroids=False): "Command line entry point for conforming a downloaded source to an output CSV." @@ -1233,7 +1343,7 @@ def conform_cli(source_config, source_path, dest_path, disable_centroids=False): format_string = source_config.data_source["conform"].get('format') - if not format_string in ["shapefile", "geojson", "csv", "xml", "gdb", "gpkg"]: + if not format_string in ["shapefile", "geojson", "csv", "xml", "gdb", "gpkg", "kml"]: _L.warning("Skipping file with unknown conform: %s", source_path) return 1 diff --git a/openaddr/tests/__init__.py b/openaddr/tests/__init__.py index e00127d6..5dfecc58 100644 --- a/openaddr/tests/__init__.py +++ b/openaddr/tests/__init__.py @@ -1072,10 +1072,16 @@ def test_single_ny_orange(self): output_path = join(dirname(state_path), state["processed"]) with open(output_path, encoding='utf8') as input: - rows = list(map(json.loads, list(input))) + contents = input.read() + self.assertNotIn('NaN', contents) + + rows = list(map(json.loads, contents.splitlines())) + + # One of the nine cached features has a NaN geometry and is skipped + self.assertEqual(len(rows), 8) self.assertEqual(rows[0]['properties']['id'], u'') self.assertEqual(rows[0]['properties']['number'], u'434') - self.assertEqual(rows[0]['properties']['hash'], u'8a72112f6b1404d8') + self.assertEqual(rows[0]['properties']['hash'], u'a0127261b0619522') self.assertEqual(rows[0]['properties']['city'], u'MONROE') self.assertEqual(rows[0]['geometry']['coordinates'], [-74.1926686, 41.3187728]) self.assertEqual(rows[0]['properties']['street'], u'') diff --git a/openaddr/tests/cache.py b/openaddr/tests/cache.py index 37ccafef..5944a347 100644 --- a/openaddr/tests/cache.py +++ b/openaddr/tests/cache.py @@ -3,6 +3,7 @@ import csv from .. import SourceConfig +from .. import cache as cache_fn from urllib.parse import urlparse, parse_qs from os.path import join, dirname @@ -16,7 +17,13 @@ import httmock import tempfile -from ..cache import guess_url_file_extension, EsriRestDownloadTask +import sys +from ..cache import guess_url_file_extension, EsriRestDownloadTask, URLDownloadTask, DownloadTask + +# openaddr/__init__.py defines a `cache` function that shadows the `cache` +# submodule on the `openaddr` package object, so `from .. import cache` +# would grab the function, not the module. Go through sys.modules instead. +cache_module = sys.modules['openaddr.cache'] class TestCacheExtensionGuessing (unittest.TestCase): @@ -93,6 +100,21 @@ def response_content(self, url, request): if request.method == 'POST' and body_qs.get('resultOffset') == ['0']: local_path = join(data_dirname, 'us-al-cullman-0.json') + if (host, path) == ('bcmaps.bradfordco.org', '/arcgis/rest/services/Address_Points/MapServer/0'): + qs = parse_qs(query) + + if qs.get('f') == ['json']: + local_path = join(data_dirname, 'us-pa-bradford-metadata.json') + + if (host, path) == ('bcmaps.bradfordco.org', '/arcgis/rest/services/Address_Points/MapServer/0/query'): + qs = parse_qs(query) + body_qs = parse_qs(request.body) + + if qs.get('returnCountOnly') == ['true']: + local_path = join(data_dirname, 'us-pa-bradford-count-only.json') + if request.method == 'POST' and body_qs.get('resultOffset') == ['0']: + local_path = join(data_dirname, 'us-pa-bradford-0.json') + if local_path: type, _ = mimetypes.guess_type(local_path) with open(local_path, 'rb') as file: @@ -392,3 +414,172 @@ def test_handle_feature_server_with_lat_lon_in_conform(self): self.assertEqual(len(all_data), 5) self.assertTrue('oa:geom' in all_data[0]) self.assertEqual(all_data[0]['oa:geom'], 'POINT (-86.82960553 34.18671398)') + + def test_skip_esri_features_with_nan_geometry(self): + """ ESRI Caching Will Skip Features Whose Geometry Is The String "NaN" """ + task = EsriRestDownloadTask('us-pa-bradford') + c = SourceConfig(dict({ + "schema": 2, + "layers": { + "addresses": [{ + "name": "default", + "conform": { + "number": "Add_Number", + "street": "FullAddr" + } + }] + } + }), "addresses", "default") + + with httmock.HTTMock(self.response_content): + output_path = task.download(["https://bcmaps.bradfordco.org/arcgis/rest/services/Address_Points/MapServer/0"], self.workdir, c) + + with open(output_path[0], 'r') as file: + all_data = list(csv.DictReader(file)) + + self.assertEqual(len(all_data), 2) + self.assertEqual([row['FullAddr'] for row in all_data], ['148 DESMOND ST APT 202', '150 DESMOND ST']) + self.assertNotIn('nan', ' '.join(row['oa:geom'] for row in all_data).lower()) + +class TestFromProtocolStringHeaders (unittest.TestCase): + + def test_headers_reach_url_download_task(self): + task = DownloadTask.from_protocol_string('http', 'us-il-champaign', headers={'Referer': 'https://example.gov/'}) + self.assertIsInstance(task, URLDownloadTask) + self.assertEqual(task.headers['Referer'], 'https://example.gov/') + # The default User-Agent is still present alongside custom headers. + self.assertIn('User-Agent', task.headers) + + def test_headers_reach_esri_download_task(self): + task = DownloadTask.from_protocol_string('ESRI', 'us-il-champaign', headers={'Referer': 'https://example.gov/'}) + self.assertIsInstance(task, EsriRestDownloadTask) + self.assertEqual(task.headers['Referer'], 'https://example.gov/') + + def test_headers_reach_esri_dumper(self): + ''' EsriRestDownloadTask.download() must pass its headers to + EsriDumper as extra_headers, since pyesridump makes its own + requests independent of openaddr.cache.request(). + ''' + workdir = tempfile.mkdtemp(prefix='testCacheHeaders-') + try: + task = EsriRestDownloadTask('us-fl-palmbeach', headers={'Referer': 'https://example.gov/'}) + c = SourceConfig(dict({ + "schema": 2, + "layers": { + "addresses": [{ + "name": "default", + "conform": None + }] + } + }), "addresses", "default") + + with patch.object(cache_module, 'EsriDumper') as dumper_patch: + dumper_patch.return_value.get_metadata.return_value = {'fields': []} + dumper_patch.return_value.get_feature_count.return_value = 0 + dumper_patch.return_value.__iter__.return_value = iter([]) + + task.download(['http://example.com/'], workdir, c) + + _, kwargs = dumper_patch.call_args + self.assertEqual(kwargs.get('extra_headers'), task.headers) + self.assertEqual(kwargs['extra_headers']['Referer'], 'https://example.gov/') + finally: + shutil.rmtree(workdir) + + def test_no_headers_still_gets_default_user_agent(self): + task = DownloadTask.from_protocol_string('http', 'us-il-champaign') + self.assertEqual(list(task.headers.keys()), ['User-Agent']) + + def test_custom_user_agent_overrides_default(self): + task = DownloadTask.from_protocol_string('http', 'us-il-champaign', headers={'User-Agent': 'custom-agent/1.0'}) + self.assertEqual(task.headers['User-Agent'], 'custom-agent/1.0') + +class TestURLDownloadTaskHeaders (unittest.TestCase): + ''' Confirm that a source's custom headers are sent on both the + file-extension pre-flight request and the real download request. + ''' + + def setUp(self): + self.workdir = tempfile.mkdtemp(prefix='testCacheHeaders-') + self.seen_referers = [] + + def tearDown(self): + shutil.rmtree(self.workdir) + + def response_content(self, url, request): + scheme, host, path, _, query, _ = urlparse(url.geturl()) + + # A query string forces guess_url_file_extension() to make a + # sniffing request instead of trusting the URL's extension, + # so this URL exercises both the pre-flight and real download. + if (host, path, query) == ('headers-test.local', '/addresses.csv', 'download=true'): + self.seen_referers.append(request.headers.get('Referer')) + return httmock.response(200, b'FAKE,FAKE\n', headers={'Content-Type': 'text/csv'}) + + raise NotImplementedError(url.geturl()) + + def test_headers_sent_on_preflight_and_download_requests(self): + task = URLDownloadTask('us-il-champaign', headers={'Referer': 'https://example.gov/gis/'}) + with httmock.HTTMock(self.response_content): + output_files = task.download(['http://headers-test.local/addresses.csv?download=true'], self.workdir, None) + + self.assertEqual(len(output_files), 1) + # One request for the extension-guessing pre-flight, one for the real download. + self.assertEqual(self.seen_referers, ['https://example.gov/gis/', 'https://example.gov/gis/']) + +class TestCacheRequestSettings (unittest.TestCase): + ''' Confirm that openaddr.cache() reads headers from the nested + request.headers key in the source config, and that a source with + no request settings at all still works. + ''' + + def setUp(self): + self.destdir = tempfile.mkdtemp(prefix='testCacheRequestSettings-') + self.seen_referers = [] + + def tearDown(self): + shutil.rmtree(self.destdir) + + def response_content(self, url, request): + scheme, host, path, _, query, _ = urlparse(url.geturl()) + + # A query string forces guess_url_file_extension() to make a + # sniffing request instead of trusting the URL's extension, + # so this URL exercises both the pre-flight and real download. + if (host, path, query) == ('request-settings-test.local', '/addresses.csv', 'download=true'): + self.seen_referers.append(request.headers.get('Referer')) + return httmock.response(200, b'FAKE,FAKE\n', headers={'Content-Type': 'text/csv'}) + + raise NotImplementedError(url.geturl()) + + def make_source_config(self, layersource_extra): + return SourceConfig(dict({ + "schema": 2, + "layers": { + "addresses": [dict({ + "name": "default", + "protocol": "http", + "data": "http://request-settings-test.local/addresses.csv?download=true", + }, **layersource_extra)] + } + }), "addresses", "default") + + def test_cache_reads_headers_from_request_settings(self): + source_config = self.make_source_config({ + "request": { + "headers": {"Referer": "https://example.gov/gis/"} + } + }) + + with httmock.HTTMock(self.response_content): + cache_fn(source_config, self.destdir, {}) + + self.assertEqual(self.seen_referers, ['https://example.gov/gis/', 'https://example.gov/gis/']) + + def test_cache_without_request_settings_sends_no_referer(self): + source_config = self.make_source_config({}) + + with httmock.HTTMock(self.response_content): + cache_fn(source_config, self.destdir, {}) + + self.assertEqual(self.seen_referers, [None, None]) diff --git a/openaddr/tests/conform.py b/openaddr/tests/conform.py index 6a0bc432..fc943488 100644 --- a/openaddr/tests/conform.py +++ b/openaddr/tests/conform.py @@ -12,6 +12,9 @@ import tempfile import shutil +from unittest import mock +from zipfile import ZipFile + from .. import SourceConfig from ..conform import ( @@ -23,9 +26,10 @@ row_fxn_postfixed_unit, row_fxn_remove_prefix, row_fxn_remove_postfix, row_fxn_chain, row_fxn_first_non_empty, row_fxn_constant, row_fxn_map, - row_canonicalize_unit_and_number, conform_cli, + row_canonicalize_unit_and_number, conform_cli, transform_to_out_geojson, convert_regexp_replace, normalize_ogr_filename_case, - is_in, geojson_source_to_csv, check_source_tests + is_in, geojson_source_to_csv, ogr_source_to_csv, check_source_tests, + ZipDecompressTask, DecompressionError, elaborate_filenames ) " Return an x,y array given a wkt point string" @@ -74,6 +78,59 @@ def test_row_convert_to_out(self): } }, r) + def test_row_convert_to_out_nonfinite_geometry(self): + "ESRI sources report a null point geometry as NaN, which isn't valid GeoJSON" + d = SourceConfig(dict({ + "schema": 2, + "layers": { + "addresses": [{ + "name": "default", + "conform": { "street": "s", "number": "n" } + }] + } + }), "addresses", "default") + + for wkt in ("POINT (NaN NaN)", "POINT (nan nan)", "POINT Z (nan nan nan)"): + r = row_convert_to_out(d, {"s": "DESMOND ST", "n": "148", GEOM_FIELDNAME: wkt}) + self.assertIsNone(r["geometry"], wkt) + + def test_transform_to_out_geojson_skips_nonfinite_geometry(self): + "A NaN coordinate must never reach the output GeoJSON" + d = SourceConfig(dict({ + "schema": 2, + "layers": { + "addresses": [{ + "name": "default", + "conform": { "street": "s", "number": "n" } + }] + } + }), "addresses", "default") + + workdir = tempfile.mkdtemp(prefix='testConform-') + try: + extract_path = os.path.join(workdir, 'extract.csv') + dest_path = os.path.join(workdir, 'out.geojson') + + with open(extract_path, 'w', encoding='utf-8') as file: + writer = csv.DictWriter(file, fieldnames=['s', 'n', GEOM_FIELDNAME]) + writer.writeheader() + writer.writerow({'s': 'DESMOND ST', 'n': '148', GEOM_FIELDNAME: 'POINT (NaN NaN)'}) + writer.writerow({'s': 'DESMOND ST', 'n': '150', GEOM_FIELDNAME: 'POINT (-76.51522 41.97925)'}) + + transform_to_out_geojson(d, extract_path, dest_path) + + with open(dest_path, 'r', encoding='utf-8') as file: + contents = file.read() + + self.assertNotIn('NaN', contents) + rows = [json.loads(line) for line in contents.splitlines()] + finally: + shutil.rmtree(workdir) + + self.assertEqual(len(rows), 2) + self.assertIsNone(rows[0]['geometry']) + self.assertEqual(rows[1]['geometry'], {'type': 'Point', 'coordinates': [-76.51522, 41.97925]}) + def test_row_merge(self): d = SourceConfig(dict({ "schema": 2, @@ -349,6 +406,37 @@ def test_row_fxn_regexp(self): d = row_fxn_regexp(c, d, "street", c.data_source["conform"]["street"]) self.assertEqual(e, d) + "regex split - replace - bad match" + c = SourceConfig(dict({ + "schema": 2, + "layers": { + "addresses": [{ + "name": "default", + "conform": { + "number": { + "function": "regexp", + "field": "ADDRESS", + "pattern": "^([0-9]+)(?:.*)", + "replace": "$1" + }, + "street": { + "function": "regexp", + "field": "ADDRESS", + "pattern": "(fake)", + "replace": "$1" + } + } + }] + } + }), "addresses", "default") + d = { "ADDRESS": "123 MAPLE ST" } + e = copy.deepcopy(d) + e.update({ "oa:number": "123", "oa:street": "" }) + + d = row_fxn_regexp(c, d, "number", c.data_source["conform"]["number"]) + d = row_fxn_regexp(c, d, "street", c.data_source["conform"]["street"]) + self.assertEqual(e, d) + def test_transform_and_convert(self): d = SourceConfig(dict({ "schema": 2, @@ -1849,6 +1937,36 @@ def test_lake_man_gpkg(self): self.assertEqual(rows[5]['properties']['number'], '5115') self.assertEqual(rows[5]['properties']['street'], 'OLD MILL RD') + def test_lake_man_kml(self): + with open(os.path.join(self.conforms_dir, "lake-man-kml.json")) as file: + source_config = SourceConfig(json.load(file), "addresses", "default") + source_path = os.path.join(self.conforms_dir, "lake-man.kml") + dest_path = os.path.join(self.testdir, 'lake-man-kml-conformed.csv') + + rc = conform_cli(source_config, source_path, dest_path) + self.assertEqual(0, rc) + + with open(dest_path) as fp: + rows = list(map(json.loads, list(fp))) + + self.assertEqual('Point', rows[0]['geometry']['type']) + self.assertAlmostEqual(-122.2592497, rows[0]['geometry']['coordinates'][0], places=4) + self.assertAlmostEqual(37.8026126, rows[0]['geometry']['coordinates'][1], places=4) + + self.assertEqual(6, len(rows)) + self.assertEqual(rows[0]['properties']['number'], '5115') + self.assertEqual(rows[0]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[1]['properties']['number'], '5121') + self.assertEqual(rows[1]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[2]['properties']['number'], '5133') + self.assertEqual(rows[2]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[3]['properties']['number'], '5126') + self.assertEqual(rows[3]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[4]['properties']['number'], '5120') + self.assertEqual(rows[4]['properties']['street'], 'FRUITED PLAINS LN') + self.assertEqual(rows[5]['properties']['number'], '5115') + self.assertEqual(rows[5]['properties']['street'], 'OLD MILL RD') + def test_lake_man_split(self): rc, dest_path = self._run_conform_on_source('lake-man-split', 'shp') self.assertEqual(0, rc) @@ -1937,6 +2055,24 @@ def test_lake_man_shp_noprj_epsg26943(self): self.assertAlmostEqual(-122.2592497, rows[0]['geometry']['coordinates'][0], places=4) self.assertAlmostEqual(37.8026126, rows[0]['geometry']['coordinates'][1], places=4) + def test_lake_man_shp_epsg4269_axis_order(self): + # Regression test for https://github.com/openaddresses/batch-machine/issues/26 + # A shapefile with an explicit geographic "srs" tag (here EPSG:4269, NAD83) + # must not have its coordinates flipped by GDAL 3's authority-compliant + # (lat, lon) axis order. The underlying shapefile geometry is stored in + # ordinary (lon, lat) order, same coordinates as the plain lake-man.shp + # fixture used by test_lake_man, so the expected output here matches that + # test's expected values. Before the fix, this source's coordinates come + # out as (lat, lon) instead of (lon, lat). + rc, dest_path = self._run_conform_on_source('lake-man-epsg4269', 'shp') + self.assertEqual(0, rc) + + with open(dest_path) as fp: + rows = list(map(json.loads, list(fp))) + self.assertEqual('Point', rows[0]['geometry']['type']) + self.assertAlmostEqual(-122.2592497, rows[0]['geometry']['coordinates'][0], places=4) + self.assertAlmostEqual(37.8026126, rows[0]['geometry']['coordinates'][1], places=4) + # TODO: add tests for non-ESRI GeoJSON sources def test_lake_man_split2(self): @@ -2016,8 +2152,11 @@ def test_lake_man_gml(self): with open(dest_path) as fp: rows = list(map(json.loads, list(fp))) self.assertEqual(6, len(rows)) - self.assertAlmostEqual(37.8026126, rows[0]['geometry']['coordinates'][0], places=4) - self.assertAlmostEqual(-122.2592497, rows[0]['geometry']['coordinates'][1], places=4) + # GeoJSON coordinates are always [lon, lat]. This source has an + # explicit "srs": "EPSG:4326" tag; see + # https://github.com/openaddresses/batch-machine/issues/26. + self.assertAlmostEqual(-122.2592497, rows[0]['geometry']['coordinates'][0], places=4) + self.assertAlmostEqual(37.8026126, rows[0]['geometry']['coordinates'][1], places=4) self.assertEqual(rows[0]['properties']['number'], '5115') self.assertEqual(rows[0]['properties']['street'], 'FRUITED PLAINS LN') @@ -2255,6 +2394,107 @@ def test_geojson_source_to_csv(self): self.assertEqual(row[GEOM_FIELDNAME], 'POINT (-74.9833483425103 40.05498715)') self.assertEqual(row['PARCEL_NUM'], '02-022-003') + def test_geojson_source_to_csv_non_uniform_properties(self): + ''' Features with different property keys should not crash the writer. + ''' + c = SourceConfig(dict({ + "schema": 2, + "layers": { + "addresses": [{ + "name": "default", + "conform": { } + }] + } + }), "addresses", "default") + + geojson_path = os.path.join(self.testdir, 'non-uniform.geojson') + with open(geojson_path, 'w', encoding='utf8') as file: + json.dump({ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {"pid": "1"}, + "geometry": {"type": "Point", "coordinates": [-121.2, 39.3]} + }, + { + "type": "Feature", + "properties": {"pid": "2", "address": "123 Maple St"}, + "geometry": {"type": "Point", "coordinates": [-121.3, 39.4]} + } + ] + }, file) + + csv_path = os.path.join(self.testdir, 'non-uniform-conformed.csv') + geojson_source_to_csv(c, geojson_path, csv_path) + + with open(csv_path, encoding='utf8') as file: + rows = list(csv.DictReader(file)) + self.assertEqual(rows[0]['pid'], '1') + self.assertEqual(rows[0]['address'], '') + self.assertEqual(rows[1]['pid'], '2') + self.assertEqual(rows[1]['address'], '123 Maple St') + + def test_ogr_source_to_csv_multisurface(self): + ''' Curve geometries (e.g. MULTISURFACE) from file geodatabases should + be linearized before being written out, since shapely/GEOS can't + parse GDAL's curve-geometry WKT types. + https://github.com/openaddresses/batch-machine/issues/62 + ''' + from osgeo import ogr, osr + from shapely.wkt import loads as wkt_loads + + srs = osr.SpatialReference() + srs.ImportFromEPSG(4326) + + source_path = os.path.join(self.testdir, 'curved-buildings.gpkg') + driver = ogr.GetDriverByName('GPKG') + datasource = driver.CreateDataSource(source_path) + layer = datasource.CreateLayer('buildings', srs=srs, geom_type=ogr.wkbMultiSurface) + layer.CreateField(ogr.FieldDefn('bin', ogr.OFTString)) + + feature = ogr.Feature(layer.GetLayerDefn()) + feature.SetField('bin', '12345') + feature.SetGeometry(ogr.CreateGeometryFromWkt( + 'MULTISURFACE (CURVEPOLYGON (CIRCULARSTRING (0 0, 1 1, 2 0, 1 -1, 0 0)))' + )) + layer.CreateFeature(feature) + feature = None + datasource = None + + c = SourceConfig(dict({ + "schema": 2, + "layers": { + "buildings": [{ + "name": "default", + "conform": {"format": "gpkg"} + }] + } + }), "buildings", "default") + + csv_path = os.path.join(self.testdir, 'curved-buildings.csv') + ogr_source_to_csv(c, source_path, csv_path) + + with open(csv_path, encoding='utf8') as file: + row = next(csv.DictReader(file)) + self.assertEqual(row['bin'], '12345') + + # The output WKT should be a linear geometry type that shapely can parse. + self.assertNotIn('CURVE', row[GEOM_FIELDNAME]) + self.assertNotIn('SURFACE', row[GEOM_FIELDNAME]) + + geom = wkt_loads(row[GEOM_FIELDNAME]) + self.assertEqual(geom.geom_type, 'MultiPolygon') + + # Sanity-check the linearized shape isn't degenerate and roughly + # matches the original curve geometry's bounds. + minx, miny, maxx, maxy = geom.bounds + self.assertAlmostEqual(minx, 0, places=2) + self.assertAlmostEqual(maxx, 2, places=2) + self.assertAlmostEqual(miny, -1, places=2) + self.assertAlmostEqual(maxy, 1, places=2) + self.assertGreater(geom.area, 0) + class TestConformCsv(unittest.TestCase): "Fixture to create real files to test csv_source_to_csv()" @@ -2540,3 +2780,171 @@ def test_row_fxn_map_else(self): d = row_fxn_map(c, d, "accuracy", c.data_source["conform"]["accuracy"]) self.assertEqual(e, d) + + +class TestZipDecompressTask(unittest.TestCase): + ''' Regression tests for GitHub issue #35: "Support zipped shapefiles + within source zip" -- an outer source zip contains a nested zip that + itself contains the shapefile, plus miscellaneous extra files + (license, readme, lookup tables) sitting next to the nested zip. + ''' + + def setUp(self): + self.testdir = tempfile.mkdtemp(prefix='openaddr-TestZipDecompressTask-') + self.workdir = os.path.join(self.testdir, 'work') + os.mkdir(self.workdir) + + # A real shapefile (.shp/.shx/.dbf/.prj) fixture, already zipped up. + self.inner_zip_path = os.path.join( + os.path.dirname(__file__), 'conforms', 'lake-man.zip') + + def tearDown(self): + shutil.rmtree(self.testdir) + + def _make_outer_zip(self, extra_files={'license.txt': 'be nice', 'readme.txt': 'read me'}): + ''' Build an outer.zip containing nested.zip (the shapefile zip) + alongside some unrelated non-zip files, matching the structure + described in issue #35. + ''' + outer_zip_path = os.path.join(self.testdir, 'outer.zip') + + with ZipFile(outer_zip_path, 'w') as outer_zip: + outer_zip.write(self.inner_zip_path, arcname='nested.zip') + for name, content in extra_files.items(): + outer_zip.writestr(name, content) + + return outer_zip_path + + def test_single_nested_zip_is_extracted(self): + ''' A zip-of-a-zip-containing-a-shapefile, with no "file" filter + (i.e. no conform "file" tag), should be fully extracted so the + shapefile members end up available. + ''' + outer_zip_path = self._make_outer_zip() + + task = ZipDecompressTask() + output_files = task.decompress([outer_zip_path], self.workdir, []) + + output_names = {os.path.basename(path) for path in output_files} + self.assertIn('lake-man.shp', output_names) + self.assertIn('lake-man.shx', output_names) + self.assertIn('lake-man.dbf', output_names) + self.assertIn('lake-man.prj', output_names) + + # The extra sibling files should also have survived extraction. + self.assertIn('license.txt', output_names) + self.assertIn('readme.txt', output_names) + + shp_path = next(path for path in output_files if path.endswith('lake-man.shp')) + self.assertGreater(os.path.getsize(shp_path), 0) + + def test_single_nested_zip_is_extracted_with_file_filter(self): + ''' Same structure as above, but exercised the way a real source + would use it: with a conform "file" tag naming the shapefile + inside the nested zip (e.g. "file": "lake-man.shp"), which is + expanded by elaborate_filenames() into the .shp/.shx/.dbf/.prj + set and passed through as the `filenames` allow-list. + ''' + outer_zip_path = self._make_outer_zip() + filenames = elaborate_filenames('lake-man.shp') + + task = ZipDecompressTask() + output_files = task.decompress([outer_zip_path], self.workdir, filenames) + + output_names = {os.path.basename(path) for path in output_files} + self.assertIn('lake-man.shp', output_names) + self.assertIn('lake-man.shx', output_names) + self.assertIn('lake-man.dbf', output_names) + self.assertIn('lake-man.prj', output_names) + + # Sibling non-zip files that don't match the filter should still be + # skipped, since the caller only asked for the named shapefile. + self.assertNotIn('license.txt', output_names) + self.assertNotIn('readme.txt', output_names) + + def test_multiple_nested_zips_raise_decompression_error(self): + ''' If more than one zip appears at the same directory level inside + the outer zip, extraction should fail loudly rather than + silently pick one. + ''' + outer_zip_path = os.path.join(self.testdir, 'outer-ambiguous.zip') + + with ZipFile(outer_zip_path, 'w') as outer_zip: + outer_zip.write(self.inner_zip_path, arcname='nested-a.zip') + outer_zip.write(self.inner_zip_path, arcname='nested-b.zip') + + task = ZipDecompressTask() + with self.assertRaises(DecompressionError): + task.decompress([outer_zip_path], self.workdir, []) + + def test_skips_nested_zip_once_filtered_file_is_already_found(self): + ''' If the outer zip already directly contains everything the + "file" filter asked for, an unrelated nested zip sitting next + to it should never be opened/extracted at all - there's no + reason to recurse once the request is satisfied, and not doing + so limits exposure to a zip bomb hiding in that nested zip. + ''' + outer_zip_path = os.path.join(self.testdir, 'outer-already-satisfied.zip') + conforms_dir = os.path.join(os.path.dirname(__file__), 'conforms') + + with ZipFile(outer_zip_path, 'w') as outer_zip: + for ext in ('.shp', '.shx', '.dbf', '.prj'): + outer_zip.write( + os.path.join(conforms_dir, 'lake-man' + ext), + arcname='lake-man' + ext + ) + # An unrelated nested zip that should never get opened. + with tempfile.NamedTemporaryFile(suffix='.zip') as decoy_zip_file: + with ZipFile(decoy_zip_file.name, 'w') as decoy_zip: + decoy_zip.writestr('decoy.txt', 'should never be extracted') + outer_zip.write(decoy_zip_file.name, arcname='decoy.zip') + + filenames = elaborate_filenames('lake-man.shp') + + task = ZipDecompressTask() + output_files = task.decompress([outer_zip_path], self.workdir, filenames) + + output_names = {os.path.basename(path) for path in output_files} + self.assertIn('lake-man.shp', output_names) + self.assertIn('lake-man.shx', output_names) + self.assertIn('lake-man.dbf', output_names) + self.assertIn('lake-man.prj', output_names) + + # Proves decoy.zip was never opened/extracted: neither its contents + # nor the zip file itself should appear anywhere in the output. + self.assertNotIn('decoy.txt', output_names) + self.assertNotIn('decoy.zip', output_names) + + def test_oversized_entry_raises_decompression_error(self): + ''' A zip entry declaring an uncompressed size over the configured + cap should be refused before extraction, as a zip bomb guard. + Uses a real (small) fixture but lowers the cap far below its + actual size, rather than crafting an actual multi-GB payload. + ''' + outer_zip_path = self._make_outer_zip() + + task = ZipDecompressTask() + with mock.patch.object(ZipDecompressTask, 'MAX_ZIP_ENTRY_BYTES', 10): + with self.assertRaises(DecompressionError): + task.decompress([outer_zip_path], self.workdir, []) + + def test_deeply_nested_zips_raise_decompression_error(self): + ''' A chain of nested zips deeper than the configured limit should + be refused, as a zip bomb guard against chained amplification. + Each level's nested zip is placed inside its own subfolder + (arcname "levelN/nested.zip") so unwrapping one level doesn't + land in the same directory as the next - matching how the + existing "one zip per directory" width check expects real + nested archives to be laid out. + ''' + current_path = self.inner_zip_path + for level in range(4): + next_path = os.path.join(self.testdir, 'wrap-{}.zip'.format(level)) + with ZipFile(next_path, 'w') as z: + z.write(current_path, arcname='level{}/nested.zip'.format(level)) + current_path = next_path + + task = ZipDecompressTask() + with mock.patch.object(ZipDecompressTask, 'MAX_NESTED_ZIP_DEPTH', 2): + with self.assertRaises(DecompressionError): + task.decompress([current_path], self.workdir, []) diff --git a/openaddr/tests/conforms/lake-man-epsg4269.dbf b/openaddr/tests/conforms/lake-man-epsg4269.dbf new file mode 100644 index 00000000..a710e002 Binary files /dev/null and b/openaddr/tests/conforms/lake-man-epsg4269.dbf differ diff --git a/openaddr/tests/conforms/lake-man-epsg4269.json b/openaddr/tests/conforms/lake-man-epsg4269.json new file mode 100644 index 00000000..4e591ef6 --- /dev/null +++ b/openaddr/tests/conforms/lake-man-epsg4269.json @@ -0,0 +1,20 @@ +{ + "schema": 2, + "layers": { + "addresses": [{ + "name": "default", + "data": "http://fake-web/lake-man-epsg4269.zip", + "cache": "http://fake-cache/lake-man-epsg4269.zip", + "protocol": "http", + "compression": "zip", + "conform": { + "lon": "X", + "lat": "Y", + "number": "NUMBER", + "street": "STRNAME", + "format": "shapefile", + "srs": "EPSG:4269" + } + }] + } +} diff --git a/openaddr/tests/conforms/lake-man-epsg4269.shp b/openaddr/tests/conforms/lake-man-epsg4269.shp new file mode 100644 index 00000000..6963b4fb Binary files /dev/null and b/openaddr/tests/conforms/lake-man-epsg4269.shp differ diff --git a/openaddr/tests/conforms/lake-man-epsg4269.shx b/openaddr/tests/conforms/lake-man-epsg4269.shx new file mode 100644 index 00000000..f7808d66 Binary files /dev/null and b/openaddr/tests/conforms/lake-man-epsg4269.shx differ diff --git a/openaddr/tests/conforms/lake-man-kml.json b/openaddr/tests/conforms/lake-man-kml.json new file mode 100644 index 00000000..bca93cde --- /dev/null +++ b/openaddr/tests/conforms/lake-man-kml.json @@ -0,0 +1,18 @@ +{ + "schema": 2, + "layers": { + "addresses": [{ + "name": "default", + "data": "http://fake-web/lake-man.kml", + "cache": "http://fake-cache/lake-man.kml", + "protocol": "http", + "conform": { + "lon": "X", + "lat": "Y", + "number": "NUMBER", + "street": "STRNAME", + "format": "kml" + } + }] + } +} diff --git a/openaddr/tests/conforms/lake-man.kml b/openaddr/tests/conforms/lake-man.kml new file mode 100644 index 00000000..a841d361 --- /dev/null +++ b/openaddr/tests/conforms/lake-man.kml @@ -0,0 +1,54 @@ + + + + lake-man + + 5115 FRUITED PLAINS LN + + 5115 + FRUITED PLAINS LN + + -122.259249687195,37.8026126376074 + + + 5121 FRUITED PLAINS LN + + 5121 + FRUITED PLAINS LN + + -122.256717681885,37.8025278661215 + + + 5133 FRUITED PLAINS LN + + 5133 + FRUITED PLAINS LN + + -122.257940769196,37.802968676786 + + + 5126 FRUITED PLAINS LN + + 5126 + FRUITED PLAINS LN + + -122.258970737457,37.8007476424409 + + + 5120 FRUITED PLAINS LN + + 5120 + FRUITED PLAINS LN + + -122.256953716278,37.800713733002 + + + 5115 OLD MILL RD + + 5115 + OLD MILL RD + + -122.257640361786,37.8043589085714 + + + diff --git a/openaddr/tests/data/us-pa-bradford-0.json b/openaddr/tests/data/us-pa-bradford-0.json new file mode 100644 index 00000000..81151554 --- /dev/null +++ b/openaddr/tests/data/us-pa-bradford-0.json @@ -0,0 +1,63 @@ +{ + "objectIdFieldName": "OBJECTID", + "globalIdFieldName": "", + "geometryType": "esriGeometryPoint", + "spatialReference": { + "wkid": 4326, + "latestWkid": 4326 + }, + "fields": [ + { + "name": "OBJECTID", + "alias": "OBJECTID", + "type": "esriFieldTypeOID" + }, + { + "name": "Add_Number", + "alias": "Add_Number", + "type": "esriFieldTypeInteger" + }, + { + "name": "FullAddr", + "alias": "FullAddr", + "type": "esriFieldTypeString", + "length": 254 + } + ], + "features": [ + { + "attributes": { + "OBJECTID": 18, + "Add_Number": 148, + "FullAddr": "148 DESMOND ST APT 206" + }, + "geometry": { + "x": "NaN", + "y": "NaN" + } + }, + { + "attributes": { + "OBJECTID": 33828, + "Add_Number": 148, + "FullAddr": "148 DESMOND ST APT 202" + }, + "geometry": { + "x": -76.51535, + "y": 41.97911 + } + }, + { + "attributes": { + "OBJECTID": 33840, + "Add_Number": 150, + "FullAddr": "150 DESMOND ST" + }, + "geometry": { + "x": -76.51522, + "y": 41.97925 + } + } + ], + "exceededTransferLimit": false +} \ No newline at end of file diff --git a/openaddr/tests/data/us-pa-bradford-count-only.json b/openaddr/tests/data/us-pa-bradford-count-only.json new file mode 100644 index 00000000..ef96c462 --- /dev/null +++ b/openaddr/tests/data/us-pa-bradford-count-only.json @@ -0,0 +1 @@ +{"count": 3} \ No newline at end of file diff --git a/openaddr/tests/data/us-pa-bradford-metadata.json b/openaddr/tests/data/us-pa-bradford-metadata.json new file mode 100644 index 00000000..7a18c4de --- /dev/null +++ b/openaddr/tests/data/us-pa-bradford-metadata.json @@ -0,0 +1,47 @@ +{ + "currentVersion": 10.51, + "id": 0, + "name": "Addresses", + "type": "Feature Layer", + "geometryType": "esriGeometryPoint", + "objectIdField": "OBJECTID", + "displayField": "FullAddr", + "advancedQueryCapabilities": { + "supportsPagination": true, + "supportsStatistics": true, + "supportsOrderBy": true, + "supportsDistinct": true + }, + "supportsStatistics": true, + "supportsAdvancedQueries": true, + "extent": { + "xmin": -76.6, + "ymin": 41.6, + "xmax": -76.2, + "ymax": 41.9, + "spatialReference": { + "wkid": 4326, + "latestWkid": 4326 + } + }, + "fields": [ + { + "name": "OBJECTID", + "alias": "OBJECTID", + "type": "esriFieldTypeOID" + }, + { + "name": "Add_Number", + "alias": "Add_Number", + "type": "esriFieldTypeInteger" + }, + { + "name": "FullAddr", + "alias": "FullAddr", + "type": "esriFieldTypeString", + "length": 254 + } + ], + "capabilities": "Map,Query,Data", + "maxRecordCount": 1000 +} \ No newline at end of file