From 9c5fa7ab0381387d8c96b1fd80590b72fe1c437f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 23 Sep 2026 01:23:42 +0000 Subject: [PATCH 01/13] Initial plan From e975cfba0cd7c6d3a93a3869ecb9650b0c5a1b95 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 23 Sep 2026 01:36:20 +0000 Subject: [PATCH 02/13] Add relational ingest and reusable dataset view flattening foundation Co-authored-by: chicoreus <3864169+chicoreus@users.noreply.github.com> --- CLAUDE.md | 5 +- README.md | 10 +- .../bdq_workbench/app/AppConfig.java | 42 +++- .../app/BdqWorkbenchApplication.java | 5 +- .../bdq_workbench/app/BdqWorkbenchGui.java | 182 +++++++++++++++- .../bdq_workbench/app/ConfigLoader.java | 3 +- .../bdq_workbench/app/WorkbenchFacade.java | 2 +- .../ingest/BuiltInDatasetViews.java | 120 +++++++++++ .../ingest/DataPackageDialectParser.java | 46 +++- .../ingest/DataPackageForeignKey.java | 30 +++ .../ingest/DataPackageResourceMeta.java | 5 +- .../bdq_workbench/ingest/DatasetViewIO.java | 75 +++++++ .../ingest/DefaultIngestService.java | 76 ++++++- .../ingest/DwcArchiveCoreMeta.java | 4 + .../ingest/DwcArchiveMetaParser.java | 5 + .../bdq_workbench/ingest/IngestService.java | 12 ++ .../ingest/RelationalDatasetIngestor.java | 196 ++++++++++++++++++ .../ingest/RelationalIngestResult.java | 45 ++++ .../ingest/SchemaFingerprint.java | 60 ++++++ .../ingest/ViewFlattenResult.java | 39 ++++ .../bdq_workbench/ingest/ViewFlattener.java | 130 ++++++++++++ .../bdq_workbench/model/CanonicalRecord.java | 56 +++-- .../bdq_workbench/model/DatasetSchema.java | 43 ++++ .../bdq_workbench/model/DatasetView.java | 45 ++++ .../model/DatasetViewCardinalityPolicy.java | 34 +++ .../bdq_workbench/model/DatasetViewJoin.java | 33 +++ .../model/DatasetViewMapping.java | 30 +++ .../bdq_workbench/model/RecordGraph.java | 42 ++++ .../model/RelationshipSchema.java | 37 ++++ .../bdq_workbench/model/SourceCell.java | 37 ++++ .../bdq_workbench/model/TableSchema.java | 46 ++++ src/main/resources/application.properties | 1 + .../app/BdqWorkbenchApplicationTest.java | 2 +- .../bdq_workbench/app/ConfigLoaderTest.java | 9 + .../ingest/DataPackageDialectIngestTest.java | 80 +++++++ .../ingest/DatasetViewSchemaTest.java | 77 +++++++ .../ingest/DwcArchiveMetaIngestTest.java | 41 ++++ .../ingest/ViewFlattenerTest.java | 87 ++++++++ 38 files changed, 1755 insertions(+), 37 deletions(-) create mode 100644 src/main/java/org/filteredpush/bdq_workbench/ingest/BuiltInDatasetViews.java create mode 100644 src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageForeignKey.java create mode 100644 src/main/java/org/filteredpush/bdq_workbench/ingest/DatasetViewIO.java create mode 100644 src/main/java/org/filteredpush/bdq_workbench/ingest/RelationalDatasetIngestor.java create mode 100644 src/main/java/org/filteredpush/bdq_workbench/ingest/RelationalIngestResult.java create mode 100644 src/main/java/org/filteredpush/bdq_workbench/ingest/SchemaFingerprint.java create mode 100644 src/main/java/org/filteredpush/bdq_workbench/ingest/ViewFlattenResult.java create mode 100644 src/main/java/org/filteredpush/bdq_workbench/ingest/ViewFlattener.java create mode 100644 src/main/java/org/filteredpush/bdq_workbench/model/DatasetSchema.java create mode 100644 src/main/java/org/filteredpush/bdq_workbench/model/DatasetView.java create mode 100644 src/main/java/org/filteredpush/bdq_workbench/model/DatasetViewCardinalityPolicy.java create mode 100644 src/main/java/org/filteredpush/bdq_workbench/model/DatasetViewJoin.java create mode 100644 src/main/java/org/filteredpush/bdq_workbench/model/DatasetViewMapping.java create mode 100644 src/main/java/org/filteredpush/bdq_workbench/model/RecordGraph.java create mode 100644 src/main/java/org/filteredpush/bdq_workbench/model/RelationshipSchema.java create mode 100644 src/main/java/org/filteredpush/bdq_workbench/model/SourceCell.java create mode 100644 src/main/java/org/filteredpush/bdq_workbench/model/TableSchema.java create mode 100644 src/test/java/org/filteredpush/bdq_workbench/ingest/DatasetViewSchemaTest.java create mode 100644 src/test/java/org/filteredpush/bdq_workbench/ingest/ViewFlattenerTest.java diff --git a/CLAUDE.md b/CLAUDE.md index 977ac86..6797a98 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -56,14 +56,15 @@ mistyped path fails fast and offline — keep that ordering, it is what keeps th touching the network. Configuration defaults live in `src/main/resources/application.properties` -(`bdq.usecase.file`, `bdq.rdf.files`, `bdq.dataset`, `bdq.dataset.table`, `bdq.usecase.id`, +(`bdq.usecase.file`, `bdq.rdf.files`, `bdq.dataset`, `bdq.dataset.table`, `bdq.dataset.view`, `bdq.usecase.id`, `bdq.discovery.packages`, `bdq.threads`, `bdq.execution.dedup`) and are merged with CLI/GUI overrides by `ConfigLoader`. `bdq.usecase.file` and `bdq.rdf.files` ship blank, which means "use the `WorkbenchDefaults` published sources"; set either to a local path or an HTTP URL to pin a run. `bdq.dataset.table` (CLI `--dataset-table`, GUI advanced options) names which table of a multi-table dataset to run against. Both entry points fetch and cache use-case/test-definition/ ontology RDF from `bdq.tdwg.org` through `CachedResourceResolver`; RDF/XML, Turtle, and JSON-LD -serializations are all supported. Logging is DEBUG-by-default to the +serializations are all supported. `bdq.dataset.view` (CLI `--dataset-view`, GUI "Build Dataset View...") +names a standalone JSON DatasetView used to flatten relational inputs before execution. Logging is DEBUG-by-default to the console via `src/main/resources/logback.xml`. ## Architecture diff --git a/README.md b/README.md index d25e334..cd440e2 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,14 @@ Configuration defaults are in `src/main/resources/application.properties` and ca java -jar target/bdq_workbench-0.1.0-SNAPSHOT.jar --dataset path/to/dataset.zip ``` +Reusable relational flattening views can be supplied with `--dataset-view` (`bdq.dataset.view` in config): + +```bash +java -jar target/bdq_workbench-0.1.0-SNAPSHOT.jar \ + --dataset path/to/datapackage.json \ + --dataset-view path/to/bdq-dataset-view.json +``` + Configuration precedence is: 1. command-line or GUI-supplied overrides @@ -76,7 +84,7 @@ Dataset input | +--> Load DwC-A zip / Data Package CSV | - +--> [optional] Build record filters from dataset terms/values (GUI) + +--> [optional] Build dataset view and/or record filters from dataset terms/values (GUI) | +--> [optional] Apply record filters | diff --git a/src/main/java/org/filteredpush/bdq_workbench/app/AppConfig.java b/src/main/java/org/filteredpush/bdq_workbench/app/AppConfig.java index 8a099ed..d4957da 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/app/AppConfig.java +++ b/src/main/java/org/filteredpush/bdq_workbench/app/AppConfig.java @@ -48,17 +48,19 @@ * @param datasetTable which of the input dataset's tables to run against — a Darwin Core * Archive's core or one of its extensions, or one of a Data Package's resources — named by * its location, resource name or Darwin Core row type; empty to let the ingestor choose + * @param datasetView optional path to a standalone dataset view JSON file */ public record AppConfig( - Path useCaseXml, - List rdfDefinitions, - Path datasetPath, - String useCaseId, - List implementationPackages, - int threadCount, - boolean dedupEnabled, - RecordFilterSpec recordFilter, - String datasetTable) { + Path useCaseXml, + List rdfDefinitions, + Path datasetPath, + String useCaseId, + List implementationPackages, + int threadCount, + boolean dedupEnabled, + RecordFilterSpec recordFilter, + String datasetTable, + String datasetView) { /** * Creates a configuration with no record filters. @@ -80,7 +82,7 @@ public AppConfig( int threadCount, boolean dedupEnabled) { this(useCaseXml, rdfDefinitions, datasetPath, useCaseId, implementationPackages, threadCount, dedupEnabled, - RecordFilterSpec.empty(), ""); + RecordFilterSpec.empty(), "", ""); } /** @@ -105,7 +107,24 @@ public AppConfig( boolean dedupEnabled, RecordFilterSpec recordFilter) { this(useCaseXml, rdfDefinitions, datasetPath, useCaseId, implementationPackages, threadCount, dedupEnabled, - recordFilter, ""); + recordFilter, "", ""); + } + + /** + * Creates a configuration with explicit table selection and no dataset view. + */ + public AppConfig( + Path useCaseXml, + List rdfDefinitions, + Path datasetPath, + String useCaseId, + List implementationPackages, + int threadCount, + boolean dedupEnabled, + RecordFilterSpec recordFilter, + String datasetTable) { + this(useCaseXml, rdfDefinitions, datasetPath, useCaseId, implementationPackages, threadCount, dedupEnabled, + recordFilter, datasetTable, ""); } /** @@ -115,5 +134,6 @@ public AppConfig( public AppConfig { recordFilter = recordFilter == null ? RecordFilterSpec.empty() : recordFilter; datasetTable = datasetTable == null ? "" : datasetTable.trim(); + datasetView = datasetView == null ? "" : datasetView.trim(); } } diff --git a/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchApplication.java b/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchApplication.java index 1e44d18..45abeaf 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchApplication.java +++ b/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchApplication.java @@ -191,7 +191,8 @@ private static AppConfig resolveDefaultUseCase(AppConfig config) { config.threadCount(), config.dedupEnabled(), config.recordFilter(), - config.datasetTable()); + config.datasetTable(), + config.datasetView()); } /** @@ -248,6 +249,7 @@ private static ParseResult parseArguments(String[] args) { String key = switch (arg) { case "--dataset" -> "bdq.dataset"; case "--dataset-table" -> "bdq.dataset.table"; + case "--dataset-view" -> "bdq.dataset.view"; case "--usecase-file" -> "bdq.usecase.file"; case "--rdf-files" -> "bdq.rdf.files"; case "--usecase-id" -> "bdq.usecase.id"; @@ -290,6 +292,7 @@ private static void renderUsage(PrintStream out) { out.println(" --dataset-table Which table of the dataset to run against, named by"); out.println(" location, resource name or Darwin Core row type"); out.println(" (default: the best-ranked table the dataset offers)"); + out.println(" --dataset-view Standalone dataset view JSON file"); out.println(" --usecase-file Use case XML file"); out.println(" --rdf-files Comma-separated RDF/OWL files"); out.println(" --usecase-id Optional use case identifier"); diff --git a/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGui.java b/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGui.java index a5cb807..e065e6a 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGui.java +++ b/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGui.java @@ -68,10 +68,17 @@ import org.filteredpush.bdq_workbench.execution.ExecutionProgressListener; import org.filteredpush.bdq_workbench.execution.ParallelPhaseExecutionService; import org.filteredpush.bdq_workbench.execution.ReflectionExecutionAdapter; +import org.filteredpush.bdq_workbench.ingest.DatasetViewIO; import org.filteredpush.bdq_workbench.ingest.DefaultIngestService; +import org.filteredpush.bdq_workbench.ingest.RelationalDatasetIngestor; import org.filteredpush.bdq_workbench.model.RecordFilterSummary; import org.filteredpush.bdq_workbench.model.BindingReview; import org.filteredpush.bdq_workbench.model.BuiltInMeasureSpec; +import org.filteredpush.bdq_workbench.model.DatasetSchema; +import org.filteredpush.bdq_workbench.model.DatasetView; +import org.filteredpush.bdq_workbench.model.DatasetViewCardinalityPolicy; +import org.filteredpush.bdq_workbench.model.DatasetViewJoin; +import org.filteredpush.bdq_workbench.model.DatasetViewMapping; import org.filteredpush.bdq_workbench.model.ExecutionPlan; import org.filteredpush.bdq_workbench.model.ExecutionSummary; import org.filteredpush.bdq_workbench.model.ImplementationBinding; @@ -305,6 +312,8 @@ private static JFrame createFrame(AppConfig defaults, Map overri form.add(setupHeaderRow); PickerField dataset = addPickerField(form, frame, "Dataset", defaults.datasetPath().toString()); + PickerField datasetView = addPickerField(form, frame, "Dataset view (JSON)", + overrides.getOrDefault("bdq.dataset.view", defaults.datasetView())); String[] configuredRecordFilters = new String[] {defaults.recordFilter().toPropertyString()}; JTextArea recordFilterSummary = new JTextArea(4, 40); recordFilterSummary.setEditable(false); @@ -317,8 +326,13 @@ private static JFrame createFrame(AppConfig defaults, Map overri recordFilterRow.add(new JLabel("Record filters"), BorderLayout.WEST); JPanel recordFilterButtons = new JPanel(new FlowLayout(FlowLayout.LEFT, 8, 0)); JButton buildRecordFilters = new JButton("Build Record Filters..."); + JButton buildDatasetView = new JButton("Build Dataset View..."); JButton clearRecordFilters = new JButton("Clear Filters"); + JButton clearDatasetView = new JButton("Clear View"); clearRecordFilters.setEnabled(!configuredRecordFilters[0].isBlank()); + clearDatasetView.setEnabled(!datasetView.field().getText().isBlank()); + recordFilterButtons.add(buildDatasetView); + recordFilterButtons.add(clearDatasetView); recordFilterButtons.add(buildRecordFilters); recordFilterButtons.add(clearRecordFilters); recordFilterRow.add(recordFilterButtons, BorderLayout.CENTER); @@ -485,6 +499,27 @@ private static JFrame createFrame(AppConfig defaults, Map overri recordFilterSummary, clearRecordFilters, buildRecordFilters)); + buildDatasetView.addActionListener(e -> loadDatasetViewDialog(frame, dataset.field().getText().trim(), datasetView.field())); + clearDatasetView.addActionListener(e -> { + datasetView.field().setText(""); + clearDatasetView.setEnabled(false); + }); + datasetView.field().getDocument().addDocumentListener(new javax.swing.event.DocumentListener() { + @Override + public void insertUpdate(javax.swing.event.DocumentEvent e) { + clearDatasetView.setEnabled(!datasetView.field().getText().isBlank()); + } + + @Override + public void removeUpdate(javax.swing.event.DocumentEvent e) { + clearDatasetView.setEnabled(!datasetView.field().getText().isBlank()); + } + + @Override + public void changedUpdate(javax.swing.event.DocumentEvent e) { + clearDatasetView.setEnabled(!datasetView.field().getText().isBlank()); + } + }); clearRecordFilters.addActionListener(e -> { configuredRecordFilters[0] = ""; updateRecordFilterSummary(recordFilterSummary, configuredRecordFilters[0]); @@ -536,6 +571,7 @@ protected PreflightState doInBackground() { selectedUseCaseId(useCaseChoice), configuredRecordFilters[0], datasetTable.getText().trim(), + datasetView.field().getText().trim(), useCaseSource.getText().trim(), testDefinitionsSource.getText().trim(), additionalTestDefinitions.getText().trim(), @@ -1395,6 +1431,7 @@ private static WorkbenchFacade createFacade( * form * @param datasetTable which of the dataset's tables to run against, named by location, * resource name or Darwin Core row type; blank to let the ingestor choose + * @param datasetView optional dataset-view JSON file path * @param useCaseSource use case RDF file/URL field value * @param testDefinitionsSource primary test definitions file/URL field value * @param additionalTestDefinitions comma-separated extra test definition files/URLs @@ -1411,6 +1448,7 @@ private static AppConfig buildConfig( String selectedUseCaseId, String recordFilters, String datasetTable, + String datasetView, String useCaseSource, String testDefinitionsSource, String additionalTestDefinitions, @@ -1446,7 +1484,8 @@ private static AppConfig buildConfig( parseThreads(threads), dedupEnabled, RecordFilterSpec.parse(recordFilters), - datasetTable); + datasetTable, + datasetView); } /** @@ -1785,6 +1824,147 @@ protected void done() { worker.execute(); } + /** + * Loads relational schema metadata and helps the user create/save a dataset view file. + * + * @param frame owner frame + * @param datasetPath selected dataset path + * @param datasetViewField dataset-view path field to update + */ + private static void loadDatasetViewDialog(JFrame frame, String datasetPath, JTextField datasetViewField) { + if (datasetPath == null || datasetPath.isBlank()) { + JOptionPane.showMessageDialog( + frame, + "Select a dataset before building a dataset view.", + "Dataset required", + JOptionPane.WARNING_MESSAGE); + return; + } + Path path = Path.of(datasetPath); + if (!Files.exists(path)) { + JOptionPane.showMessageDialog( + frame, + "Dataset input not found: " + datasetPath, + "Dataset required", + JOptionPane.ERROR_MESSAGE); + return; + } + RelationalDatasetIngestor ingestor = new RelationalDatasetIngestor(); + DatasetViewIO io = new DatasetViewIO(); + var relational = ingestor.ingest(path, ""); + DatasetSchema schema = relational.schema(); + DatasetView suggested = suggestDatasetView(schema); + var preview = new org.filteredpush.bdq_workbench.ingest.ViewFlattener().flatten(relational, suggested); + + JTextArea details = new JTextArea(18, 80); + details.setEditable(false); + details.setLineWrap(true); + details.setWrapStyleWord(true); + StringBuilder builder = new StringBuilder(); + builder.append("Schema fingerprint: ").append(schema.schemaFingerprint()).append('\n'); + builder.append("Tables:\n"); + schema.tables().forEach(table -> builder.append(" - ") + .append(table.name()) + .append(" [") + .append(table.rowType()) + .append("] columns=") + .append(table.columns().size()) + .append('\n')); + builder.append("Relationships:\n"); + schema.relationships().forEach(relationship -> builder.append(" - ") + .append(relationship.fromTable()) + .append('.') + .append(relationship.fromColumn()) + .append(" -> ") + .append(relationship.toTable()) + .append('.') + .append(relationship.toColumn()) + .append('\n')); + builder.append("Suggested mappings:\n"); + suggested.mappings().forEach(mapping -> builder.append(" - ") + .append(mapping.term()) + .append(" <- ") + .append(mapping.sourceTable()) + .append('.') + .append(mapping.sourceColumn()) + .append('\n')); + builder.append("Preview rows: ").append(Math.min(5, preview.dataset().records().size())).append('\n'); + preview.dataset().records().stream().limit(5).forEach(row -> builder.append(" - ") + .append(row.id()) + .append(" => ") + .append(row.terms()) + .append('\n')); + if (!preview.diagnostics().isEmpty()) { + builder.append("Cardinality warnings:\n"); + preview.diagnostics().forEach(message -> builder.append(" - ").append(message).append('\n')); + } + details.setText(builder.toString()); + + JButton save = new JButton("Save View..."); + JButton load = new JButton("Load View..."); + JPanel buttons = new JPanel(new FlowLayout(FlowLayout.RIGHT)); + buttons.add(load); + buttons.add(save); + JPanel panel = new JPanel(new BorderLayout(8, 8)); + panel.add(new JScrollPane(details), BorderLayout.CENTER); + panel.add(buttons, BorderLayout.SOUTH); + JDialog dialog = new JDialog(frame, "Build Dataset View", true); + dialog.setContentPane(panel); + dialog.pack(); + dialog.setLocationRelativeTo(frame); + + load.addActionListener(e -> { + String selected = chooseFile(frame, "Select dataset view JSON"); + if (selected == null) { + return; + } + DatasetView view = io.load(Path.of(selected)); + io.validateCompatibility(view, schema); + datasetViewField.setText(selected); + dialog.dispose(); + }); + save.addActionListener(e -> { + String selected = chooseSaveFile(frame, "Save dataset view JSON", "bdq-dataset-view.json"); + if (selected == null) { + return; + } + io.save(Path.of(selected), suggested); + datasetViewField.setText(selected); + dialog.dispose(); + }); + dialog.setVisible(true); + } + + /** + * Creates a simple direct-mapping view suggestion from discovered schema. + */ + private static DatasetView suggestDatasetView(DatasetSchema schema) { + String grain = schema.tables().stream() + .filter(table -> "OCCURRENCE".equalsIgnoreCase(table.rowType())) + .findFirst() + .map(table -> table.name()) + .orElse(schema.tables().isEmpty() ? "core" : schema.tables().get(0).name()); + List joins = schema.relationships().stream() + .filter(relationship -> relationship.toTable().equals(grain)) + .map(relationship -> new DatasetViewJoin( + relationship.relationName(), + relationship.fromTable(), + DatasetViewCardinalityPolicy.REJECT)) + .toList(); + List requestedTerms = List.of( + "occurrenceID", "scientificName", "eventDate", "decimalLatitude", "decimalLongitude"); + List mappings = new ArrayList<>(); + for (String term : requestedTerms) { + String sourceTable = schema.tables().stream() + .filter(table -> table.columns().contains(term)) + .findFirst() + .map(table -> table.name()) + .orElse("core"); + mappings.add(new DatasetViewMapping(term, sourceTable, term)); + } + return new DatasetView(grain, schema.schemaFingerprint(), joins, mappings); + } + /** * Opens the interactive record-filter dialog for one already-profiled dataset. * diff --git a/src/main/java/org/filteredpush/bdq_workbench/app/ConfigLoader.java b/src/main/java/org/filteredpush/bdq_workbench/app/ConfigLoader.java index 7c3439c..4a39a13 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/app/ConfigLoader.java +++ b/src/main/java/org/filteredpush/bdq_workbench/app/ConfigLoader.java @@ -123,7 +123,8 @@ private AppConfig load(Map overrides, CachedResourceResolver res parseThreadCount(getValue(defaults, overrides, "bdq.threads", "4")), parseBoolean(getValue(defaults, overrides, "bdq.execution.dedup", "true"), "bdq.execution.dedup"), RecordFilterSpec.parse(getValue(defaults, overrides, "bdq.record.filters", "")), - getValue(defaults, overrides, "bdq.dataset.table", "")); + getValue(defaults, overrides, "bdq.dataset.table", ""), + getValue(defaults, overrides, "bdq.dataset.view", "")); } /** diff --git a/src/main/java/org/filteredpush/bdq_workbench/app/WorkbenchFacade.java b/src/main/java/org/filteredpush/bdq_workbench/app/WorkbenchFacade.java index 794022d..306fa38 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/app/WorkbenchFacade.java +++ b/src/main/java/org/filteredpush/bdq_workbench/app/WorkbenchFacade.java @@ -159,7 +159,7 @@ public WorkbenchFacade( * @return the prepared run, ready for execution */ public PreparedRun prepare(AppConfig config) { - var ingestedDataset = ingestService.ingest(config.datasetPath(), config.datasetTable()); + var ingestedDataset = ingestService.ingest(config.datasetPath(), config.datasetTable(), config.datasetView()); RecordFilterSummary filterSummary = recordFilterService.apply(ingestedDataset, config.recordFilter()); var dataset = filterSummary.filteredDataset(); ExecutionPlan plan = policyResolverService.resolve(config.useCaseId()); diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/BuiltInDatasetViews.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/BuiltInDatasetViews.java new file mode 100644 index 0000000..719f497 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/BuiltInDatasetViews.java @@ -0,0 +1,120 @@ +/** BuiltInDatasetViews.java + * + * Built-in reusable dataset view definitions. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.ingest; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import org.filteredpush.bdq_workbench.model.DatasetSchema; +import org.filteredpush.bdq_workbench.model.DatasetView; +import org.filteredpush.bdq_workbench.model.DatasetViewCardinalityPolicy; +import org.filteredpush.bdq_workbench.model.DatasetViewJoin; +import org.filteredpush.bdq_workbench.model.DatasetViewMapping; +import org.filteredpush.bdq_workbench.model.RelationshipSchema; +import org.filteredpush.bdq_workbench.model.TableSchema; + +/** + * Selector for built-in default dataset views. + */ +final class BuiltInDatasetViews { + + private BuiltInDatasetViews() { + } + + /** + * Picks an applicable built-in view for the given schema, if any. + * + * @param schema discovered schema + * @param diagnostics receives applicability diagnostics + * @return built-in view when applicable + */ + static Optional select(DatasetSchema schema, List diagnostics) { + Optional eventOccurrence = eventCoreOccurrenceExtension(schema); + if (eventOccurrence.isPresent()) { + return eventOccurrence; + } + Optional dataPackage = dwcDataPackageOccurrenceView(schema); + if (dataPackage.isPresent()) { + return dataPackage; + } + diagnostics.add("No built-in dataset view matched the dataset schema fingerprint " + + schema.schemaFingerprint() + "; falling back to flat ingest"); + return Optional.empty(); + } + + private static Optional dwcDataPackageOccurrenceView(DatasetSchema schema) { + TableSchema occurrence = findByRowType(schema, "OCCURRENCE"); + if (occurrence == null) { + return Optional.empty(); + } + if (schema.relationships().isEmpty()) { + return Optional.empty(); + } + List joins = new ArrayList<>(); + for (RelationshipSchema relation : schema.relationships()) { + if (relation.toTable().equals(occurrence.name())) { + joins.add(new DatasetViewJoin(relation.relationName(), relation.fromTable(), + DatasetViewCardinalityPolicy.FIRST_ROW)); + } + } + if (joins.isEmpty()) { + return Optional.empty(); + } + List mappings = defaultOccurrenceMappings(occurrence.name(), joins); + return Optional.of(new DatasetView(occurrence.name(), schema.schemaFingerprint(), joins, mappings)); + } + + private static Optional eventCoreOccurrenceExtension(DatasetSchema schema) { + TableSchema event = findByRowType(schema, "EVENT"); + TableSchema occurrence = findByRowType(schema, "OCCURRENCE"); + if (event == null || occurrence == null) { + return Optional.empty(); + } + boolean hasEventToOccurrence = schema.relationships().stream() + .anyMatch(relation -> relation.toTable().equals(event.name()) + && relation.fromTable().equals(occurrence.name())); + if (!hasEventToOccurrence) { + return Optional.empty(); + } + List joins = List.of(new DatasetViewJoin( + occurrence.name(), + occurrence.name(), + DatasetViewCardinalityPolicy.FIRST_ROW)); + List mappings = defaultOccurrenceMappings(occurrence.name(), joins); + return Optional.of(new DatasetView(event.name(), schema.schemaFingerprint(), joins, mappings)); + } + + private static List defaultOccurrenceMappings(String occurrenceTable, List joins) { + String sourceTable = joins.isEmpty() ? "core" : occurrenceTable; + return List.of( + new DatasetViewMapping("occurrenceID", sourceTable, "occurrenceID"), + new DatasetViewMapping("scientificName", sourceTable, "scientificName"), + new DatasetViewMapping("eventDate", sourceTable, "eventDate"), + new DatasetViewMapping("decimalLatitude", sourceTable, "decimalLatitude"), + new DatasetViewMapping("decimalLongitude", sourceTable, "decimalLongitude")); + } + + private static TableSchema findByRowType(DatasetSchema schema, String rowType) { + return schema.tables().stream() + .filter(table -> table.rowType().equalsIgnoreCase(rowType)) + .findFirst() + .orElse(null); + } +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageDialectParser.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageDialectParser.java index 1a843b3..3444727 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageDialectParser.java +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageDialectParser.java @@ -183,7 +183,8 @@ static DataPackageResourceMeta parseResource(ObjectMapper mapper, JsonNode resou dialect.path("skipInitialSpace").asBoolean(false), resolveHeaderLines(dialect), columnNames, - readPrimaryKey(resource.path("schema"))); + readPrimaryKey(resource.path("schema")), + readForeignKeys(resource.path("schema"))); LOG.debug("Parsed data package resource '{}': paths={}, encoding={}, delimiter={}, quoteChar={}," + " escapeChar={}, headerLines={}, schemaColumns={}", meta.label(), meta.paths(), meta.encoding(), describe(meta.delimiter()), describe(meta.quoteChar()), @@ -317,6 +318,49 @@ private static List readSchemaFieldNames(JsonNode schema) { if (!fields.isArray()) { return names; } + + /** + * Reads single-column foreign-key declarations from a resource schema. + * + * @param schema the resource's {@code schema} property + * @return declared single-column foreign keys + */ + private static List readForeignKeys(JsonNode schema) { + List keys = new ArrayList<>(); + JsonNode foreignKeys = schema.path("foreignKeys"); + if (!foreignKeys.isArray()) { + return keys; + } + for (JsonNode key : foreignKeys) { + String field = firstText(key.path("fields")); + JsonNode reference = key.path("reference"); + String referenceField = firstText(reference.path("fields")); + if (field == null || referenceField == null) { + continue; + } + keys.add(new DataPackageForeignKey( + field.trim(), + text(reference, "resource") == null ? "" : text(reference, "resource").trim(), + referenceField.trim())); + } + return keys; + } + + /** + * Reads a string value from either a text node or a single-element text array. + * + * @param node node to inspect + * @return extracted text, or {@code null} + */ + private static String firstText(JsonNode node) { + if (node.isTextual()) { + return node.asText(null); + } + if (node.isArray() && node.size() == 1 && node.get(0).isTextual()) { + return node.get(0).asText(null); + } + return null; + } for (JsonNode field : fields) { String name = field.path("name").asText(null); if (name == null || name.isBlank()) { diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageForeignKey.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageForeignKey.java new file mode 100644 index 0000000..7154db5 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageForeignKey.java @@ -0,0 +1,30 @@ +/** DataPackageForeignKey.java + * + * One foreign-key declaration from a Data Package table schema. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.ingest; + +/** + * Single-column foreign-key metadata. + * + * @param field local child-table column + * @param referenceResource referenced resource name (blank means same table) + * @param referenceField referenced parent-table column + */ +public record DataPackageForeignKey(String field, String referenceResource, String referenceField) { +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageResourceMeta.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageResourceMeta.java index 734265b..2577d69 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageResourceMeta.java +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageResourceMeta.java @@ -48,6 +48,7 @@ * @param columnNames column names from the resource's table schema, empty when it declares none * @param idColumn the single-column primary key the resource's table schema declares, or * {@code ""} when it declares none or declares a composite one + * @param foreignKeys single-column foreign keys the table schema declares */ public record DataPackageResourceMeta( String name, @@ -62,7 +63,8 @@ public record DataPackageResourceMeta( boolean skipInitialSpace, int headerLines, List columnNames, - String idColumn) { + String idColumn, + List foreignKeys) { /** * Canonical constructor; copies the collection components defensively. @@ -73,6 +75,7 @@ public record DataPackageResourceMeta( paths = List.copyOf(paths); columnNames = List.copyOf(columnNames); idColumn = idColumn == null ? "" : idColumn; + foreignKeys = List.copyOf(foreignKeys); } /** diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/DatasetViewIO.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/DatasetViewIO.java new file mode 100644 index 0000000..68bef92 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/DatasetViewIO.java @@ -0,0 +1,75 @@ +/** DatasetViewIO.java + * + * JSON load/save helpers for standalone DatasetView files. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.ingest; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.nio.file.Path; +import org.filteredpush.bdq_workbench.app.AppException; +import org.filteredpush.bdq_workbench.model.DatasetSchema; +import org.filteredpush.bdq_workbench.model.DatasetView; + +/** + * Loads and saves standalone dataset view JSON files. + */ +public class DatasetViewIO { + private final ObjectMapper mapper = new ObjectMapper(); + + /** + * Reads a dataset view JSON file. + * + * @param path view file path + * @return parsed dataset view + */ + public DatasetView load(Path path) { + try { + return mapper.readValue(path.toFile(), DatasetView.class); + } catch (IOException e) { + throw new AppException("Unable to read dataset view file " + path, e); + } + } + + /** + * Writes a dataset view JSON file. + * + * @param path view file path + * @param view view to persist + */ + public void save(Path path, DatasetView view) { + try { + mapper.writerWithDefaultPrettyPrinter().writeValue(path.toFile(), view); + } catch (IOException e) { + throw new AppException("Unable to save dataset view file " + path, e); + } + } + + /** + * Verifies a view is compatible with the current schema fingerprint. + * + * @param view loaded view + * @param schema current schema + */ + public void validateCompatibility(DatasetView view, DatasetSchema schema) { + if (!view.schemaFingerprint().equals(schema.schemaFingerprint())) { + throw new AppException("Dataset view fingerprint does not match dataset schema fingerprint: view=" + + view.schemaFingerprint() + ", dataset=" + schema.schemaFingerprint()); + } + } +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/DefaultIngestService.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/DefaultIngestService.java index d05036e..f39abf3 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/ingest/DefaultIngestService.java +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/DefaultIngestService.java @@ -20,8 +20,14 @@ package org.filteredpush.bdq_workbench.ingest; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; import org.filteredpush.bdq_workbench.app.AppException; +import org.filteredpush.bdq_workbench.model.DatasetView; import org.filteredpush.bdq_workbench.model.RecordDataset; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Dispatches ingestion based on source format. @@ -31,15 +37,20 @@ * {@code datapackage} Darwin Core Data Packages. */ public class DefaultIngestService implements IngestService { + private static final Logger LOG = LoggerFactory.getLogger(DefaultIngestService.class); private final DwcArchiveIngestor dwcArchiveIngestor; private final DataPackageIngestor dataPackageIngestor; + private final RelationalDatasetIngestor relationalDatasetIngestor; + private final DatasetViewIO datasetViewIO; + private final ViewFlattener viewFlattener; /** * Creates a service with default {@link DwcArchiveIngestor} and {@link DataPackageIngestor} * instances. */ public DefaultIngestService() { - this(new DwcArchiveIngestor(), new DataPackageIngestor()); + this(new DwcArchiveIngestor(), new DataPackageIngestor(), + new RelationalDatasetIngestor(), new DatasetViewIO(), new ViewFlattener()); } /** @@ -49,8 +60,24 @@ public DefaultIngestService() { * @param dataPackageIngestor ingestor used for {@code .json}/{@code datapackage} inputs */ public DefaultIngestService(DwcArchiveIngestor dwcArchiveIngestor, DataPackageIngestor dataPackageIngestor) { + this(dwcArchiveIngestor, dataPackageIngestor, + new RelationalDatasetIngestor(), new DatasetViewIO(), new ViewFlattener()); + } + + /** + * Creates a service wired to explicit flat and relational ingestion components. + */ + public DefaultIngestService( + DwcArchiveIngestor dwcArchiveIngestor, + DataPackageIngestor dataPackageIngestor, + RelationalDatasetIngestor relationalDatasetIngestor, + DatasetViewIO datasetViewIO, + ViewFlattener viewFlattener) { this.dwcArchiveIngestor = dwcArchiveIngestor; this.dataPackageIngestor = dataPackageIngestor; + this.relationalDatasetIngestor = relationalDatasetIngestor; + this.datasetViewIO = datasetViewIO; + this.viewFlattener = viewFlattener; } /** @@ -80,6 +107,22 @@ public RecordDataset ingest(Path inputPath) { */ @Override public RecordDataset ingest(Path inputPath, String requestedTable) { + return ingest(inputPath, requestedTable, ""); + } + + @Override + public RecordDataset ingest(Path inputPath, String requestedTable, String datasetView) { + if (datasetView != null && !datasetView.isBlank()) { + return ingestThroughView(inputPath, requestedTable, datasetView); + } + RecordDataset maybeBuiltIn = ingestThroughBuiltInView(inputPath, requestedTable); + if (maybeBuiltIn != null) { + return maybeBuiltIn; + } + return ingestFlat(inputPath, requestedTable); + } + + private RecordDataset ingestFlat(Path inputPath, String requestedTable) { String fileName = inputPath.getFileName().toString().toLowerCase(); if (fileName.endsWith(".zip")) { return dwcArchiveIngestor.ingest(inputPath, requestedTable); @@ -89,4 +132,35 @@ public RecordDataset ingest(Path inputPath, String requestedTable) { } throw new AppException("Unsupported dataset input: " + inputPath); } + + private RecordDataset ingestThroughView(Path inputPath, String requestedTable, String datasetViewPath) { + RelationalIngestResult relational = relationalDatasetIngestor.ingest(inputPath, requestedTable); + DatasetView view = datasetViewIO.load(Path.of(datasetViewPath)); + datasetViewIO.validateCompatibility(view, relational.schema()); + ViewFlattenResult flattened = viewFlattener.flatten(relational, view); + logDiagnostics(relational.diagnostics(), flattened.diagnostics()); + return flattened.dataset(); + } + + private RecordDataset ingestThroughBuiltInView(Path inputPath, String requestedTable) { + RelationalIngestResult relational = relationalDatasetIngestor.ingest(inputPath, requestedTable); + if (relational.graphs().isEmpty()) { + return null; + } + List diagnostics = new ArrayList<>(); + Optional builtIn = BuiltInDatasetViews.select(relational.schema(), diagnostics); + if (builtIn.isEmpty()) { + logDiagnostics(relational.diagnostics(), diagnostics); + return null; + } + ViewFlattenResult flattened = viewFlattener.flatten(relational, builtIn.get()); + logDiagnostics(relational.diagnostics(), diagnostics, flattened.diagnostics()); + return flattened.dataset(); + } + + private void logDiagnostics(List... groups) { + for (List group : groups) { + group.forEach(message -> LOG.warn("Dataset view diagnostic: {}", message)); + } + } } diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/DwcArchiveCoreMeta.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/DwcArchiveCoreMeta.java index c9c2e03..7f7b169 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/ingest/DwcArchiveCoreMeta.java +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/DwcArchiveCoreMeta.java @@ -47,6 +47,8 @@ * @param columnNames column names by zero-based column index, with gaps filled by placeholders * @param idColumn the name of the column the archive declares as this table's record * identifier, or {@code ""} when it declares none + * @param coreIdColumn the name of the extension column declaring the related core record + * identifier, or {@code ""} when this table is a core or declares none * @param constantTerms term values declared in {@code meta.xml} as defaults for columns that are * absent from the data files, applied to every record */ @@ -59,6 +61,7 @@ public record DwcArchiveCoreMeta( int ignoreHeaderLines, List columnNames, String idColumn, + String coreIdColumn, Map constantTerms) { /** @@ -69,6 +72,7 @@ public record DwcArchiveCoreMeta( locations = List.copyOf(locations); columnNames = List.copyOf(columnNames); idColumn = idColumn == null ? "" : idColumn; + coreIdColumn = coreIdColumn == null ? "" : coreIdColumn; constantTerms = Map.copyOf(new LinkedHashMap<>(constantTerms)); } } diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/DwcArchiveMetaParser.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/DwcArchiveMetaParser.java index a3b58a5..e35c24a 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/ingest/DwcArchiveMetaParser.java +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/DwcArchiveMetaParser.java @@ -229,6 +229,10 @@ private static Optional buildCoreMeta(Element core) { if (idIndex >= 0) { namesByIndex.putIfAbsent(idIndex, "id"); } + int coreIdIndex = parseInt(attribute(firstChildElement(core, "coreid"), "index"), -1); + if (coreIdIndex >= 0) { + namesByIndex.putIfAbsent(coreIdIndex, "coreid"); + } DwcArchiveCoreMeta meta = new DwcArchiveCoreMeta( attributeOrDefault(core, "rowType", ""), locations, @@ -238,6 +242,7 @@ private static Optional buildCoreMeta(Element core) { Math.max(0, parseInt(attribute(core, "ignoreHeaderLines"), 0)), toColumnNames(namesByIndex), idIndex >= 0 ? namesByIndex.getOrDefault(idIndex, "") : "", + coreIdIndex >= 0 ? namesByIndex.getOrDefault(coreIdIndex, "") : "", constantTerms); LOG.debug("Parsed meta.xml table: rowType={}, locations={}, encoding={}, delimiter={}, enclosedBy={}, " + "ignoreHeaderLines={}, columns={}, constantTerms={}", diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/IngestService.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/IngestService.java index 900e1c3..019c587 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/ingest/IngestService.java +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/IngestService.java @@ -56,4 +56,16 @@ public interface IngestService { default RecordDataset ingest(Path inputPath, String requestedTable) { return ingest(inputPath); } + + /** + * Ingests the dataset at the given input path, optionally applying a dataset-view definition. + * + * @param inputPath path to dataset input file + * @param requestedTable optional requested table name + * @param datasetView optional dataset view JSON path + * @return ingested (possibly view-flattened) dataset + */ + default RecordDataset ingest(Path inputPath, String requestedTable, String datasetView) { + return ingest(inputPath, requestedTable); + } } diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/RelationalDatasetIngestor.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/RelationalDatasetIngestor.java new file mode 100644 index 0000000..b66953a --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/RelationalDatasetIngestor.java @@ -0,0 +1,196 @@ +/** RelationalDatasetIngestor.java + * + * Builds RecordGraph structures from DwC-A/Data Package inputs. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.ingest; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipFile; +import org.filteredpush.bdq_workbench.app.AppException; +import org.filteredpush.bdq_workbench.model.CanonicalRecord; +import org.filteredpush.bdq_workbench.model.DatasetSchema; +import org.filteredpush.bdq_workbench.model.RecordGraph; +import org.filteredpush.bdq_workbench.model.RelationshipSchema; +import org.filteredpush.bdq_workbench.model.SourceCell; +import org.filteredpush.bdq_workbench.model.TableSchema; + +/** + * Relational ingestion for non-flat datasets. + */ +public class RelationalDatasetIngestor { + private final DwcArchiveIngestor dwcArchiveIngestor = new DwcArchiveIngestor(); + private final DataPackageIngestor dataPackageIngestor = new DataPackageIngestor(); + private final ObjectMapper mapper = new ObjectMapper(); + + /** + * Ingests a dataset and emits relational graphs plus schema metadata. + * + * @param inputPath dataset path + * @param requestedTable optional preferred core/grain table + * @return relational ingest output + */ + public RelationalIngestResult ingest(Path inputPath, String requestedTable) { + String fileName = inputPath.getFileName().toString().toLowerCase(); + if (fileName.endsWith(".zip")) { + return ingestDwcArchive(inputPath, requestedTable); + } + if (fileName.endsWith(".json") || fileName.endsWith("datapackage")) { + return ingestDataPackage(inputPath, requestedTable); + } + throw new AppException("Unsupported dataset input: " + inputPath); + } + + private RelationalIngestResult ingestDwcArchive(Path inputPath, String requestedTable) { + try (ZipFile zipFile = new ZipFile(inputPath.toFile())) { + List> tables = DwcArchiveMetaParser.parseTables(zipFile); + if (tables.isEmpty()) { + return new RelationalIngestResult(List.of(), new DatasetSchema(List.of(), List.of(), ""), List.of()); + } + CoreTableCandidate selected = CoreTableSelector.select(tables, requestedTable).selected(); + Map> rowsByTable = new LinkedHashMap<>(); + for (CoreTableCandidate table : tables) { + List rows = dwcArchiveIngestor.ingest(inputPath, table.label()).records().stream() + .map(row -> withTableProvenance(row, table.label())) + .toList(); + rowsByTable.put(table.label(), rows); + } + List relationships = tables.stream() + .filter(table -> !table.descriptor().coreIdColumn().isBlank()) + .map(table -> new RelationshipSchema( + table.label(), + table.descriptor().coreIdColumn(), + selected.label(), + selected.descriptor().idColumn().isBlank() + ? selected.rowType().identifierTerm() + : selected.descriptor().idColumn(), + table.label())) + .toList(); + return assembleResult(selected.label(), rowsByTable, tables.stream() + .map(table -> new TableSchema( + table.label(), + table.label(), + table.rowType().name(), + table.descriptor().idColumn().isBlank() + ? table.rowType().identifierTerm() + : table.descriptor().idColumn(), + table.descriptor().columnNames())) + .toList(), relationships); + } catch (IOException e) { + throw new AppException("Failed relational ingest for DwC-A " + inputPath, e); + } + } + + private RelationalIngestResult ingestDataPackage(Path inputPath, String requestedTable) { + try { + JsonNode root = mapper.readTree(Files.newBufferedReader(inputPath)); + Path packageDir = inputPath.toAbsolutePath().getParent(); + List> tables = + DataPackageDialectParser.parseResources(mapper, root, packageDir); + if (tables.isEmpty()) { + return new RelationalIngestResult(List.of(), new DatasetSchema(List.of(), List.of(), ""), List.of()); + } + CoreTableCandidate selected = CoreTableSelector.select(tables, requestedTable).selected(); + Map> rowsByTable = new LinkedHashMap<>(); + for (CoreTableCandidate table : tables) { + List rows = dataPackageIngestor.ingest(inputPath, table.label()).records().stream() + .map(row -> withTableProvenance(row, table.label())) + .toList(); + rowsByTable.put(table.label(), rows); + } + List relationships = new ArrayList<>(); + for (CoreTableCandidate table : tables) { + for (DataPackageForeignKey key : table.descriptor().foreignKeys()) { + String refResource = key.referenceResource().isBlank() + ? table.descriptor().name() + : key.referenceResource(); + if (!refResource.equalsIgnoreCase(selected.descriptor().name())) { + continue; + } + relationships.add(new RelationshipSchema( + table.label(), + key.field(), + selected.label(), + key.referenceField(), + table.label())); + } + } + return assembleResult(selected.label(), rowsByTable, tables.stream() + .map(table -> new TableSchema( + table.label(), + table.label(), + table.rowType().name(), + table.descriptor().idColumn().isBlank() + ? table.rowType().identifierTerm() + : table.descriptor().idColumn(), + table.descriptor().columnNames())) + .toList(), relationships); + } catch (IOException e) { + throw new AppException("Failed relational ingest for Data Package " + inputPath, e); + } + } + + private RelationalIngestResult assembleResult(String coreTable, Map> rowsByTable, + List tables, List relationships) { + List diagnostics = new ArrayList<>(); + Map>> relatedByTableByCoreId = new LinkedHashMap<>(); + for (RelationshipSchema relation : relationships) { + Map> byCore = new LinkedHashMap<>(); + for (CanonicalRecord related : rowsByTable.getOrDefault(relation.fromTable(), List.of())) { + String key = related.terms().getOrDefault(relation.fromColumn(), ""); + byCore.computeIfAbsent(key, ignored -> new ArrayList<>()).add(related); + } + relatedByTableByCoreId.put(relation.relationName(), byCore); + } + List graphs = new ArrayList<>(); + for (CanonicalRecord core : rowsByTable.getOrDefault(coreTable, List.of())) { + Map> relatedByRelation = new LinkedHashMap<>(); + for (RelationshipSchema relation : relationships) { + String coreValue = core.terms().getOrDefault(relation.toColumn(), ""); + if (coreValue.isBlank()) { + continue; + } + List related = relatedByTableByCoreId + .getOrDefault(relation.relationName(), Map.of()) + .getOrDefault(coreValue, List.of()); + if (related.isEmpty()) { + diagnostics.add("No related rows found for relation " + relation.relationName() + + " and core record " + core.id()); + } + relatedByRelation.put(relation.relationName(), related); + } + graphs.add(new RecordGraph(core, relatedByRelation)); + } + String fingerprint = SchemaFingerprint.of(tables); + return new RelationalIngestResult(graphs, new DatasetSchema(tables, relationships, fingerprint), diagnostics); + } + + private CanonicalRecord withTableProvenance(CanonicalRecord row, String tableName) { + Map> provenance = new LinkedHashMap<>(); + row.terms().forEach((term, value) -> provenance.put(term, List.of( + new SourceCell(tableName, tableName, row.id(), term, term)))); + return new CanonicalRecord(row.id(), row.terms(), provenance); + } +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/RelationalIngestResult.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/RelationalIngestResult.java new file mode 100644 index 0000000..1a63ee1 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/RelationalIngestResult.java @@ -0,0 +1,45 @@ +/** RelationalIngestResult.java + * + * Result of relational dataset ingest plus schema discovery. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.ingest; + +import java.util.List; +import org.filteredpush.bdq_workbench.model.DatasetSchema; +import org.filteredpush.bdq_workbench.model.RecordGraph; + +/** + * Relational ingest output. + * + * @param graphs core-record graphs in deterministic row order + * @param schema discovered schema metadata and fingerprint + * @param diagnostics non-fatal ingest diagnostics + */ +public record RelationalIngestResult( + List graphs, + DatasetSchema schema, + List diagnostics) { + + /** + * Canonical constructor; copies list components defensively. + */ + public RelationalIngestResult { + graphs = List.copyOf(graphs); + diagnostics = List.copyOf(diagnostics); + } +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/SchemaFingerprint.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/SchemaFingerprint.java new file mode 100644 index 0000000..cca9896 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/SchemaFingerprint.java @@ -0,0 +1,60 @@ +/** SchemaFingerprint.java + * + * Deterministic schema fingerprint helper. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.ingest; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Comparator; +import java.util.HexFormat; +import java.util.stream.Collectors; +import org.filteredpush.bdq_workbench.model.TableSchema; + +/** + * Computes a stable schema-shape fingerprint. + */ +final class SchemaFingerprint { + + private SchemaFingerprint() { + } + + /** + * Builds a fingerprint from table names, row types and column-name sets. + * + * @param tables discovered tables + * @return fingerprint hash + */ + static String of(Iterable tables) { + String canonical = java.util.stream.StreamSupport.stream(tables.spliterator(), false) + .sorted(Comparator.comparing(TableSchema::name)) + .map(table -> table.name().toLowerCase() + + "|" + + table.rowType().toLowerCase() + + "|" + + table.columns().stream().map(String::toLowerCase).sorted().collect(Collectors.joining(","))) + .collect(Collectors.joining("||")); + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + return HexFormat.of().formatHex(digest.digest(canonical.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 not available", e); + } + } +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/ViewFlattenResult.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/ViewFlattenResult.java new file mode 100644 index 0000000..c86d4bd --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/ViewFlattenResult.java @@ -0,0 +1,39 @@ +/** ViewFlattenResult.java + * + * Flattening output with diagnostics. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.ingest; + +import java.util.List; +import org.filteredpush.bdq_workbench.model.RecordDataset; + +/** + * Flattened dataset and diagnostics. + * + * @param dataset flattened dataset + * @param diagnostics non-fatal flattening diagnostics + */ +public record ViewFlattenResult(RecordDataset dataset, List diagnostics) { + + /** + * Canonical constructor; copies diagnostics defensively. + */ + public ViewFlattenResult { + diagnostics = List.copyOf(diagnostics); + } +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/ViewFlattener.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/ViewFlattener.java new file mode 100644 index 0000000..22c1fc9 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/ViewFlattener.java @@ -0,0 +1,130 @@ +/** ViewFlattener.java + * + * Flattens relational record graphs with a reusable DatasetView. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.ingest; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.filteredpush.bdq_workbench.model.CanonicalRecord; +import org.filteredpush.bdq_workbench.model.DatasetView; +import org.filteredpush.bdq_workbench.model.DatasetViewCardinalityPolicy; +import org.filteredpush.bdq_workbench.model.DatasetViewJoin; +import org.filteredpush.bdq_workbench.model.DatasetViewMapping; +import org.filteredpush.bdq_workbench.model.RecordDataset; +import org.filteredpush.bdq_workbench.model.RecordGraph; +import org.filteredpush.bdq_workbench.model.SourceCell; + +/** + * Applies a dataset view to relational graphs and emits flat canonical records. + */ +public class ViewFlattener { + + /** + * Flattens relational graphs according to a reusable dataset view. + * + * @param relational relational ingest result + * @param view view definition to apply + * @return flattened dataset and non-fatal diagnostics + */ + public ViewFlattenResult flatten(RelationalIngestResult relational, DatasetView view) { + List diagnostics = new ArrayList<>(); + List flattened = new ArrayList<>(); + for (RecordGraph graph : relational.graphs()) { + Map terms = new LinkedHashMap<>(); + Map> provenance = new LinkedHashMap<>(); + for (DatasetViewMapping mapping : view.mappings()) { + ValueSelection selected = selectValue(graph, mapping, view.joins(), diagnostics); + terms.put(mapping.term(), selected.value()); + if (!selected.cells().isEmpty()) { + provenance.put(mapping.term(), selected.cells()); + } + } + flattened.add(new CanonicalRecord(graph.core().id(), terms, provenance)); + } + return new ViewFlattenResult(new RecordDataset(flattened), diagnostics); + } + + /** + * Resolves one mapping from either the core row or one related relation. + */ + private ValueSelection selectValue(RecordGraph graph, DatasetViewMapping mapping, List joins, + List diagnostics) { + if (mapping.sourceTable().equalsIgnoreCase("core")) { + String value = graph.core().terms().getOrDefault(mapping.sourceColumn(), ""); + return new ValueSelection(value, sourceCells(graph.core(), "core", mapping.sourceColumn(), mapping.term())); + } + DatasetViewJoin join = joins.stream() + .filter(candidate -> candidate.sourceTable().equals(mapping.sourceTable())) + .findFirst() + .orElse(null); + if (join == null) { + diagnostics.add("View mapping for term " + mapping.term() + " references source table " + + mapping.sourceTable() + " but the view has no join for it"); + return ValueSelection.empty(); + } + List related = graph.relatedByRelation().getOrDefault(join.relationName(), List.of()); + if (related.isEmpty()) { + return ValueSelection.empty(); + } + List values = related.stream() + .map(row -> row.terms().getOrDefault(mapping.sourceColumn(), "")) + .toList(); + List cells = related.stream() + .flatMap(row -> sourceCells(row, join.sourceTable(), mapping.sourceColumn(), mapping.term()).stream()) + .toList(); + if (related.size() > 1 && join.cardinalityPolicy() == DatasetViewCardinalityPolicy.REJECT) { + diagnostics.add("Cardinality conflict for relation " + join.relationName() + " on record " + + graph.core().id() + " while mapping term " + mapping.term()); + return ValueSelection.empty(); + } + if (join.cardinalityPolicy() == DatasetViewCardinalityPolicy.FIRST_ROW) { + return new ValueSelection(values.get(0), cells.isEmpty() ? List.of() : List.of(cells.get(0))); + } + if (join.cardinalityPolicy() == DatasetViewCardinalityPolicy.AGGREGATE && values.size() > 1) { + String aggregated = values.stream() + .filter(value -> !value.isBlank()) + .collect(Collectors.joining(" | ")); + return new ValueSelection(aggregated, cells); + } + return new ValueSelection(values.get(0), cells); + } + + /** + * Builds source-cell provenance for one value read from one row. + */ + private List sourceCells(CanonicalRecord row, String sourceTable, String sourceColumn, String term) { + List existing = row.provenanceByTerm().get(sourceColumn); + if (existing != null && !existing.isEmpty()) { + return existing.stream() + .map(cell -> new SourceCell(cell.table(), cell.sourceLocation(), cell.rowRef(), sourceColumn, term)) + .toList(); + } + return List.of(new SourceCell(sourceTable, sourceTable, row.id(), sourceColumn, term)); + } + + private record ValueSelection(String value, List cells) { + + private static ValueSelection empty() { + return new ValueSelection("", List.of()); + } + } +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/model/CanonicalRecord.java b/src/main/java/org/filteredpush/bdq_workbench/model/CanonicalRecord.java index 889d18c..e375457 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/model/CanonicalRecord.java +++ b/src/main/java/org/filteredpush/bdq_workbench/model/CanonicalRecord.java @@ -20,6 +20,8 @@ package org.filteredpush.bdq_workbench.model; import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; /** @@ -30,8 +32,9 @@ * can update term values in place as a run progresses. */ public final class CanonicalRecord { - private final String id; - private final Map terms; + private final String id; + private final Map terms; + private final Map> provenanceByTerm; /** * Creates a canonical record, copying the supplied terms into a new mutable map. @@ -39,35 +42,58 @@ public final class CanonicalRecord { * @param id the record's identifier * @param terms the record's initial Darwin Core term values, keyed by term name */ - public CanonicalRecord(String id, Map terms) { - this.id = id; - this.terms = new HashMap<>(terms); - } + public CanonicalRecord(String id, Map terms) { + this(id, terms, Map.of()); + } + + /** + * Creates a canonical record with value-level source provenance. + * + * @param id the record's identifier + * @param terms the record's Darwin Core term values, keyed by term name + * @param provenanceByTerm provenance cells for each term value + */ + public CanonicalRecord(String id, Map terms, Map> provenanceByTerm) { + this.id = id; + this.terms = new HashMap<>(terms); + Map> copy = new LinkedHashMap<>(); + provenanceByTerm.forEach((term, cells) -> copy.put(term, List.copyOf(cells))); + this.provenanceByTerm = Map.copyOf(copy); + } /** * Returns this record's identifier. * * @return the record ID */ - public String id() { - return id; - } + public String id() { + return id; + } /** * Returns this record's Darwin Core term values. * * @return the mutable map of term name to value backing this record */ - public Map terms() { - return terms; - } + public Map terms() { + return terms; + } + + /** + * Returns source provenance for this record's term values. + * + * @return immutable map of term name to one or more source cells + */ + public Map> provenanceByTerm() { + return provenanceByTerm; + } /** * Creates an independent copy of this record, with its own copy of the terms map. * * @return a new {@code CanonicalRecord} with the same ID and a copy of the current term values */ - public CanonicalRecord copy() { - return new CanonicalRecord(id, terms); - } + public CanonicalRecord copy() { + return new CanonicalRecord(id, terms, provenanceByTerm); + } } diff --git a/src/main/java/org/filteredpush/bdq_workbench/model/DatasetSchema.java b/src/main/java/org/filteredpush/bdq_workbench/model/DatasetSchema.java new file mode 100644 index 0000000..1657b44 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/model/DatasetSchema.java @@ -0,0 +1,43 @@ +/** DatasetSchema.java + * + * Relational table/relationship metadata discovered during ingest. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.model; + +import java.util.List; + +/** + * Discovered schema used by dataset views. + * + * @param tables discovered tables/resources + * @param relationships discovered table relationships + * @param schemaFingerprint deterministic schema-shape fingerprint + */ +public record DatasetSchema( + List tables, + List relationships, + String schemaFingerprint) { + + /** + * Canonical constructor; copies list components defensively. + */ + public DatasetSchema { + tables = List.copyOf(tables); + relationships = List.copyOf(relationships); + } +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/model/DatasetView.java b/src/main/java/org/filteredpush/bdq_workbench/model/DatasetView.java new file mode 100644 index 0000000..4733819 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/model/DatasetView.java @@ -0,0 +1,45 @@ +/** DatasetView.java + * + * Standalone JSON-serializable dataset view definition. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.model; + +import java.util.List; + +/** + * Reusable flattening view. + * + * @param grainTable table whose rows define flattened output records + * @param schemaFingerprint required schema fingerprint compatibility key + * @param joins ordered joins to include while flattening + * @param mappings direct term-to-column mappings + */ +public record DatasetView( + String grainTable, + String schemaFingerprint, + List joins, + List mappings) { + + /** + * Canonical constructor; copies list components defensively. + */ + public DatasetView { + joins = List.copyOf(joins); + mappings = List.copyOf(mappings); + } +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/model/DatasetViewCardinalityPolicy.java b/src/main/java/org/filteredpush/bdq_workbench/model/DatasetViewCardinalityPolicy.java new file mode 100644 index 0000000..2850094 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/model/DatasetViewCardinalityPolicy.java @@ -0,0 +1,34 @@ +/** DatasetViewCardinalityPolicy.java + * + * Cardinality handling choices when flattening related rows. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.model; + +/** + * Per-join policy for handling multiple related rows. + */ +public enum DatasetViewCardinalityPolicy { + /** Concatenate all matching row values in deterministic related-row order using {@code " | "}. */ + AGGREGATE, + + /** Use only the first matching row value in deterministic related-row order. */ + FIRST_ROW, + + /** Keep flattening but emit a diagnostic when multiple related rows are present. */ + REJECT +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/model/DatasetViewJoin.java b/src/main/java/org/filteredpush/bdq_workbench/model/DatasetViewJoin.java new file mode 100644 index 0000000..305fc15 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/model/DatasetViewJoin.java @@ -0,0 +1,33 @@ +/** DatasetViewJoin.java + * + * Join definition for dataset view flattening. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.model; + +/** + * One ordered relationship in a dataset view. + * + * @param relationName relation key from {@link RecordGraph#relatedByRelation()} + * @param sourceTable source table name for this join + * @param cardinalityPolicy multiple-row handling policy + */ +public record DatasetViewJoin( + String relationName, + String sourceTable, + DatasetViewCardinalityPolicy cardinalityPolicy) { +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/model/DatasetViewMapping.java b/src/main/java/org/filteredpush/bdq_workbench/model/DatasetViewMapping.java new file mode 100644 index 0000000..b873b0c --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/model/DatasetViewMapping.java @@ -0,0 +1,30 @@ +/** DatasetViewMapping.java + * + * Direct term-to-column mapping in a dataset view. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.model; + +/** + * One direct term mapping in a dataset view. + * + * @param term Darwin Core term in the flattened output + * @param sourceTable source table name + * @param sourceColumn source table column + */ +public record DatasetViewMapping(String term, String sourceTable, String sourceColumn) { +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/model/RecordGraph.java b/src/main/java/org/filteredpush/bdq_workbench/model/RecordGraph.java new file mode 100644 index 0000000..90e9587 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/model/RecordGraph.java @@ -0,0 +1,42 @@ +/** RecordGraph.java + * + * Non-flat record model with a core record and related rows. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.model; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * One relationally-connected record graph. + * + * @param core the core record for the graph + * @param relatedByRelation related subrecords keyed by relation/table name + */ +public record RecordGraph(CanonicalRecord core, Map> relatedByRelation) { + + /** + * Canonical constructor; copies relation lists defensively. + */ + public RecordGraph { + Map> copy = new LinkedHashMap<>(); + relatedByRelation.forEach((relation, records) -> copy.put(relation, List.copyOf(records))); + relatedByRelation = Map.copyOf(copy); + } +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/model/RelationshipSchema.java b/src/main/java/org/filteredpush/bdq_workbench/model/RelationshipSchema.java new file mode 100644 index 0000000..9462f54 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/model/RelationshipSchema.java @@ -0,0 +1,37 @@ +/** RelationshipSchema.java + * + * Relationship metadata between two tables. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.model; + +/** + * One directed table relationship. + * + * @param fromTable related/child table + * @param fromColumn related-table foreign-key column + * @param toTable referenced/parent table + * @param toColumn referenced-table identifier column + * @param relationName relation key used in {@link RecordGraph#relatedByRelation()} + */ +public record RelationshipSchema( + String fromTable, + String fromColumn, + String toTable, + String toColumn, + String relationName) { +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/model/SourceCell.java b/src/main/java/org/filteredpush/bdq_workbench/model/SourceCell.java new file mode 100644 index 0000000..c029dfd --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/model/SourceCell.java @@ -0,0 +1,37 @@ +/** SourceCell.java + * + * Provenance for one value in a flattened record. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.model; + +/** + * Source location of a flattened term value. + * + * @param table the source table/resource label + * @param sourceLocation the source file or manifest path for the table + * @param rowRef deterministic source row reference within the table + * @param column the source column name + * @param term the flattened Darwin Core term that took this value + */ +public record SourceCell( + String table, + String sourceLocation, + String rowRef, + String column, + String term) { +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/model/TableSchema.java b/src/main/java/org/filteredpush/bdq_workbench/model/TableSchema.java new file mode 100644 index 0000000..883dd47 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/model/TableSchema.java @@ -0,0 +1,46 @@ +/** TableSchema.java + * + * Table discovery metadata. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.model; + +import java.util.List; + +/** + * One discovered table/resource shape. + * + * @param name stable table name used in view definitions + * @param label human-readable label + * @param rowType detected Darwin Core row type name + * @param identifierColumn row identifier column, if known + * @param columns declared column names + */ +public record TableSchema( + String name, + String label, + String rowType, + String identifierColumn, + List columns) { + + /** + * Canonical constructor; copies columns defensively. + */ + public TableSchema { + columns = List.copyOf(columns); + } +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 5923315..cceb18a 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -5,6 +5,7 @@ bdq.usecase.file= bdq.rdf.files= bdq.dataset=dataset.zip bdq.dataset.table= +bdq.dataset.view= bdq.usecase.id= bdq.discovery.packages=org.filteredpush bdq.threads=4 diff --git a/src/test/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchApplicationTest.java b/src/test/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchApplicationTest.java index b554802..3c1b66a 100644 --- a/src/test/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchApplicationTest.java +++ b/src/test/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchApplicationTest.java @@ -131,7 +131,7 @@ void helpDocumentsTheGuiFlag() { int exitCode = BdqWorkbenchApplication.run(new String[] {"--help"}, printStream(out), printStream(err)); assertThat(exitCode).isZero(); - assertThat(out.toString()).contains("--gui").contains("--dataset-table "); + assertThat(out.toString()).contains("--gui").contains("--dataset-table ").contains("--dataset-view "); } /** diff --git a/src/test/java/org/filteredpush/bdq_workbench/app/ConfigLoaderTest.java b/src/test/java/org/filteredpush/bdq_workbench/app/ConfigLoaderTest.java index 39bd58c..00c27b6 100644 --- a/src/test/java/org/filteredpush/bdq_workbench/app/ConfigLoaderTest.java +++ b/src/test/java/org/filteredpush/bdq_workbench/app/ConfigLoaderTest.java @@ -27,4 +27,13 @@ void rejectsBlankRecordFilterValues() { .isInstanceOf(AppException.class) .hasMessageContaining("values must not be blank"); } + + @Test + void loadsDatasetViewPathFromOverrides() { + AppConfig config = new ConfigLoader().load(Map.of( + "bdq.dataset", "dataset.zip", + "bdq.dataset.view", "/tmp/view.json")); + + assertThat(config.datasetView()).isEqualTo("/tmp/view.json"); + } } diff --git a/src/test/java/org/filteredpush/bdq_workbench/ingest/DataPackageDialectIngestTest.java b/src/test/java/org/filteredpush/bdq_workbench/ingest/DataPackageDialectIngestTest.java index 8fbd9a8..a3d17f4 100644 --- a/src/test/java/org/filteredpush/bdq_workbench/ingest/DataPackageDialectIngestTest.java +++ b/src/test/java/org/filteredpush/bdq_workbench/ingest/DataPackageDialectIngestTest.java @@ -3,10 +3,12 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import com.fasterxml.jackson.databind.ObjectMapper; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; import org.filteredpush.bdq_workbench.app.AppException; import org.filteredpush.bdq_workbench.model.RecordDataset; import org.junit.jupiter.api.Test; @@ -247,6 +249,84 @@ void resourcePathEscapingThePackageDirectoryIsRejected(@TempDir Path tempDir) th .hasMessageContaining("no resource with a readable data file path"); } + @Test + void schemaForeignKeysAreParsed(@TempDir Path tempDir) throws Exception { + writeFile(tempDir, "occurrence.csv", StandardCharsets.UTF_8, "occurrenceID\nocc-1\n"); + writeFile(tempDir, "identification.csv", StandardCharsets.UTF_8, "identificationID,occurrenceID\nid-1,occ-1\n"); + Path manifest = writeManifest(tempDir, """ + { + "resources": [ + { + "name": "occurrence", + "path": "occurrence.csv", + "schema": { "fields": [ { "name": "occurrenceID" } ], "primaryKey": "occurrenceID" } + }, + { + "name": "identification", + "path": "identification.csv", + "schema": { + "fields": [ { "name": "identificationID" }, { "name": "occurrenceID" } ], + "foreignKeys": [ + { "fields": "occurrenceID", "reference": { "resource": "occurrence", "fields": "occurrenceID" } } + ] + } + } + ] + } + """); + + var root = new ObjectMapper().readTree(Files.newBufferedReader(manifest)); + List> tables = + DataPackageDialectParser.parseResources(new ObjectMapper(), root, tempDir); + + assertThat(tables).hasSize(2); + assertThat(tables.get(1).descriptor().foreignKeys()) + .singleElement() + .satisfies(key -> { + assertThat(key.field()).isEqualTo("occurrenceID"); + assertThat(key.referenceResource()).isEqualTo("occurrence"); + assertThat(key.referenceField()).isEqualTo("occurrenceID"); + }); + } + + @Test + void relationalIngestBuildsCoreGraphsFromForeignKeys(@TempDir Path tempDir) throws Exception { + writeFile(tempDir, "occurrence.csv", StandardCharsets.UTF_8, + "occurrenceID,eventDate\nocc-1,2020-01-01\n"); + writeFile(tempDir, "identification.csv", StandardCharsets.UTF_8, + "identificationID,occurrenceID,scientificName\nid-1,occ-1,Abies balsamea\n"); + Path manifest = writeManifest(tempDir, """ + { + "resources": [ + { + "name": "occurrence", + "path": "occurrence.csv", + "schema": { + "fields": [ { "name": "occurrenceID" }, { "name": "eventDate" } ], + "primaryKey": "occurrenceID" + } + }, + { + "name": "identification", + "path": "identification.csv", + "schema": { + "fields": [ { "name": "identificationID" }, { "name": "occurrenceID" }, { "name": "scientificName" } ], + "foreignKeys": [ + { "fields": "occurrenceID", "reference": { "resource": "occurrence", "fields": "occurrenceID" } } + ] + } + } + ] + } + """); + + RelationalIngestResult relational = new RelationalDatasetIngestor().ingest(manifest, "occurrence"); + + assertThat(relational.graphs()).hasSize(1); + assertThat(relational.graphs().get(0).relatedByRelation()).containsKey("identification"); + assertThat(relational.graphs().get(0).relatedByRelation().get("identification")).hasSize(1); + } + /** * Writes a {@code datapackage.json} manifest into a directory. * diff --git a/src/test/java/org/filteredpush/bdq_workbench/ingest/DatasetViewSchemaTest.java b/src/test/java/org/filteredpush/bdq_workbench/ingest/DatasetViewSchemaTest.java new file mode 100644 index 0000000..5a92a67 --- /dev/null +++ b/src/test/java/org/filteredpush/bdq_workbench/ingest/DatasetViewSchemaTest.java @@ -0,0 +1,77 @@ +package org.filteredpush.bdq_workbench.ingest; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.nio.file.Path; +import java.util.List; +import org.filteredpush.bdq_workbench.app.AppException; +import org.filteredpush.bdq_workbench.model.DatasetSchema; +import org.filteredpush.bdq_workbench.model.DatasetView; +import org.filteredpush.bdq_workbench.model.DatasetViewCardinalityPolicy; +import org.filteredpush.bdq_workbench.model.DatasetViewJoin; +import org.filteredpush.bdq_workbench.model.DatasetViewMapping; +import org.filteredpush.bdq_workbench.model.RelationshipSchema; +import org.filteredpush.bdq_workbench.model.TableSchema; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class DatasetViewSchemaTest { + + @Test + void schemaFingerprintIsStableForEquivalentShapes() { + List first = List.of( + new TableSchema("occurrence", "occurrence", "OCCURRENCE", "occurrenceID", + List.of("occurrenceID", "scientificName")), + new TableSchema("identification", "identification", "OTHER", "identificationID", + List.of("scientificName", "occurrenceID"))); + List second = List.of( + new TableSchema("identification", "identification", "OTHER", "identificationID", + List.of("occurrenceID", "scientificName")), + new TableSchema("occurrence", "occurrence", "OCCURRENCE", "occurrenceID", + List.of("scientificName", "occurrenceID"))); + + assertThat(SchemaFingerprint.of(first)).isEqualTo(SchemaFingerprint.of(second)); + } + + @Test + void datasetViewRoundTripsToJsonAndValidatesFingerprint(@TempDir Path tempDir) { + DatasetView view = new DatasetView( + "occurrence", + "fingerprint-1", + List.of(new DatasetViewJoin("identification", "identification", DatasetViewCardinalityPolicy.FIRST_ROW)), + List.of(new DatasetViewMapping("scientificName", "identification", "scientificName"))); + DatasetViewIO io = new DatasetViewIO(); + Path file = tempDir.resolve("view.json"); + + io.save(file, view); + DatasetView loaded = io.load(file); + + assertThat(loaded).isEqualTo(view); + + DatasetSchema compatible = new DatasetSchema(List.of(), List.of(), "fingerprint-1"); + io.validateCompatibility(loaded, compatible); + + DatasetSchema incompatible = new DatasetSchema(List.of(), List.of(), "fingerprint-2"); + assertThatThrownBy(() -> io.validateCompatibility(loaded, incompatible)) + .isInstanceOf(AppException.class) + .hasMessageContaining("fingerprint does not match"); + } + + @Test + void builtInViewIsOnlyAppliedWhenRelationshipsArePresent() { + DatasetSchema withoutRelationship = new DatasetSchema( + List.of( + new TableSchema("event", "event", "EVENT", "eventID", List.of("eventID")), + new TableSchema("occurrence", "occurrence", "OCCURRENCE", "occurrenceID", List.of("occurrenceID"))), + List.of(), + "fp"); + DatasetSchema withRelationship = new DatasetSchema( + withoutRelationship.tables(), + List.of(new RelationshipSchema("occurrence", "coreid", "event", "eventID", "occurrence")), + "fp"); + + assertThat(BuiltInDatasetViews.select(withoutRelationship, new java.util.ArrayList<>())).isEmpty(); + assertThat(BuiltInDatasetViews.select(withRelationship, new java.util.ArrayList<>())).isPresent(); + } +} diff --git a/src/test/java/org/filteredpush/bdq_workbench/ingest/DwcArchiveMetaIngestTest.java b/src/test/java/org/filteredpush/bdq_workbench/ingest/DwcArchiveMetaIngestTest.java index c6b97d5..c9165a0 100644 --- a/src/test/java/org/filteredpush/bdq_workbench/ingest/DwcArchiveMetaIngestTest.java +++ b/src/test/java/org/filteredpush/bdq_workbench/ingest/DwcArchiveMetaIngestTest.java @@ -7,8 +7,10 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; import java.util.zip.ZipOutputStream; import org.filteredpush.bdq_workbench.model.RecordDataset; import org.junit.jupiter.api.Test; @@ -193,6 +195,45 @@ void metaXmlNamingAMissingCoreFileFallsBackToConvention(@TempDir Path tempDir) t assertThat(dataset.records().get(0).terms()).containsEntry("country", "Canada"); } + @Test + void extensionCoreIdColumnIsParsedFromMetaXml(@TempDir Path tempDir) throws Exception { + String meta = """ + + + + event.txt + + + + + occurrence.txt + + + + + """; + Map entries = new LinkedHashMap<>(); + entries.put("meta.xml", meta.getBytes(StandardCharsets.UTF_8)); + entries.put("event.txt", "eventID\nevt-1\n".getBytes(StandardCharsets.UTF_8)); + entries.put("occurrence.txt", "coreid\toccurrenceID\nevt-1\tocc-1\n".getBytes(StandardCharsets.UTF_8)); + Path archive = Files.createTempFile(tempDir, "dataset", ".zip"); + try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(archive), StandardCharsets.UTF_8)) { + for (Map.Entry entry : entries.entrySet()) { + zip.putNextEntry(new ZipEntry(entry.getKey())); + zip.write(entry.getValue()); + zip.closeEntry(); + } + } + + try (ZipFile zipFile = new ZipFile(archive.toFile())) { + List> tables = DwcArchiveMetaParser.parseTables(zipFile); + assertThat(tables).hasSize(2); + assertThat(tables.get(1).descriptor().coreIdColumn()).isEqualTo("coreid"); + } + } + /** * Writes a single-core Darwin Core Archive to a temporary zip file. * diff --git a/src/test/java/org/filteredpush/bdq_workbench/ingest/ViewFlattenerTest.java b/src/test/java/org/filteredpush/bdq_workbench/ingest/ViewFlattenerTest.java new file mode 100644 index 0000000..bd79815 --- /dev/null +++ b/src/test/java/org/filteredpush/bdq_workbench/ingest/ViewFlattenerTest.java @@ -0,0 +1,87 @@ +package org.filteredpush.bdq_workbench.ingest; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import java.util.Map; +import org.filteredpush.bdq_workbench.model.CanonicalRecord; +import org.filteredpush.bdq_workbench.model.DatasetSchema; +import org.filteredpush.bdq_workbench.model.DatasetView; +import org.filteredpush.bdq_workbench.model.DatasetViewCardinalityPolicy; +import org.filteredpush.bdq_workbench.model.DatasetViewJoin; +import org.filteredpush.bdq_workbench.model.DatasetViewMapping; +import org.filteredpush.bdq_workbench.model.RecordGraph; +import org.filteredpush.bdq_workbench.model.SourceCell; +import org.junit.jupiter.api.Test; + +class ViewFlattenerTest { + + @Test + void aggregatePolicyConcatenatesDeterministicallyAndRetainsProvenance() { + CanonicalRecord core = new CanonicalRecord("occ-1", Map.of("occurrenceID", "occ-1")); + CanonicalRecord idA = new CanonicalRecord("id-1", Map.of("scientificName", "Abies")); + CanonicalRecord idB = new CanonicalRecord("id-2", Map.of("scientificName", "Picea")); + RecordGraph graph = new RecordGraph(core, Map.of("identification", List.of(idA, idB))); + RelationalIngestResult relational = new RelationalIngestResult( + List.of(graph), + new DatasetSchema(List.of(), List.of(), "fp"), + List.of()); + DatasetView view = new DatasetView( + "core", + "fp", + List.of(new DatasetViewJoin("identification", "identification", DatasetViewCardinalityPolicy.AGGREGATE)), + List.of(new DatasetViewMapping("scientificName", "identification", "scientificName"))); + + ViewFlattenResult flattened = new ViewFlattener().flatten(relational, view); + + assertThat(flattened.dataset().records()).hasSize(1); + assertThat(flattened.dataset().records().get(0).terms().get("scientificName")) + .isEqualTo("Abies | Picea"); + assertThat(flattened.dataset().records().get(0).provenanceByTerm().get("scientificName")) + .extracting(SourceCell::rowRef) + .containsExactly("id-1", "id-2"); + } + + @Test + void firstRowPolicyUsesFirstRowDeterministically() { + CanonicalRecord core = new CanonicalRecord("occ-1", Map.of("occurrenceID", "occ-1")); + CanonicalRecord idA = new CanonicalRecord("id-1", Map.of("scientificName", "Abies")); + CanonicalRecord idB = new CanonicalRecord("id-2", Map.of("scientificName", "Picea")); + RecordGraph graph = new RecordGraph(core, Map.of("identification", List.of(idA, idB))); + RelationalIngestResult relational = new RelationalIngestResult( + List.of(graph), + new DatasetSchema(List.of(), List.of(), "fp"), + List.of()); + DatasetView view = new DatasetView( + "core", + "fp", + List.of(new DatasetViewJoin("identification", "identification", DatasetViewCardinalityPolicy.FIRST_ROW)), + List.of(new DatasetViewMapping("scientificName", "identification", "scientificName"))); + + ViewFlattenResult flattened = new ViewFlattener().flatten(relational, view); + + assertThat(flattened.dataset().records().get(0).terms().get("scientificName")).isEqualTo("Abies"); + } + + @Test + void rejectPolicyAddsDiagnosticInsteadOfCrashing() { + CanonicalRecord core = new CanonicalRecord("occ-1", Map.of("occurrenceID", "occ-1")); + CanonicalRecord idA = new CanonicalRecord("id-1", Map.of("scientificName", "Abies")); + CanonicalRecord idB = new CanonicalRecord("id-2", Map.of("scientificName", "Picea")); + RecordGraph graph = new RecordGraph(core, Map.of("identification", List.of(idA, idB))); + RelationalIngestResult relational = new RelationalIngestResult( + List.of(graph), + new DatasetSchema(List.of(), List.of(), "fp"), + List.of()); + DatasetView view = new DatasetView( + "core", + "fp", + List.of(new DatasetViewJoin("identification", "identification", DatasetViewCardinalityPolicy.REJECT)), + List.of(new DatasetViewMapping("scientificName", "identification", "scientificName"))); + + ViewFlattenResult flattened = new ViewFlattener().flatten(relational, view); + + assertThat(flattened.dataset().records().get(0).terms().get("scientificName")).isEmpty(); + assertThat(flattened.diagnostics()).anyMatch(message -> message.contains("Cardinality conflict")); + } +} From fa8846d33fcd8fa01e174fddde9bb3d0516f8fda Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 23 Sep 2026 01:43:53 +0000 Subject: [PATCH 03/13] Address review feedback for relational ingest and view flattening Co-authored-by: chicoreus <3864169+chicoreus@users.noreply.github.com> --- .../bdq_workbench/app/BdqWorkbenchGui.java | 68 ++++++++++++--- .../ingest/DataPackageDialectParser.java | 87 ++++++++++--------- .../ingest/DefaultIngestService.java | 15 ++-- .../ingest/RelationalDatasetIngestor.java | 23 +++-- .../ingest/SchemaFingerprint.java | 23 ++++- .../bdq_workbench/ingest/ViewFlattener.java | 34 ++++---- .../ingest/DatasetViewSchemaTest.java | 23 ++++- .../ingest/ViewFlattenerTest.java | 44 ++++++++++ 8 files changed, 230 insertions(+), 87 deletions(-) diff --git a/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGui.java b/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGui.java index e065e6a..e646028 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGui.java +++ b/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGui.java @@ -1849,13 +1849,39 @@ private static void loadDatasetViewDialog(JFrame frame, String datasetPath, JTex JOptionPane.ERROR_MESSAGE); return; } - RelationalDatasetIngestor ingestor = new RelationalDatasetIngestor(); - DatasetViewIO io = new DatasetViewIO(); - var relational = ingestor.ingest(path, ""); - DatasetSchema schema = relational.schema(); - DatasetView suggested = suggestDatasetView(schema); - var preview = new org.filteredpush.bdq_workbench.ingest.ViewFlattener().flatten(relational, suggested); + SwingWorker worker = new SwingWorker<>() { + @Override + protected DatasetViewPreview doInBackground() { + RelationalDatasetIngestor ingestor = new RelationalDatasetIngestor(); + var relational = ingestor.ingest(path, ""); + DatasetSchema schema = relational.schema(); + DatasetView suggested = suggestDatasetView(schema); + var preview = new org.filteredpush.bdq_workbench.ingest.ViewFlattener().flatten(relational, suggested); + return new DatasetViewPreview(schema, suggested, preview); + } + + @Override + protected void done() { + try { + openDatasetViewDialog(frame, datasetViewField, get()); + } catch (Exception e) { + Throwable cause = e.getCause() == null ? e : e.getCause(); + JOptionPane.showMessageDialog( + frame, + "Unable to inspect dataset for dataset views: " + cause.getMessage(), + "Dataset view setup failed", + JOptionPane.ERROR_MESSAGE); + } + } + }; + worker.execute(); + } + private static void openDatasetViewDialog(JFrame frame, JTextField datasetViewField, DatasetViewPreview previewData) { + DatasetViewIO io = new DatasetViewIO(); + DatasetSchema schema = previewData.schema(); + DatasetView suggested = previewData.suggested(); + var preview = previewData.preview(); JTextArea details = new JTextArea(18, 80); details.setEditable(false); details.setLineWrap(true); @@ -1899,7 +1925,6 @@ private static void loadDatasetViewDialog(JFrame frame, String datasetPath, JTex preview.diagnostics().forEach(message -> builder.append(" - ").append(message).append('\n')); } details.setText(builder.toString()); - JButton save = new JButton("Save View..."); JButton load = new JButton("Load View..."); JPanel buttons = new JPanel(new FlowLayout(FlowLayout.RIGHT)); @@ -1918,10 +1943,18 @@ private static void loadDatasetViewDialog(JFrame frame, String datasetPath, JTex if (selected == null) { return; } - DatasetView view = io.load(Path.of(selected)); - io.validateCompatibility(view, schema); - datasetViewField.setText(selected); - dialog.dispose(); + try { + DatasetView view = io.load(Path.of(selected)); + io.validateCompatibility(view, schema); + datasetViewField.setText(selected); + dialog.dispose(); + } catch (AppException ex) { + JOptionPane.showMessageDialog( + frame, + "Unable to load dataset view: " + ex.getMessage(), + "Dataset view load failed", + JOptionPane.ERROR_MESSAGE); + } }); save.addActionListener(e -> { String selected = chooseSaveFile(frame, "Save dataset view JSON", "bdq-dataset-view.json"); @@ -1951,15 +1984,19 @@ private static DatasetView suggestDatasetView(DatasetSchema schema) { relationship.fromTable(), DatasetViewCardinalityPolicy.REJECT)) .toList(); + java.util.Set allowedTables = new java.util.LinkedHashSet<>(); + allowedTables.add(grain); + joins.forEach(join -> allowedTables.add(join.sourceTable())); List requestedTerms = List.of( "occurrenceID", "scientificName", "eventDate", "decimalLatitude", "decimalLongitude"); List mappings = new ArrayList<>(); for (String term : requestedTerms) { String sourceTable = schema.tables().stream() + .filter(table -> allowedTables.contains(table.name())) .filter(table -> table.columns().contains(term)) .findFirst() .map(table -> table.name()) - .orElse("core"); + .orElse(grain); mappings.add(new DatasetViewMapping(term, sourceTable, term)); } return new DatasetView(grain, schema.schemaFingerprint(), joins, mappings); @@ -3579,6 +3616,13 @@ private record PickerField(JTextField field, JButton button) { private record RecordFilterValueOption(String value, long count) { } + /** Prepared schema/view/preview tuple for the dataset-view builder dialog. */ + private record DatasetViewPreview( + DatasetSchema schema, + DatasetView suggested, + org.filteredpush.bdq_workbench.ingest.ViewFlattenResult preview) { + } + /** Dataset-derived terms and value counts used to build record filters interactively. */ private record RecordFilterDatasetProfile( int recordCount, diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageDialectParser.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageDialectParser.java index 3444727..868478e 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageDialectParser.java +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageDialectParser.java @@ -318,49 +318,6 @@ private static List readSchemaFieldNames(JsonNode schema) { if (!fields.isArray()) { return names; } - - /** - * Reads single-column foreign-key declarations from a resource schema. - * - * @param schema the resource's {@code schema} property - * @return declared single-column foreign keys - */ - private static List readForeignKeys(JsonNode schema) { - List keys = new ArrayList<>(); - JsonNode foreignKeys = schema.path("foreignKeys"); - if (!foreignKeys.isArray()) { - return keys; - } - for (JsonNode key : foreignKeys) { - String field = firstText(key.path("fields")); - JsonNode reference = key.path("reference"); - String referenceField = firstText(reference.path("fields")); - if (field == null || referenceField == null) { - continue; - } - keys.add(new DataPackageForeignKey( - field.trim(), - text(reference, "resource") == null ? "" : text(reference, "resource").trim(), - referenceField.trim())); - } - return keys; - } - - /** - * Reads a string value from either a text node or a single-element text array. - * - * @param node node to inspect - * @return extracted text, or {@code null} - */ - private static String firstText(JsonNode node) { - if (node.isTextual()) { - return node.asText(null); - } - if (node.isArray() && node.size() == 1 && node.get(0).isTextual()) { - return node.get(0).asText(null); - } - return null; - } for (JsonNode field : fields) { String name = field.path("name").asText(null); if (name == null || name.isBlank()) { @@ -373,6 +330,50 @@ private static String firstText(JsonNode node) { return names; } + /** + * Reads single-column foreign-key declarations from a resource schema. + * + * @param schema the resource's {@code schema} property + * @return declared single-column foreign keys + */ + private static List readForeignKeys(JsonNode schema) { + List keys = new ArrayList<>(); + JsonNode foreignKeys = schema.path("foreignKeys"); + if (!foreignKeys.isArray()) { + return keys; + } + for (JsonNode key : foreignKeys) { + String field = firstText(key.path("fields")); + JsonNode reference = key.path("reference"); + String referenceField = firstText(reference.path("fields")); + if (field == null || referenceField == null) { + continue; + } + String referenceResource = text(reference, "resource"); + keys.add(new DataPackageForeignKey( + field.trim(), + referenceResource == null ? "" : referenceResource.trim(), + referenceField.trim())); + } + return keys; + } + + /** + * Reads a string value from either a text node or a single-element text array. + * + * @param node node to inspect + * @return extracted text, or {@code null} + */ + private static String firstText(JsonNode node) { + if (node.isTextual()) { + return node.asText(null); + } + if (node.isArray() && node.size() == 1 && node.get(0).isTextual()) { + return node.get(0).asText(null); + } + return null; + } + /** * Determines how many leading header lines the dialect declares. * diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/DefaultIngestService.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/DefaultIngestService.java index f39abf3..7118cd4 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/ingest/DefaultIngestService.java +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/DefaultIngestService.java @@ -115,11 +115,7 @@ public RecordDataset ingest(Path inputPath, String requestedTable, String datase if (datasetView != null && !datasetView.isBlank()) { return ingestThroughView(inputPath, requestedTable, datasetView); } - RecordDataset maybeBuiltIn = ingestThroughBuiltInView(inputPath, requestedTable); - if (maybeBuiltIn != null) { - return maybeBuiltIn; - } - return ingestFlat(inputPath, requestedTable); + return ingestWithOptionalBuiltInView(inputPath, requestedTable); } private RecordDataset ingestFlat(Path inputPath, String requestedTable) { @@ -142,16 +138,19 @@ private RecordDataset ingestThroughView(Path inputPath, String requestedTable, S return flattened.dataset(); } - private RecordDataset ingestThroughBuiltInView(Path inputPath, String requestedTable) { + private RecordDataset ingestWithOptionalBuiltInView(Path inputPath, String requestedTable) { RelationalIngestResult relational = relationalDatasetIngestor.ingest(inputPath, requestedTable); if (relational.graphs().isEmpty()) { - return null; + return ingestFlat(inputPath, requestedTable); } List diagnostics = new ArrayList<>(); Optional builtIn = BuiltInDatasetViews.select(relational.schema(), diagnostics); if (builtIn.isEmpty()) { logDiagnostics(relational.diagnostics(), diagnostics); - return null; + List rows = relational.graphs().stream() + .map(org.filteredpush.bdq_workbench.model.RecordGraph::core) + .toList(); + return new RecordDataset(rows); } ViewFlattenResult flattened = viewFlattener.flatten(relational, builtIn.get()); logDiagnostics(relational.diagnostics(), diagnostics, flattened.diagnostics()); diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/RelationalDatasetIngestor.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/RelationalDatasetIngestor.java index b66953a..50fdc3d 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/ingest/RelationalDatasetIngestor.java +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/RelationalDatasetIngestor.java @@ -123,20 +123,31 @@ private RelationalIngestResult ingestDataPackage(Path inputPath, String requeste List relationships = new ArrayList<>(); for (CoreTableCandidate table : tables) { for (DataPackageForeignKey key : table.descriptor().foreignKeys()) { - String refResource = key.referenceResource().isBlank() - ? table.descriptor().name() - : key.referenceResource(); - if (!refResource.equalsIgnoreCase(selected.descriptor().name())) { + String referencedTableLabel = resolveReferencedTableLabel(tables, table, key); + if (referencedTableLabel == null || !referencedTableLabel.equals(selected.label())) { continue; } relationships.add(new RelationshipSchema( table.label(), key.field(), - selected.label(), + referencedTableLabel, key.referenceField(), table.label())); } } + + private String resolveReferencedTableLabel(List> tables, + CoreTableCandidate source, + DataPackageForeignKey key) { + if (key.referenceResource().isBlank()) { + return source.label(); + } + return tables.stream() + .filter(candidate -> candidate.descriptor().name().equalsIgnoreCase(key.referenceResource())) + .map(CoreTableCandidate::label) + .findFirst() + .orElse(null); + } return assembleResult(selected.label(), rowsByTable, tables.stream() .map(table -> new TableSchema( table.label(), @@ -183,7 +194,7 @@ private RelationalIngestResult assembleResult(String coreTable, Map tables) { - String canonical = java.util.stream.StreamSupport.stream(tables.spliterator(), false) + static String of(List tables, List relationships) { + String tablePart = tables.stream() .sorted(Comparator.comparing(TableSchema::name)) .map(table -> table.name().toLowerCase() + "|" @@ -50,6 +52,23 @@ static String of(Iterable tables) { + "|" + table.columns().stream().map(String::toLowerCase).sorted().collect(Collectors.joining(","))) .collect(Collectors.joining("||")); + String relationshipPart = relationships.stream() + .sorted(Comparator.comparing(RelationshipSchema::relationName) + .thenComparing(RelationshipSchema::fromTable) + .thenComparing(RelationshipSchema::fromColumn) + .thenComparing(RelationshipSchema::toTable) + .thenComparing(RelationshipSchema::toColumn)) + .map(relationship -> relationship.relationName().toLowerCase() + + "|" + + relationship.fromTable().toLowerCase() + + "." + + relationship.fromColumn().toLowerCase() + + "->" + + relationship.toTable().toLowerCase() + + "." + + relationship.toColumn().toLowerCase()) + .collect(Collectors.joining("||")); + String canonical = tablePart + "##" + relationshipPart; try { MessageDigest digest = MessageDigest.getInstance("SHA-256"); return HexFormat.of().formatHex(digest.digest(canonical.getBytes(StandardCharsets.UTF_8))); diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/ViewFlattener.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/ViewFlattener.java index 22c1fc9..495e37a 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/ingest/ViewFlattener.java +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/ViewFlattener.java @@ -52,7 +52,7 @@ public ViewFlattenResult flatten(RelationalIngestResult relational, DatasetView Map terms = new LinkedHashMap<>(); Map> provenance = new LinkedHashMap<>(); for (DatasetViewMapping mapping : view.mappings()) { - ValueSelection selected = selectValue(graph, mapping, view.joins(), diagnostics); + ValueSelection selected = selectValue(graph, mapping, view.grainTable(), view.joins(), diagnostics); terms.put(mapping.term(), selected.value()); if (!selected.cells().isEmpty()) { provenance.put(mapping.term(), selected.cells()); @@ -66,14 +66,15 @@ public ViewFlattenResult flatten(RelationalIngestResult relational, DatasetView /** * Resolves one mapping from either the core row or one related relation. */ - private ValueSelection selectValue(RecordGraph graph, DatasetViewMapping mapping, List joins, + private ValueSelection selectValue(RecordGraph graph, DatasetViewMapping mapping, String grainTable, + List joins, List diagnostics) { - if (mapping.sourceTable().equalsIgnoreCase("core")) { + if (mapping.sourceTable().equalsIgnoreCase(grainTable)) { String value = graph.core().terms().getOrDefault(mapping.sourceColumn(), ""); - return new ValueSelection(value, sourceCells(graph.core(), "core", mapping.sourceColumn(), mapping.term())); + return new ValueSelection(value, sourceCells(graph.core(), grainTable, mapping.sourceColumn(), mapping.term())); } DatasetViewJoin join = joins.stream() - .filter(candidate -> candidate.sourceTable().equals(mapping.sourceTable())) + .filter(candidate -> candidate.sourceTable().equalsIgnoreCase(mapping.sourceTable())) .findFirst() .orElse(null); if (join == null) { @@ -85,11 +86,10 @@ private ValueSelection selectValue(RecordGraph graph, DatasetViewMapping mapping if (related.isEmpty()) { return ValueSelection.empty(); } - List values = related.stream() - .map(row -> row.terms().getOrDefault(mapping.sourceColumn(), "")) - .toList(); - List cells = related.stream() - .flatMap(row -> sourceCells(row, join.sourceTable(), mapping.sourceColumn(), mapping.term()).stream()) + List selections = related.stream() + .map(row -> new ValueSelection( + row.terms().getOrDefault(mapping.sourceColumn(), ""), + sourceCells(row, join.sourceTable(), mapping.sourceColumn(), mapping.term()))) .toList(); if (related.size() > 1 && join.cardinalityPolicy() == DatasetViewCardinalityPolicy.REJECT) { diagnostics.add("Cardinality conflict for relation " + join.relationName() + " on record " @@ -97,15 +97,19 @@ private ValueSelection selectValue(RecordGraph graph, DatasetViewMapping mapping return ValueSelection.empty(); } if (join.cardinalityPolicy() == DatasetViewCardinalityPolicy.FIRST_ROW) { - return new ValueSelection(values.get(0), cells.isEmpty() ? List.of() : List.of(cells.get(0))); + return selections.get(0); } - if (join.cardinalityPolicy() == DatasetViewCardinalityPolicy.AGGREGATE && values.size() > 1) { - String aggregated = values.stream() - .filter(value -> !value.isBlank()) + if (join.cardinalityPolicy() == DatasetViewCardinalityPolicy.AGGREGATE) { + List included = selections.stream() + .filter(selection -> !selection.value().isBlank()) + .toList(); + String aggregated = included.stream() + .map(ValueSelection::value) .collect(Collectors.joining(" | ")); + List cells = included.stream().flatMap(selection -> selection.cells().stream()).toList(); return new ValueSelection(aggregated, cells); } - return new ValueSelection(values.get(0), cells); + return selections.get(0); } /** diff --git a/src/test/java/org/filteredpush/bdq_workbench/ingest/DatasetViewSchemaTest.java b/src/test/java/org/filteredpush/bdq_workbench/ingest/DatasetViewSchemaTest.java index 5a92a67..ae05ae2 100644 --- a/src/test/java/org/filteredpush/bdq_workbench/ingest/DatasetViewSchemaTest.java +++ b/src/test/java/org/filteredpush/bdq_workbench/ingest/DatasetViewSchemaTest.java @@ -31,7 +31,7 @@ void schemaFingerprintIsStableForEquivalentShapes() { new TableSchema("occurrence", "occurrence", "OCCURRENCE", "occurrenceID", List.of("scientificName", "occurrenceID"))); - assertThat(SchemaFingerprint.of(first)).isEqualTo(SchemaFingerprint.of(second)); + assertThat(SchemaFingerprint.of(first, List.of())).isEqualTo(SchemaFingerprint.of(second, List.of())); } @Test @@ -74,4 +74,25 @@ void builtInViewIsOnlyAppliedWhenRelationshipsArePresent() { assertThat(BuiltInDatasetViews.select(withoutRelationship, new java.util.ArrayList<>())).isEmpty(); assertThat(BuiltInDatasetViews.select(withRelationship, new java.util.ArrayList<>())).isPresent(); } + + @Test + void viewLoadFailureIsReportedAsAppException(@TempDir Path tempDir) { + DatasetViewIO io = new DatasetViewIO(); + Path missing = tempDir.resolve("missing-view.json"); + + assertThatThrownBy(() -> io.load(missing)) + .isInstanceOf(AppException.class) + .hasMessageContaining("Unable to read dataset view file"); + } + + @Test + void viewSaveFailureIsReportedAsAppException(@TempDir Path tempDir) { + DatasetViewIO io = new DatasetViewIO(); + DatasetView view = new DatasetView("occurrence", "fp", List.of(), List.of()); + Path invalid = tempDir.resolve("missing").resolve("view.json"); + + assertThatThrownBy(() -> io.save(invalid, view)) + .isInstanceOf(AppException.class) + .hasMessageContaining("Unable to save dataset view file"); + } } diff --git a/src/test/java/org/filteredpush/bdq_workbench/ingest/ViewFlattenerTest.java b/src/test/java/org/filteredpush/bdq_workbench/ingest/ViewFlattenerTest.java index bd79815..17a4a8c 100644 --- a/src/test/java/org/filteredpush/bdq_workbench/ingest/ViewFlattenerTest.java +++ b/src/test/java/org/filteredpush/bdq_workbench/ingest/ViewFlattenerTest.java @@ -84,4 +84,48 @@ void rejectPolicyAddsDiagnosticInsteadOfCrashing() { assertThat(flattened.dataset().records().get(0).terms().get("scientificName")).isEmpty(); assertThat(flattened.diagnostics()).anyMatch(message -> message.contains("Cardinality conflict")); } + + @Test + void missingJoinAddsDiagnostic() { + CanonicalRecord core = new CanonicalRecord("occ-1", Map.of("occurrenceID", "occ-1")); + RecordGraph graph = new RecordGraph(core, Map.of()); + RelationalIngestResult relational = new RelationalIngestResult( + List.of(graph), + new DatasetSchema(List.of(), List.of(), "fp"), + List.of()); + DatasetView view = new DatasetView( + "occurrence", + "fp", + List.of(), + List.of(new DatasetViewMapping("scientificName", "identification", "scientificName"))); + + ViewFlattenResult flattened = new ViewFlattener().flatten(relational, view); + + assertThat(flattened.diagnostics()).anyMatch(message -> message.contains("no join")); + } + + @Test + void firstRowPolicyKeepsFirstRowEvenWhenBlankAndUsesItsProvenance() { + CanonicalRecord core = new CanonicalRecord("occ-1", Map.of("occurrenceID", "occ-1")); + CanonicalRecord idA = new CanonicalRecord("id-1", Map.of("scientificName", "")); + CanonicalRecord idB = new CanonicalRecord("id-2", Map.of("scientificName", "Picea")); + RecordGraph graph = new RecordGraph(core, Map.of("identification", List.of(idA, idB))); + RelationalIngestResult relational = new RelationalIngestResult( + List.of(graph), + new DatasetSchema(List.of(), List.of(), "fp"), + List.of()); + DatasetView view = new DatasetView( + "occurrence", + "fp", + List.of(new DatasetViewJoin("identification", "identification", DatasetViewCardinalityPolicy.FIRST_ROW)), + List.of(new DatasetViewMapping("scientificName", "identification", "scientificName"))); + + ViewFlattenResult flattened = new ViewFlattener().flatten(relational, view); + + assertThat(flattened.dataset().records().get(0).terms().get("scientificName")).isEmpty(); + assertThat(flattened.dataset().records().get(0).provenanceByTerm().get("scientificName")) + .singleElement() + .extracting(SourceCell::rowRef) + .isEqualTo("id-1"); + } } From 478c7ff64a4b47320458858e86e1a4a075ed8385 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 23 Sep 2026 02:08:11 +0000 Subject: [PATCH 04/13] Fix RelationalDatasetIngestor method placement compile error Co-authored-by: chicoreus <3864169+chicoreus@users.noreply.github.com> --- .../ingest/RelationalDatasetIngestor.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/RelationalDatasetIngestor.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/RelationalDatasetIngestor.java index 50fdc3d..6d55add 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/ingest/RelationalDatasetIngestor.java +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/RelationalDatasetIngestor.java @@ -135,19 +135,6 @@ private RelationalIngestResult ingestDataPackage(Path inputPath, String requeste table.label())); } } - - private String resolveReferencedTableLabel(List> tables, - CoreTableCandidate source, - DataPackageForeignKey key) { - if (key.referenceResource().isBlank()) { - return source.label(); - } - return tables.stream() - .filter(candidate -> candidate.descriptor().name().equalsIgnoreCase(key.referenceResource())) - .map(CoreTableCandidate::label) - .findFirst() - .orElse(null); - } return assembleResult(selected.label(), rowsByTable, tables.stream() .map(table -> new TableSchema( table.label(), @@ -163,6 +150,19 @@ private String resolveReferencedTableLabel(List> tables, + CoreTableCandidate source, + DataPackageForeignKey key) { + if (key.referenceResource().isBlank()) { + return source.label(); + } + return tables.stream() + .filter(candidate -> candidate.descriptor().name().equalsIgnoreCase(key.referenceResource())) + .map(CoreTableCandidate::label) + .findFirst() + .orElse(null); + } + private RelationalIngestResult assembleResult(String coreTable, Map> rowsByTable, List tables, List relationships) { List diagnostics = new ArrayList<>(); From ad5fc82d70091974af8ff5b9f3336bb6abfd1014 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 23 Sep 2026 02:16:09 +0000 Subject: [PATCH 05/13] Restore backward-compatible BdqWorkbenchGui buildConfig overload Co-authored-by: chicoreus <3864169+chicoreus@users.noreply.github.com> --- .../bdq_workbench/app/BdqWorkbenchGui.java | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGui.java b/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGui.java index e646028..8aacbe7 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGui.java +++ b/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGui.java @@ -1488,6 +1488,57 @@ private static AppConfig buildConfig( datasetView); } + /** + * Backward-compatible build-config helper used by reflection-based tests. + * + *

Delegates to the dataset-view-aware overload with an empty dataset view path. + * + * @param dataset dataset file path field value + * @param selectedUseCaseId ID of the use case chosen in the combo box + * @param recordFilters record-filter field value + * @param datasetTable selected dataset table name + * @param useCaseSource use case RDF file/URL field value + * @param testDefinitionsSource primary test definitions file/URL field value + * @param additionalTestDefinitions comma-separated extra test definition files/URLs + * @param ontologySource BDQ FFDQ ontology file/URL field value + * @param discoveryPackages comma-separated implementation discovery packages + * @param threads thread count field value + * @param dedupEnabled whether distinct-value execution is enabled for this run + * @param resolver resolves and caches remote/local resource paths + * @param defaults fallback values used when a field is blank + * @return the assembled configuration + */ + private static AppConfig buildConfig( + String dataset, + String selectedUseCaseId, + String recordFilters, + String datasetTable, + String useCaseSource, + String testDefinitionsSource, + String additionalTestDefinitions, + String ontologySource, + String discoveryPackages, + String threads, + boolean dedupEnabled, + CachedResourceResolver resolver, + AppConfig defaults) { + return buildConfig( + dataset, + selectedUseCaseId, + recordFilters, + datasetTable, + "", + useCaseSource, + testDefinitionsSource, + additionalTestDefinitions, + ontologySource, + discoveryPackages, + threads, + dedupEnabled, + resolver, + defaults); + } + /** * Parses the thread-count field value, rejecting non-numeric or non-positive values. * From 79e8a0cca4281ca7ea675d8f859b340ac15c2749 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 23 Sep 2026 02:34:20 +0000 Subject: [PATCH 06/13] Short-circuit dataset-view builder for single-table datasets Co-authored-by: chicoreus <3864169+chicoreus@users.noreply.github.com> --- .../bdq_workbench/app/BdqWorkbenchGui.java | 10 ++ .../ingest/DatasetSchemaInspector.java | 160 ++++++++++++++++++ .../ingest/DatasetSchemaInspectorTest.java | 68 ++++++++ 3 files changed, 238 insertions(+) create mode 100644 src/main/java/org/filteredpush/bdq_workbench/ingest/DatasetSchemaInspector.java create mode 100644 src/test/java/org/filteredpush/bdq_workbench/ingest/DatasetSchemaInspectorTest.java diff --git a/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGui.java b/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGui.java index 8aacbe7..152c288 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGui.java +++ b/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGui.java @@ -70,6 +70,7 @@ import org.filteredpush.bdq_workbench.execution.ReflectionExecutionAdapter; import org.filteredpush.bdq_workbench.ingest.DatasetViewIO; import org.filteredpush.bdq_workbench.ingest.DefaultIngestService; +import org.filteredpush.bdq_workbench.ingest.DatasetSchemaInspector; import org.filteredpush.bdq_workbench.ingest.RelationalDatasetIngestor; import org.filteredpush.bdq_workbench.model.RecordFilterSummary; import org.filteredpush.bdq_workbench.model.BindingReview; @@ -1900,6 +1901,15 @@ private static void loadDatasetViewDialog(JFrame frame, String datasetPath, JTex JOptionPane.ERROR_MESSAGE); return; } + DatasetSchemaInspector.DatasetSchemaOverview overview = new DatasetSchemaInspector().inspect(path); + if (overview.tables().size() <= 1) { + JOptionPane.showMessageDialog( + frame, + overview.describeTables() + ".\nBuild Dataset View is only needed when the dataset has related tables.", + "Dataset view not needed", + JOptionPane.INFORMATION_MESSAGE); + return; + } SwingWorker worker = new SwingWorker<>() { @Override protected DatasetViewPreview doInBackground() { diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/DatasetSchemaInspector.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/DatasetSchemaInspector.java new file mode 100644 index 0000000..88bdc64 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/DatasetSchemaInspector.java @@ -0,0 +1,160 @@ +/** DatasetSchemaInspector.java + * + * Lightweight schema inspection for dataset-view setup. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.ingest; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import org.filteredpush.bdq_workbench.app.AppException; + +/** + * Inspects dataset table metadata without reading table rows. + */ +public class DatasetSchemaInspector { + private final ObjectMapper mapper = new ObjectMapper(); + + /** + * Inspects available dataset tables for UI decisions. + * + * @param inputPath dataset input path + * @return metadata-only table summary + */ + public DatasetSchemaOverview inspect(Path inputPath) { + String fileName = inputPath.getFileName().toString().toLowerCase(); + if (fileName.endsWith(".zip")) { + return inspectDwcArchive(inputPath); + } + if (fileName.endsWith(".json") || fileName.endsWith("datapackage")) { + return inspectDataPackage(inputPath); + } + throw new AppException("Unsupported dataset input: " + inputPath); + } + + private DatasetSchemaOverview inspectDwcArchive(Path inputPath) { + try (ZipFile zipFile = new ZipFile(inputPath.toFile())) { + List> tables = DwcArchiveMetaParser.parseTables(zipFile); + if (!tables.isEmpty()) { + return new DatasetSchemaOverview(tables.stream() + .map(table -> new DatasetTableSummary( + table.label(), + table.rowType(), + table.rowTypeEvidence(), + table.declaredCore())) + .toList()); + } + return new DatasetSchemaOverview(List.of(new DatasetTableSummary( + resolveConventionalCoreEntry(zipFile).map(ZipEntry::getName).orElse("occurrence.txt"), + DatasetRowType.OCCURRENCE, + "conventional default", + true))); + } catch (IOException e) { + throw new AppException("Failed to inspect DwC-A " + inputPath, e); + } + } + + private DatasetSchemaOverview inspectDataPackage(Path inputPath) { + try { + JsonNode root = mapper.readTree(Files.newBufferedReader(inputPath)); + Path packageDir = inputPath.toAbsolutePath().getParent(); + List> tables = + DataPackageDialectParser.parseResources(mapper, root, packageDir); + return new DatasetSchemaOverview(tables.stream() + .map(table -> new DatasetTableSummary( + table.label(), + table.rowType(), + table.rowTypeEvidence(), + table.declaredCore())) + .toList()); + } catch (IOException e) { + throw new AppException("Failed to inspect data package " + inputPath, e); + } + } + + private Optional resolveConventionalCoreEntry(ZipFile zipFile) { + ZipEntry occurrence = zipFile.getEntry("occurrence.txt"); + if (occurrence != null) { + return Optional.of(occurrence); + } + return zipFile.stream() + .filter(entry -> !entry.isDirectory() && entry.getName().endsWith(".txt")) + .findFirst(); + } + + /** + * Metadata-only dataset summary. + * + * @param tables offered tables + */ + public record DatasetSchemaOverview(List tables) { + + /** + * Canonical constructor; copies list defensively. + */ + public DatasetSchemaOverview { + tables = List.copyOf(tables); + } + + /** + * Renders a short table list for user-facing messages. + * + * @return one-line table summary + */ + public String describeTables() { + if (tables.isEmpty()) { + return "Dataset offers no readable tables"; + } + if (tables.size() == 1) { + return "Dataset offers one table, " + tables.get(0).describe(); + } + return "Dataset offers " + tables.size() + " tables"; + } + } + + /** + * Summary of one offered table. + * + * @param label table label + * @param rowType inferred row type + * @param rowTypeEvidence evidence for inferred row type + * @param declaredCore whether this table is the declared core + */ + public record DatasetTableSummary( + String label, + DatasetRowType rowType, + String rowTypeEvidence, + boolean declaredCore) { + + /** + * Renders one table as a short human-readable descriptor. + * + * @return rendered table descriptor + */ + public String describe() { + String declared = declaredCore ? ", declared core" : ""; + return label + " [rowType=" + rowType.name() + " (" + rowTypeEvidence + ")" + declared + "]"; + } + } +} diff --git a/src/test/java/org/filteredpush/bdq_workbench/ingest/DatasetSchemaInspectorTest.java b/src/test/java/org/filteredpush/bdq_workbench/ingest/DatasetSchemaInspectorTest.java new file mode 100644 index 0000000..c337726 --- /dev/null +++ b/src/test/java/org/filteredpush/bdq_workbench/ingest/DatasetSchemaInspectorTest.java @@ -0,0 +1,68 @@ +package org.filteredpush.bdq_workbench.ingest; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class DatasetSchemaInspectorTest { + + @Test + void dwcaWithOneDeclaredCoreReportsSingleTable(@TempDir Path tempDir) throws Exception { + String meta = """ + + + + occurrence.txt + + + + + """; + Map entries = new LinkedHashMap<>(); + entries.put("meta.xml", meta.getBytes(StandardCharsets.UTF_8)); + entries.put("occurrence.txt", "occurrenceID\nocc-1\n".getBytes(StandardCharsets.UTF_8)); + Path archive = Files.createTempFile(tempDir, "dataset", ".zip"); + try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(archive), StandardCharsets.UTF_8)) { + for (Map.Entry entry : entries.entrySet()) { + zip.putNextEntry(new ZipEntry(entry.getKey())); + zip.write(entry.getValue()); + zip.closeEntry(); + } + } + + DatasetSchemaInspector.DatasetSchemaOverview overview = new DatasetSchemaInspector().inspect(archive); + + assertThat(overview.tables()).hasSize(1); + assertThat(overview.describeTables()).contains("Dataset offers one table, occurrence.txt"); + assertThat(overview.tables().get(0).describe()).contains("declared core"); + } + + @Test + void datapackageWithTwoResourcesReportsTwoTables(@TempDir Path tempDir) throws Exception { + Files.writeString(tempDir.resolve("occurrence.csv"), "occurrenceID\nocc-1\n", StandardCharsets.UTF_8); + Files.writeString(tempDir.resolve("event.csv"), "eventID\nevt-1\n", StandardCharsets.UTF_8); + Files.writeString(tempDir.resolve("datapackage.json"), """ + { + "resources": [ + { "name": "occurrence", "path": "occurrence.csv" }, + { "name": "event", "path": "event.csv" } + ] + } + """, StandardCharsets.UTF_8); + + DatasetSchemaInspector.DatasetSchemaOverview overview = + new DatasetSchemaInspector().inspect(tempDir.resolve("datapackage.json")); + + assertThat(overview.tables()).hasSize(2); + assertThat(overview.describeTables()).isEqualTo("Dataset offers 2 tables"); + } +} From a2f8d203246bef09afbcb713e439108148ac800a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 23 Sep 2026 02:44:02 +0000 Subject: [PATCH 07/13] Fix DatasetSchemaInspector Optional type mismatch Co-authored-by: chicoreus <3864169+chicoreus@users.noreply.github.com> --- .../bdq_workbench/ingest/DatasetSchemaInspector.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/DatasetSchemaInspector.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/DatasetSchemaInspector.java index 88bdc64..f28a6ef 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/ingest/DatasetSchemaInspector.java +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/DatasetSchemaInspector.java @@ -25,7 +25,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; -import java.util.Optional; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import org.filteredpush.bdq_workbench.app.AppException; @@ -66,7 +65,7 @@ private DatasetSchemaOverview inspectDwcArchive(Path inputPath) { .toList()); } return new DatasetSchemaOverview(List.of(new DatasetTableSummary( - resolveConventionalCoreEntry(zipFile).map(ZipEntry::getName).orElse("occurrence.txt"), + resolveConventionalCoreEntryName(zipFile), DatasetRowType.OCCURRENCE, "conventional default", true))); @@ -93,14 +92,16 @@ private DatasetSchemaOverview inspectDataPackage(Path inputPath) { } } - private Optional resolveConventionalCoreEntry(ZipFile zipFile) { + private String resolveConventionalCoreEntryName(ZipFile zipFile) { ZipEntry occurrence = zipFile.getEntry("occurrence.txt"); if (occurrence != null) { - return Optional.of(occurrence); + return occurrence.getName(); } return zipFile.stream() .filter(entry -> !entry.isDirectory() && entry.getName().endsWith(".txt")) - .findFirst(); + .map(ZipEntry::getName) + .findFirst() + .orElse("occurrence.txt"); } /** From b2807c0d5d3f39cd9333996ae6de42964b511820 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 23 Sep 2026 09:23:39 +0000 Subject: [PATCH 08/13] Tighten record-filter row sizing and expand distinct values panel Co-authored-by: chicoreus <3864169+chicoreus@users.noreply.github.com> --- .../bdq_workbench/app/BdqWorkbenchGui.java | 58 +++++++++++++++---- .../app/BdqWorkbenchGuiTest.java | 25 ++++++++ 2 files changed, 71 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGui.java b/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGui.java index 152c288..e11d9c9 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGui.java +++ b/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGui.java @@ -130,6 +130,7 @@ */ final class BdqWorkbenchGui { private static final Logger LOG = LoggerFactory.getLogger(BdqWorkbenchGui.class); + private static final int RECORD_FILTER_SUGGESTION_LIMIT = 20; private static final int FINALIZATION_STAGE_STEP_COUNT = 5; private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @@ -2188,14 +2189,17 @@ private static RecordFilterRowWidgets addRecordFilterRow( } JTextField values = new JTextField(initialValues == null ? "" : initialValues); forceSingleLineControlHeight(fieldChoice, values.getPreferredSize().height); - JTextArea suggestionArea = new JTextArea(3, 30); + forceSingleLineControlHeight(values, values.getPreferredSize().height); + JTextArea suggestionArea = new JTextArea(8, 30); suggestionArea.setEditable(false); suggestionArea.setLineWrap(true); suggestionArea.setWrapStyleWord(true); suggestionArea.setBorder(BorderFactory.createEtchedBorder()); - lockTextAreaHeight(suggestionArea); installTextAreaClipboardSupport(suggestionArea); + JScrollPane suggestionScroll = new JScrollPane(suggestionArea); + lockTextAreaHeight(suggestionScroll, suggestionArea.getPreferredSize().height + 60); JButton remove = new JButton("Remove"); + forceSingleLineControlHeight(remove, values.getPreferredSize().height); remove.addActionListener(e -> { rowsPanel.remove(row); rowsPanel.revalidate(); @@ -2222,10 +2226,11 @@ private static RecordFilterRowWidgets addRecordFilterRow( stacked.setLayout(new BoxLayout(stacked, BoxLayout.Y_AXIS)); stacked.add(inputRow); stacked.add(valuesRow); - stacked.add(suggestionArea); + stacked.add(suggestionScroll); row.add(stacked, BorderLayout.CENTER); row.setBorder(BorderFactory.createEmptyBorder(4, 0, 4, 0)); + row.setMaximumSize(new Dimension(Integer.MAX_VALUE, row.getPreferredSize().height)); rowsPanel.add(row); return new RecordFilterRowWidgets(row, fieldChoice, values, unresolvedField, unresolvedMessage); } @@ -2255,7 +2260,7 @@ private static RecordFilterDatasetProfile profileRecordFilters(RecordDataset dat .thenComparing(Map.Entry::getKey, String.CASE_INSENSITIVE_ORDER)); distinctValueCounts.put(term, entries.size()); topValuesByTerm.put(term, entries.stream() - .limit(8) + .limit(RECORD_FILTER_SUGGESTION_LIMIT) .map(entry -> new RecordFilterValueOption(entry.getKey(), entry.getValue())) .toList()); }); @@ -2528,17 +2533,46 @@ private static void forceSingleLineControlHeight(JComboBox combo, int targetH } /** - * Fixes a text area's preferred/minimum/maximum height so dynamic wrapped text does not cause + * Forces a text field to keep a single-line control height. + * + * @param textField the text field to normalize + * @param targetHeight the desired control height in pixels + */ + private static void forceSingleLineControlHeight(JTextField textField, int targetHeight) { + Dimension preferred = textField.getPreferredSize(); + Dimension normalized = new Dimension(preferred.width, targetHeight); + textField.setPreferredSize(normalized); + textField.setMinimumSize(normalized); + textField.setMaximumSize(new Dimension(Integer.MAX_VALUE, targetHeight)); + } + + /** + * Forces a button to keep a single-line control height. + * + * @param button the button to normalize + * @param targetHeight the desired control height in pixels + */ + private static void forceSingleLineControlHeight(JButton button, int targetHeight) { + Dimension preferred = button.getPreferredSize(); + Dimension normalized = new Dimension(preferred.width, targetHeight); + button.setPreferredSize(normalized); + button.setMinimumSize(normalized); + button.setMaximumSize(normalized); + } + + /** + * Fixes a scroll pane's preferred/minimum/maximum height so dynamic wrapped text does not cause * surrounding filter rows to resize and scroll the selected controls out of view. * - * @param textArea the text area whose height should remain stable + * @param scrollPane the scroll pane whose height should remain stable + * @param targetHeight the desired control height in pixels */ - private static void lockTextAreaHeight(JTextArea textArea) { - Dimension preferred = textArea.getPreferredSize(); - Dimension normalized = new Dimension(Math.max(preferred.width, 320), preferred.height); - textArea.setPreferredSize(normalized); - textArea.setMinimumSize(normalized); - textArea.setMaximumSize(new Dimension(Integer.MAX_VALUE, normalized.height)); + private static void lockTextAreaHeight(JScrollPane scrollPane, int targetHeight) { + Dimension preferred = scrollPane.getPreferredSize(); + Dimension normalized = new Dimension(Math.max(preferred.width, 320), targetHeight); + scrollPane.setPreferredSize(normalized); + scrollPane.setMinimumSize(normalized); + scrollPane.setMaximumSize(new Dimension(Integer.MAX_VALUE, normalized.height)); } /** diff --git a/src/test/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGuiTest.java b/src/test/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGuiTest.java index f9d40e8..93cd78a 100644 --- a/src/test/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGuiTest.java +++ b/src/test/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGuiTest.java @@ -6,6 +6,7 @@ import java.lang.reflect.Method; import java.nio.file.Path; import java.time.Instant; +import java.util.ArrayList; import java.util.List; import java.util.Map; import javax.swing.JButton; @@ -713,6 +714,30 @@ void recordFilterProfileSuggestionsSummarizeCommonDatasetValues() throws Excepti assertThat(suggestions).contains(" - Mexico (1)"); } + @Test + void recordFilterProfileSuggestionsShowTwentyValuesThenSummarizeRemainder() throws Exception { + Method profileHelper = BdqWorkbenchGui.class.getDeclaredMethod("profileRecordFilters", RecordDataset.class); + profileHelper.setAccessible(true); + List records = new ArrayList<>(); + for (int i = 1; i <= 21; i++) { + records.add(new CanonicalRecord("r" + i, Map.of("dwc:country", "v" + i))); + } + Object profile = profileHelper.invoke(null, new RecordDataset(records)); + Method suggestionHelper = BdqWorkbenchGui.class.getDeclaredMethod( + "renderRecordFilterValueSuggestions", + profile.getClass(), + String.class, + String.class); + suggestionHelper.setAccessible(true); + + String suggestions = (String) suggestionHelper.invoke(null, profile, "dwc:country", null); + + assertThat(suggestions).contains(" - v1 (1)"); + assertThat(suggestions).contains(" - v20 (1)"); + assertThat(suggestions).doesNotContain(" - v21 (1)"); + assertThat(suggestions).contains("... and 1 more distinct value(s)"); + } + @Test void recordFilterSuggestionsWarnWhenSavedFieldIsMissingFromCurrentDataset() throws Exception { Method profileHelper = BdqWorkbenchGui.class.getDeclaredMethod("profileRecordFilters", RecordDataset.class); From 29fe2eb85543394ed3c33853043dab26e5cb5193 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 23 Sep 2026 09:30:33 +0000 Subject: [PATCH 09/13] Fix record-filter suggestion limit test ordering expectation Co-authored-by: chicoreus <3864169+chicoreus@users.noreply.github.com> --- .../filteredpush/bdq_workbench/app/BdqWorkbenchGuiTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGuiTest.java b/src/test/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGuiTest.java index 93cd78a..061370d 100644 --- a/src/test/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGuiTest.java +++ b/src/test/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGuiTest.java @@ -733,8 +733,8 @@ void recordFilterProfileSuggestionsShowTwentyValuesThenSummarizeRemainder() thro String suggestions = (String) suggestionHelper.invoke(null, profile, "dwc:country", null); assertThat(suggestions).contains(" - v1 (1)"); - assertThat(suggestions).contains(" - v20 (1)"); - assertThat(suggestions).doesNotContain(" - v21 (1)"); + assertThat(suggestions).contains(" - v21 (1)"); + assertThat(suggestions).doesNotContain(" - v9 (1)"); assertThat(suggestions).contains("... and 1 more distinct value(s)"); } From 494744e534849263fb61469b9066ef45481d299d Mon Sep 17 00:00:00 2001 From: chicoreus Date: Wed, 23 Sep 2026 05:33:39 -0400 Subject: [PATCH 10/13] Adding instructions for copilot. --- .github/copilot-instructions.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/copilot-instructions.md diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..a0abb04 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,17 @@ +# Copilot instructions for this repository + +When creating or modifying Java code: + +1. Always add/update Javadocs for every class and every method (including private, but not test methods). +2. Keep Javadocs synchronized with signature changes: + - update all `@param` + - add/update `@return` when non-void + - add/update `@throws` when exceptions are declared +3. Use tabs (size 4), LF line endings, and K&R braces. +4. Follow Google Java naming conventions. +5. Do not leave TODO Javadocs; provide meaningful descriptions. +6. Ensure that `mvn verify` passes. +7. Ensure that each file has a license header at the top, as specified in the LICENSE file. +8. Provide comprehensive test coverage for new or modified code, and ensure that all tests pass. +9. When making changes, provide a clear and concise commit message that describes the purpose of the change. +10. When making changes, unless specifically instructed to change existing functionality, make sure that new code is backward compatible with existing code and does not break existing functionality. From ce30c5c3b71bd173f2d99633e33879bab3945ffa Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 23 Sep 2026 19:42:27 +0000 Subject: [PATCH 11/13] Handle zipped data package inputs during ingest and schema inspection Co-authored-by: chicoreus <3864169+chicoreus@users.noreply.github.com> --- .../ingest/DataPackageArchiveSupport.java | 147 ++++++++++++++++++ .../ingest/DataPackageIngestor.java | 51 ++++-- .../ingest/DatasetSchemaInspector.java | 27 ++-- .../ingest/DefaultIngestService.java | 10 +- .../ingest/RelationalDatasetIngestor.java | 98 +++++++----- .../ingest/DataPackageDialectIngestTest.java | 60 +++++++ 6 files changed, 322 insertions(+), 71 deletions(-) create mode 100644 src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageArchiveSupport.java diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageArchiveSupport.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageArchiveSupport.java new file mode 100644 index 0000000..8fb8f22 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageArchiveSupport.java @@ -0,0 +1,147 @@ +/** DataPackageArchiveSupport.java + * + * Utilities for detecting and opening Data Package manifests stored in zip archives. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.ingest; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +/** + * Utilities for Data Package manifests inside zip inputs. + */ +final class DataPackageArchiveSupport { + + /** Utility class; not instantiable. */ + private DataPackageArchiveSupport() { + } + + /** + * Reports whether a zip input contains a {@code datapackage.json} manifest. + * + * @param inputPath candidate dataset path + * @return {@code true} when the path is a zip containing a data package manifest + */ + static boolean isDataPackageArchive(Path inputPath) { + String fileName = inputPath.getFileName().toString().toLowerCase(); + if (!fileName.endsWith(".zip")) { + return false; + } + try (ZipFile zipFile = new ZipFile(inputPath.toFile())) { + return findManifestEntryName(zipFile).isPresent(); + } catch (IOException e) { + return false; + } + } + + /** + * Resolves the manifest path for either an unpacked data package or a zip-contained one. + * + *

For zip inputs this keeps the zip filesystem open while {@code reader} executes, so the + * caller may also read package data files referenced from the manifest. + * + * @param inputPath path to either {@code datapackage.json} or a zip containing it + * @param reader callback that consumes the resolved manifest path + * @param result type produced by the callback + * @return callback result + * @throws IOException if the manifest cannot be resolved or read + */ + static T withManifestPath(Path inputPath, ManifestPathReader reader) throws IOException { + if (!inputPath.getFileName().toString().toLowerCase().endsWith(".zip")) { + return reader.read(inputPath); + } + URI archiveUri = URI.create("jar:" + inputPath.toUri()); + try (FileSystem zipFs = FileSystems.newFileSystem(archiveUri, Map.of())) { + Path manifestPath = findManifestPath(zipFs) + .orElseThrow(() -> new IOException("Zip input contains no datapackage.json manifest: " + inputPath)); + return reader.read(manifestPath); + } + } + + /** + * Reads one value from a resolved manifest path. + * + * @param callback result type + */ + @FunctionalInterface + interface ManifestPathReader { + /** + * Reads one value from the provided manifest path. + * + * @param manifestPath resolved path to the data package manifest + * @return callback result + * @throws IOException if reading fails + */ + T read(Path manifestPath) throws IOException; + } + + /** + * Finds a manifest entry path within a zip archive. + * + * @param zipFile zip file to inspect + * @return matching manifest entry name when present + */ + private static Optional findManifestEntryName(ZipFile zipFile) { + return zipFile.stream() + .filter(entry -> !entry.isDirectory()) + .map(ZipEntry::getName) + .filter(DataPackageArchiveSupport::isDataPackageManifestName) + .min(Comparator.comparingInt(String::length).thenComparing(String::compareTo)); + } + + /** + * Finds a manifest path within an opened zip filesystem. + * + * @param zipFs opened zip filesystem + * @return manifest path when present + * @throws IOException if directory traversal fails + */ + private static Optional findManifestPath(FileSystem zipFs) throws IOException { + try (Stream paths = Files.walk(zipFs.getPath("/"))) { + return paths + .filter(Files::isRegularFile) + .filter(path -> isDataPackageManifestName(path.toString().replace('\\', '/'))) + .min(Comparator + .comparingInt((Path path) -> path.toString().length()) + .thenComparing(path -> path.toString())); + } + } + + /** + * Reports whether a zip entry/path names {@code datapackage.json}. + * + * @param entryName candidate path or entry name + * @return {@code true} when the name resolves to a data package manifest + */ + private static boolean isDataPackageManifestName(String entryName) { + String normalized = entryName.toLowerCase(); + return normalized.equals("datapackage.json") + || normalized.endsWith("/datapackage.json") + || normalized.endsWith("\\datapackage.json"); + } +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageIngestor.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageIngestor.java index cab0556..a2e7d65 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageIngestor.java +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageIngestor.java @@ -69,7 +69,8 @@ public class DataPackageIngestor { /** * Ingests a Darwin Core Data Package into canonical records. * - * @param dataPackagePath path to the {@code datapackage.json} manifest file + * @param dataPackagePath path to the {@code datapackage.json} manifest file, or to a zip that + * contains one * @return the dataset parsed from the manifest's first resource * @throws AppException if the manifest has no resources, or the resource cannot be read */ @@ -81,7 +82,8 @@ public RecordDataset ingest(Path dataPackagePath) { * Ingests a Darwin Core Data Package into canonical records, optionally naming which of the * package's resources to read. * - * @param dataPackagePath path to the {@code datapackage.json} manifest file + * @param dataPackagePath path to the {@code datapackage.json} manifest file, or to a zip that + * contains one * @param requestedTable the name, file name or row type of the resource to read; blank to * select one automatically * @return the dataset parsed from the selected resource @@ -90,26 +92,41 @@ public RecordDataset ingest(Path dataPackagePath) { */ public RecordDataset ingest(Path dataPackagePath, String requestedTable) { try { - JsonNode root = mapper.readTree(Files.newBufferedReader(dataPackagePath)); - JsonNode resources = root.path("resources"); - if (!resources.isArray() || resources.isEmpty()) { - throw new AppException("Data package does not include resources"); - } - Path packageDir = dataPackagePath.toAbsolutePath().getParent(); - List> tables = - DataPackageDialectParser.parseResources(mapper, root, packageDir); - if (tables.isEmpty()) { - throw new AppException("Data package declares no resource with a readable data file path" - + " (inline data and remote resource URLs are not supported): " + dataPackagePath); - } - CoreTableCandidate selected = - CoreTableSelector.select(tables, requestedTable).selected(); - return ingestResource(dataPackagePath, selected); + return DataPackageArchiveSupport.withManifestPath(dataPackagePath, + manifestPath -> ingestManifest(manifestPath, dataPackagePath, requestedTable)); } catch (IOException e) { throw new AppException("Failed to ingest Darwin Core Data Package from " + dataPackagePath, e); } } + /** + * Ingests a data package manifest already resolved to a readable filesystem path. + * + * @param manifestPath resolved path to {@code datapackage.json} + * @param sourcePath original user-supplied dataset path, used in diagnostics + * @param requestedTable the name, file name or row type of the resource to read; blank to + * select one automatically + * @return the dataset parsed from the selected resource + * @throws IOException if the manifest cannot be read + */ + private RecordDataset ingestManifest(Path manifestPath, Path sourcePath, String requestedTable) throws IOException { + JsonNode root = mapper.readTree(Files.newBufferedReader(manifestPath)); + JsonNode resources = root.path("resources"); + if (!resources.isArray() || resources.isEmpty()) { + throw new AppException("Data package does not include resources"); + } + Path packageDir = manifestPath.toAbsolutePath().getParent(); + List> tables = + DataPackageDialectParser.parseResources(mapper, root, packageDir); + if (tables.isEmpty()) { + throw new AppException("Data package declares no resource with a readable data file path" + + " (inline data and remote resource URLs are not supported): " + sourcePath); + } + CoreTableCandidate selected = + CoreTableSelector.select(tables, requestedTable).selected(); + return ingestResource(sourcePath, selected); + } + /** * Parses every data file of a resource into canonical records. * diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/DatasetSchemaInspector.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/DatasetSchemaInspector.java index f28a6ef..cfe3ea9 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/ingest/DatasetSchemaInspector.java +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/DatasetSchemaInspector.java @@ -44,6 +44,9 @@ public class DatasetSchemaInspector { public DatasetSchemaOverview inspect(Path inputPath) { String fileName = inputPath.getFileName().toString().toLowerCase(); if (fileName.endsWith(".zip")) { + if (DataPackageArchiveSupport.isDataPackageArchive(inputPath)) { + return inspectDataPackage(inputPath); + } return inspectDwcArchive(inputPath); } if (fileName.endsWith(".json") || fileName.endsWith("datapackage")) { @@ -76,17 +79,19 @@ private DatasetSchemaOverview inspectDwcArchive(Path inputPath) { private DatasetSchemaOverview inspectDataPackage(Path inputPath) { try { - JsonNode root = mapper.readTree(Files.newBufferedReader(inputPath)); - Path packageDir = inputPath.toAbsolutePath().getParent(); - List> tables = - DataPackageDialectParser.parseResources(mapper, root, packageDir); - return new DatasetSchemaOverview(tables.stream() - .map(table -> new DatasetTableSummary( - table.label(), - table.rowType(), - table.rowTypeEvidence(), - table.declaredCore())) - .toList()); + return DataPackageArchiveSupport.withManifestPath(inputPath, manifestPath -> { + JsonNode root = mapper.readTree(Files.newBufferedReader(manifestPath)); + Path packageDir = manifestPath.toAbsolutePath().getParent(); + List> tables = + DataPackageDialectParser.parseResources(mapper, root, packageDir); + return new DatasetSchemaOverview(tables.stream() + .map(table -> new DatasetTableSummary( + table.label(), + table.rowType(), + table.rowTypeEvidence(), + table.declaredCore())) + .toList()); + }); } catch (IOException e) { throw new AppException("Failed to inspect data package " + inputPath, e); } diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/DefaultIngestService.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/DefaultIngestService.java index 7118cd4..3300261 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/ingest/DefaultIngestService.java +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/DefaultIngestService.java @@ -32,9 +32,10 @@ /** * Dispatches ingestion based on source format. * - *

Inspects the input path's file name and delegates to {@link DwcArchiveIngestor} for - * {@code .zip} Darwin Core Archives or {@link DataPackageIngestor} for {@code .json}/ - * {@code datapackage} Darwin Core Data Packages. + *

Inspects the input path and delegates to {@link DwcArchiveIngestor} for Darwin Core + * Archives, and to {@link DataPackageIngestor} for data package manifests + * ({@code .json}/{@code datapackage}) and zipped data packages containing + * {@code datapackage.json}. */ public class DefaultIngestService implements IngestService { private static final Logger LOG = LoggerFactory.getLogger(DefaultIngestService.class); @@ -121,6 +122,9 @@ public RecordDataset ingest(Path inputPath, String requestedTable, String datase private RecordDataset ingestFlat(Path inputPath, String requestedTable) { String fileName = inputPath.getFileName().toString().toLowerCase(); if (fileName.endsWith(".zip")) { + if (DataPackageArchiveSupport.isDataPackageArchive(inputPath)) { + return dataPackageIngestor.ingest(inputPath, requestedTable); + } return dwcArchiveIngestor.ingest(inputPath, requestedTable); } if (fileName.endsWith(".json") || fileName.endsWith("datapackage")) { diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/RelationalDatasetIngestor.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/RelationalDatasetIngestor.java index 6d55add..8b91326 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/ingest/RelationalDatasetIngestor.java +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/RelationalDatasetIngestor.java @@ -55,6 +55,9 @@ public class RelationalDatasetIngestor { public RelationalIngestResult ingest(Path inputPath, String requestedTable) { String fileName = inputPath.getFileName().toString().toLowerCase(); if (fileName.endsWith(".zip")) { + if (DataPackageArchiveSupport.isDataPackageArchive(inputPath)) { + return ingestDataPackage(inputPath, requestedTable); + } return ingestDwcArchive(inputPath, requestedTable); } if (fileName.endsWith(".json") || fileName.endsWith("datapackage")) { @@ -105,51 +108,66 @@ private RelationalIngestResult ingestDwcArchive(Path inputPath, String requested private RelationalIngestResult ingestDataPackage(Path inputPath, String requestedTable) { try { - JsonNode root = mapper.readTree(Files.newBufferedReader(inputPath)); - Path packageDir = inputPath.toAbsolutePath().getParent(); - List> tables = - DataPackageDialectParser.parseResources(mapper, root, packageDir); - if (tables.isEmpty()) { - return new RelationalIngestResult(List.of(), new DatasetSchema(List.of(), List.of(), ""), List.of()); - } - CoreTableCandidate selected = CoreTableSelector.select(tables, requestedTable).selected(); - Map> rowsByTable = new LinkedHashMap<>(); - for (CoreTableCandidate table : tables) { - List rows = dataPackageIngestor.ingest(inputPath, table.label()).records().stream() - .map(row -> withTableProvenance(row, table.label())) - .toList(); - rowsByTable.put(table.label(), rows); - } - List relationships = new ArrayList<>(); - for (CoreTableCandidate table : tables) { - for (DataPackageForeignKey key : table.descriptor().foreignKeys()) { - String referencedTableLabel = resolveReferencedTableLabel(tables, table, key); - if (referencedTableLabel == null || !referencedTableLabel.equals(selected.label())) { - continue; - } - relationships.add(new RelationshipSchema( - table.label(), - key.field(), - referencedTableLabel, - key.referenceField(), - table.label())); - } - } - return assembleResult(selected.label(), rowsByTable, tables.stream() - .map(table -> new TableSchema( - table.label(), - table.label(), - table.rowType().name(), - table.descriptor().idColumn().isBlank() - ? table.rowType().identifierTerm() - : table.descriptor().idColumn(), - table.descriptor().columnNames())) - .toList(), relationships); + return DataPackageArchiveSupport.withManifestPath(inputPath, + manifestPath -> relationalFromDataPackageManifest(manifestPath, inputPath, requestedTable)); } catch (IOException e) { throw new AppException("Failed relational ingest for Data Package " + inputPath, e); } } + /** + * Builds a relational ingest result from a resolved Data Package manifest. + * + * @param manifestPath resolved path to {@code datapackage.json} + * @param sourcePath original user-supplied dataset path + * @param requestedTable optional preferred core/grain table + * @return relational ingest output + * @throws IOException if the manifest cannot be read + */ + private RelationalIngestResult relationalFromDataPackageManifest(Path manifestPath, Path sourcePath, + String requestedTable) throws IOException { + JsonNode root = mapper.readTree(Files.newBufferedReader(manifestPath)); + Path packageDir = manifestPath.toAbsolutePath().getParent(); + List> tables = + DataPackageDialectParser.parseResources(mapper, root, packageDir); + if (tables.isEmpty()) { + return new RelationalIngestResult(List.of(), new DatasetSchema(List.of(), List.of(), ""), List.of()); + } + CoreTableCandidate selected = CoreTableSelector.select(tables, requestedTable).selected(); + Map> rowsByTable = new LinkedHashMap<>(); + for (CoreTableCandidate table : tables) { + List rows = dataPackageIngestor.ingest(sourcePath, table.label()).records().stream() + .map(row -> withTableProvenance(row, table.label())) + .toList(); + rowsByTable.put(table.label(), rows); + } + List relationships = new ArrayList<>(); + for (CoreTableCandidate table : tables) { + for (DataPackageForeignKey key : table.descriptor().foreignKeys()) { + String referencedTableLabel = resolveReferencedTableLabel(tables, table, key); + if (referencedTableLabel == null || !referencedTableLabel.equals(selected.label())) { + continue; + } + relationships.add(new RelationshipSchema( + table.label(), + key.field(), + referencedTableLabel, + key.referenceField(), + table.label())); + } + } + return assembleResult(selected.label(), rowsByTable, tables.stream() + .map(table -> new TableSchema( + table.label(), + table.label(), + table.rowType().name(), + table.descriptor().idColumn().isBlank() + ? table.rowType().identifierTerm() + : table.descriptor().idColumn(), + table.descriptor().columnNames())) + .toList(), relationships); + } + private String resolveReferencedTableLabel(List> tables, CoreTableCandidate source, DataPackageForeignKey key) { diff --git a/src/test/java/org/filteredpush/bdq_workbench/ingest/DataPackageDialectIngestTest.java b/src/test/java/org/filteredpush/bdq_workbench/ingest/DataPackageDialectIngestTest.java index a3d17f4..f089d54 100644 --- a/src/test/java/org/filteredpush/bdq_workbench/ingest/DataPackageDialectIngestTest.java +++ b/src/test/java/org/filteredpush/bdq_workbench/ingest/DataPackageDialectIngestTest.java @@ -8,7 +8,11 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import org.filteredpush.bdq_workbench.app.AppException; import org.filteredpush.bdq_workbench.model.RecordDataset; import org.junit.jupiter.api.Test; @@ -327,6 +331,45 @@ void relationalIngestBuildsCoreGraphsFromForeignKeys(@TempDir Path tempDir) thro assertThat(relational.graphs().get(0).relatedByRelation().get("identification")).hasSize(1); } + @Test + void defaultIngestServiceReadsZippedDataPackageWithMultipleTables(@TempDir Path tempDir) throws Exception { + Map entries = new LinkedHashMap<>(); + entries.put("datapackage.json", """ + { + "resources": [ + { + "name": "occurrence", + "path": "occurrence.csv", + "schema": { + "fields": [ { "name": "occurrenceID" }, { "name": "eventDate" } ], + "primaryKey": "occurrenceID" + } + }, + { + "name": "identification", + "path": "identification.csv", + "schema": { + "fields": [ { "name": "identificationID" }, { "name": "occurrenceID" }, { "name": "scientificName" } ], + "foreignKeys": [ + { "fields": "occurrenceID", "reference": { "resource": "occurrence", "fields": "occurrenceID" } } + ] + } + } + ] + } + """.getBytes(StandardCharsets.UTF_8)); + entries.put("occurrence.csv", "occurrenceID,eventDate\nocc-1,2020-01-01\n".getBytes(StandardCharsets.UTF_8)); + entries.put("identification.csv", + "identificationID,occurrenceID,scientificName\nid-1,occ-1,Abies balsamea\n".getBytes(StandardCharsets.UTF_8)); + Path archive = Files.createTempFile(tempDir, "datapackage", ".zip"); + writeZipArchive(archive, entries); + + RecordDataset dataset = new DefaultIngestService().ingest(archive, ""); + + assertThat(dataset.records()).hasSize(1); + assertThat(dataset.records().get(0).terms()).containsEntry("scientificName", "Abies balsamea"); + } + /** * Writes a {@code datapackage.json} manifest into a directory. * @@ -354,4 +397,21 @@ private Path writeFile(Path packageDir, String fileName, Charset encoding, Strin Files.writeString(file, content, encoding); return file; } + + /** + * Writes a zip archive from named byte entries. + * + * @param archivePath path of the zip file to create + * @param entries ordered map of entry names to contents + * @throws Exception if the archive cannot be written + */ + private void writeZipArchive(Path archivePath, Map entries) throws Exception { + try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(archivePath), StandardCharsets.UTF_8)) { + for (Map.Entry entry : entries.entrySet()) { + zip.putNextEntry(new ZipEntry(entry.getKey())); + zip.write(entry.getValue()); + zip.closeEntry(); + } + } + } } From 5f67909c44c55634acf59e54f5bd19d1b64110e4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 23 Sep 2026 19:56:42 +0000 Subject: [PATCH 12/13] Fix FileSystemAlreadyExists for zipped data package ingest Co-authored-by: chicoreus <3864169+chicoreus@users.noreply.github.com> --- .../ingest/DataPackageArchiveSupport.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageArchiveSupport.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageArchiveSupport.java index 8fb8f22..5560162 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageArchiveSupport.java +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageArchiveSupport.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.net.URI; import java.nio.file.FileSystem; +import java.nio.file.FileSystemAlreadyExistsException; import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; @@ -76,10 +77,22 @@ static T withManifestPath(Path inputPath, ManifestPathReader reader) thro return reader.read(inputPath); } URI archiveUri = URI.create("jar:" + inputPath.toUri()); - try (FileSystem zipFs = FileSystems.newFileSystem(archiveUri, Map.of())) { + FileSystem zipFs; + boolean shouldClose = false; + try { + zipFs = FileSystems.newFileSystem(archiveUri, Map.of()); + shouldClose = true; + } catch (FileSystemAlreadyExistsException alreadyOpen) { + zipFs = FileSystems.getFileSystem(archiveUri); + } + try { Path manifestPath = findManifestPath(zipFs) .orElseThrow(() -> new IOException("Zip input contains no datapackage.json manifest: " + inputPath)); return reader.read(manifestPath); + } finally { + if (shouldClose) { + zipFs.close(); + } } } From 1f97fabdd73ad9075f845574dba695e46628b5d0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 23 Sep 2026 20:39:53 +0000 Subject: [PATCH 13/13] Fix built-in mapping for scientificName from joined tables Co-authored-by: chicoreus <3864169+chicoreus@users.noreply.github.com> --- .../ingest/BuiltInDatasetViews.java | 66 ++++++++++++++++--- .../ingest/DatasetViewSchemaTest.java | 26 ++++++++ 2 files changed, 83 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/BuiltInDatasetViews.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/BuiltInDatasetViews.java index 719f497..9e50760 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/ingest/BuiltInDatasetViews.java +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/BuiltInDatasetViews.java @@ -20,8 +20,10 @@ package org.filteredpush.bdq_workbench.ingest; import java.util.ArrayList; +import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; +import java.util.Set; import org.filteredpush.bdq_workbench.model.DatasetSchema; import org.filteredpush.bdq_workbench.model.DatasetView; import org.filteredpush.bdq_workbench.model.DatasetViewCardinalityPolicy; @@ -77,7 +79,7 @@ private static Optional dwcDataPackageOccurrenceView(DatasetSchema if (joins.isEmpty()) { return Optional.empty(); } - List mappings = defaultOccurrenceMappings(occurrence.name(), joins); + List mappings = defaultOccurrenceMappings(schema, occurrence.name(), joins); return Optional.of(new DatasetView(occurrence.name(), schema.schemaFingerprint(), joins, mappings)); } @@ -97,18 +99,64 @@ private static Optional eventCoreOccurrenceExtension(DatasetSchema occurrence.name(), occurrence.name(), DatasetViewCardinalityPolicy.FIRST_ROW)); - List mappings = defaultOccurrenceMappings(occurrence.name(), joins); + List mappings = defaultOccurrenceMappings(schema, occurrence.name(), joins); return Optional.of(new DatasetView(event.name(), schema.schemaFingerprint(), joins, mappings)); } - private static List defaultOccurrenceMappings(String occurrenceTable, List joins) { - String sourceTable = joins.isEmpty() ? "core" : occurrenceTable; + /** + * Builds default occurrence-oriented mappings, preferring a joined source table when the + * occurrence table does not carry a mapped term. + * + * @param schema discovered dataset schema + * @param occurrenceTable name of the occurrence table + * @param joins joins included in the selected built-in view + * @return direct source mappings for common occurrence terms + */ + private static List defaultOccurrenceMappings( + DatasetSchema schema, + String occurrenceTable, + List joins) { + Set allowedSourceTables = new LinkedHashSet<>(); + allowedSourceTables.add(occurrenceTable); + joins.forEach(join -> allowedSourceTables.add(join.sourceTable())); return List.of( - new DatasetViewMapping("occurrenceID", sourceTable, "occurrenceID"), - new DatasetViewMapping("scientificName", sourceTable, "scientificName"), - new DatasetViewMapping("eventDate", sourceTable, "eventDate"), - new DatasetViewMapping("decimalLatitude", sourceTable, "decimalLatitude"), - new DatasetViewMapping("decimalLongitude", sourceTable, "decimalLongitude")); + new DatasetViewMapping("occurrenceID", + resolveSourceTable(schema, allowedSourceTables, occurrenceTable, "occurrenceID"), + "occurrenceID"), + new DatasetViewMapping("scientificName", + resolveSourceTable(schema, allowedSourceTables, occurrenceTable, "scientificName"), + "scientificName"), + new DatasetViewMapping("eventDate", + resolveSourceTable(schema, allowedSourceTables, occurrenceTable, "eventDate"), + "eventDate"), + new DatasetViewMapping("decimalLatitude", + resolveSourceTable(schema, allowedSourceTables, occurrenceTable, "decimalLatitude"), + "decimalLatitude"), + new DatasetViewMapping("decimalLongitude", + resolveSourceTable(schema, allowedSourceTables, occurrenceTable, "decimalLongitude"), + "decimalLongitude")); + } + + /** + * Resolves which allowed table should source a mapped term. + * + * @param schema discovered dataset schema + * @param allowedSourceTables tables permitted by the built-in view + * @param fallbackTable table name used when no candidate table carries the mapped column + * @param columnName column/term being mapped + * @return table name to use as mapping source + */ + private static String resolveSourceTable( + DatasetSchema schema, + Set allowedSourceTables, + String fallbackTable, + String columnName) { + return schema.tables().stream() + .filter(table -> allowedSourceTables.contains(table.name())) + .filter(table -> table.columns().contains(columnName)) + .map(TableSchema::name) + .findFirst() + .orElse(fallbackTable); } private static TableSchema findByRowType(DatasetSchema schema, String rowType) { diff --git a/src/test/java/org/filteredpush/bdq_workbench/ingest/DatasetViewSchemaTest.java b/src/test/java/org/filteredpush/bdq_workbench/ingest/DatasetViewSchemaTest.java index ae05ae2..3b56f35 100644 --- a/src/test/java/org/filteredpush/bdq_workbench/ingest/DatasetViewSchemaTest.java +++ b/src/test/java/org/filteredpush/bdq_workbench/ingest/DatasetViewSchemaTest.java @@ -75,6 +75,32 @@ void builtInViewIsOnlyAppliedWhenRelationshipsArePresent() { assertThat(BuiltInDatasetViews.select(withRelationship, new java.util.ArrayList<>())).isPresent(); } + @Test + void builtInDataPackageViewSourcesScientificNameFromJoinedTableWhenOccurrenceLacksColumn() { + DatasetSchema schema = new DatasetSchema( + List.of( + new TableSchema("occurrence", "occurrence", "OCCURRENCE", "occurrenceID", + List.of("occurrenceID", "eventDate", "decimalLatitude", "decimalLongitude")), + new TableSchema("identification", "identification", "OTHER", "identificationID", + List.of("occurrenceID", "scientificName"))), + List.of(new RelationshipSchema( + "identification", + "occurrenceID", + "occurrence", + "occurrenceID", + "identification")), + "fp"); + + DatasetView view = BuiltInDatasetViews.select(schema, new java.util.ArrayList<>()).orElseThrow(); + + assertThat(view.mappings()) + .anySatisfy(mapping -> { + assertThat(mapping.term()).isEqualTo("scientificName"); + assertThat(mapping.sourceTable()).isEqualTo("identification"); + assertThat(mapping.sourceColumn()).isEqualTo("scientificName"); + }); + } + @Test void viewLoadFailureIsReportedAsAppException(@TempDir Path tempDir) { DatasetViewIO io = new DatasetViewIO();