diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..a0abb04 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,17 @@ +# Copilot instructions for this repository + +When creating or modifying Java code: + +1. Always add/update Javadocs for every class and every method (including private, but not test methods). +2. Keep Javadocs synchronized with signature changes: + - update all `@param` + - add/update `@return` when non-void + - add/update `@throws` when exceptions are declared +3. Use tabs (size 4), LF line endings, and K&R braces. +4. Follow Google Java naming conventions. +5. Do not leave TODO Javadocs; provide meaningful descriptions. +6. Ensure that `mvn verify` passes. +7. Ensure that each file has a license header at the top, as specified in the LICENSE file. +8. Provide comprehensive test coverage for new or modified code, and ensure that all tests pass. +9. When making changes, provide a clear and concise commit message that describes the purpose of the change. +10. When making changes, unless specifically instructed to change existing functionality, make sure that new code is backward compatible with existing code and does not break existing functionality. diff --git a/CLAUDE.md b/CLAUDE.md index 977ac86..6797a98 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -56,14 +56,15 @@ mistyped path fails fast and offline — keep that ordering, it is what keeps th touching the network. Configuration defaults live in `src/main/resources/application.properties` -(`bdq.usecase.file`, `bdq.rdf.files`, `bdq.dataset`, `bdq.dataset.table`, `bdq.usecase.id`, +(`bdq.usecase.file`, `bdq.rdf.files`, `bdq.dataset`, `bdq.dataset.table`, `bdq.dataset.view`, `bdq.usecase.id`, `bdq.discovery.packages`, `bdq.threads`, `bdq.execution.dedup`) and are merged with CLI/GUI overrides by `ConfigLoader`. `bdq.usecase.file` and `bdq.rdf.files` ship blank, which means "use the `WorkbenchDefaults` published sources"; set either to a local path or an HTTP URL to pin a run. `bdq.dataset.table` (CLI `--dataset-table`, GUI advanced options) names which table of a multi-table dataset to run against. Both entry points fetch and cache use-case/test-definition/ ontology RDF from `bdq.tdwg.org` through `CachedResourceResolver`; RDF/XML, Turtle, and JSON-LD -serializations are all supported. Logging is DEBUG-by-default to the +serializations are all supported. `bdq.dataset.view` (CLI `--dataset-view`, GUI "Build Dataset View...") +names a standalone JSON DatasetView used to flatten relational inputs before execution. Logging is DEBUG-by-default to the console via `src/main/resources/logback.xml`. ## Architecture diff --git a/README.md b/README.md index d25e334..cd440e2 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,14 @@ Configuration defaults are in `src/main/resources/application.properties` and ca java -jar target/bdq_workbench-0.1.0-SNAPSHOT.jar --dataset path/to/dataset.zip ``` +Reusable relational flattening views can be supplied with `--dataset-view` (`bdq.dataset.view` in config): + +```bash +java -jar target/bdq_workbench-0.1.0-SNAPSHOT.jar \ + --dataset path/to/datapackage.json \ + --dataset-view path/to/bdq-dataset-view.json +``` + Configuration precedence is: 1. command-line or GUI-supplied overrides @@ -76,7 +84,7 @@ Dataset input | +--> Load DwC-A zip / Data Package CSV | - +--> [optional] Build record filters from dataset terms/values (GUI) + +--> [optional] Build dataset view and/or record filters from dataset terms/values (GUI) | +--> [optional] Apply record filters | diff --git a/src/main/java/org/filteredpush/bdq_workbench/app/AppConfig.java b/src/main/java/org/filteredpush/bdq_workbench/app/AppConfig.java index 8a099ed..d4957da 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/app/AppConfig.java +++ b/src/main/java/org/filteredpush/bdq_workbench/app/AppConfig.java @@ -48,17 +48,19 @@ * @param datasetTable which of the input dataset's tables to run against — a Darwin Core * Archive's core or one of its extensions, or one of a Data Package's resources — named by * its location, resource name or Darwin Core row type; empty to let the ingestor choose + * @param datasetView optional path to a standalone dataset view JSON file */ public record AppConfig( - Path useCaseXml, - List rdfDefinitions, - Path datasetPath, - String useCaseId, - List implementationPackages, - int threadCount, - boolean dedupEnabled, - RecordFilterSpec recordFilter, - String datasetTable) { + Path useCaseXml, + List rdfDefinitions, + Path datasetPath, + String useCaseId, + List implementationPackages, + int threadCount, + boolean dedupEnabled, + RecordFilterSpec recordFilter, + String datasetTable, + String datasetView) { /** * Creates a configuration with no record filters. @@ -80,7 +82,7 @@ public AppConfig( int threadCount, boolean dedupEnabled) { this(useCaseXml, rdfDefinitions, datasetPath, useCaseId, implementationPackages, threadCount, dedupEnabled, - RecordFilterSpec.empty(), ""); + RecordFilterSpec.empty(), "", ""); } /** @@ -105,7 +107,24 @@ public AppConfig( boolean dedupEnabled, RecordFilterSpec recordFilter) { this(useCaseXml, rdfDefinitions, datasetPath, useCaseId, implementationPackages, threadCount, dedupEnabled, - recordFilter, ""); + recordFilter, "", ""); + } + + /** + * Creates a configuration with explicit table selection and no dataset view. + */ + public AppConfig( + Path useCaseXml, + List rdfDefinitions, + Path datasetPath, + String useCaseId, + List implementationPackages, + int threadCount, + boolean dedupEnabled, + RecordFilterSpec recordFilter, + String datasetTable) { + this(useCaseXml, rdfDefinitions, datasetPath, useCaseId, implementationPackages, threadCount, dedupEnabled, + recordFilter, datasetTable, ""); } /** @@ -115,5 +134,6 @@ public AppConfig( public AppConfig { recordFilter = recordFilter == null ? RecordFilterSpec.empty() : recordFilter; datasetTable = datasetTable == null ? "" : datasetTable.trim(); + datasetView = datasetView == null ? "" : datasetView.trim(); } } diff --git a/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchApplication.java b/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchApplication.java index 1e44d18..45abeaf 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchApplication.java +++ b/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchApplication.java @@ -191,7 +191,8 @@ private static AppConfig resolveDefaultUseCase(AppConfig config) { config.threadCount(), config.dedupEnabled(), config.recordFilter(), - config.datasetTable()); + config.datasetTable(), + config.datasetView()); } /** @@ -248,6 +249,7 @@ private static ParseResult parseArguments(String[] args) { String key = switch (arg) { case "--dataset" -> "bdq.dataset"; case "--dataset-table" -> "bdq.dataset.table"; + case "--dataset-view" -> "bdq.dataset.view"; case "--usecase-file" -> "bdq.usecase.file"; case "--rdf-files" -> "bdq.rdf.files"; case "--usecase-id" -> "bdq.usecase.id"; @@ -290,6 +292,7 @@ private static void renderUsage(PrintStream out) { out.println(" --dataset-table Which table of the dataset to run against, named by"); out.println(" location, resource name or Darwin Core row type"); out.println(" (default: the best-ranked table the dataset offers)"); + out.println(" --dataset-view Standalone dataset view JSON file"); out.println(" --usecase-file Use case XML file"); out.println(" --rdf-files Comma-separated RDF/OWL files"); out.println(" --usecase-id Optional use case identifier"); diff --git a/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGui.java b/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGui.java index a5cb807..e11d9c9 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGui.java +++ b/src/main/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGui.java @@ -68,10 +68,18 @@ import org.filteredpush.bdq_workbench.execution.ExecutionProgressListener; import org.filteredpush.bdq_workbench.execution.ParallelPhaseExecutionService; import org.filteredpush.bdq_workbench.execution.ReflectionExecutionAdapter; +import org.filteredpush.bdq_workbench.ingest.DatasetViewIO; import org.filteredpush.bdq_workbench.ingest.DefaultIngestService; +import org.filteredpush.bdq_workbench.ingest.DatasetSchemaInspector; +import org.filteredpush.bdq_workbench.ingest.RelationalDatasetIngestor; import org.filteredpush.bdq_workbench.model.RecordFilterSummary; import org.filteredpush.bdq_workbench.model.BindingReview; import org.filteredpush.bdq_workbench.model.BuiltInMeasureSpec; +import org.filteredpush.bdq_workbench.model.DatasetSchema; +import org.filteredpush.bdq_workbench.model.DatasetView; +import org.filteredpush.bdq_workbench.model.DatasetViewCardinalityPolicy; +import org.filteredpush.bdq_workbench.model.DatasetViewJoin; +import org.filteredpush.bdq_workbench.model.DatasetViewMapping; import org.filteredpush.bdq_workbench.model.ExecutionPlan; import org.filteredpush.bdq_workbench.model.ExecutionSummary; import org.filteredpush.bdq_workbench.model.ImplementationBinding; @@ -122,6 +130,7 @@ */ final class BdqWorkbenchGui { private static final Logger LOG = LoggerFactory.getLogger(BdqWorkbenchGui.class); + private static final int RECORD_FILTER_SUGGESTION_LIMIT = 20; private static final int FINALIZATION_STAGE_STEP_COUNT = 5; private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @@ -305,6 +314,8 @@ private static JFrame createFrame(AppConfig defaults, Map overri form.add(setupHeaderRow); PickerField dataset = addPickerField(form, frame, "Dataset", defaults.datasetPath().toString()); + PickerField datasetView = addPickerField(form, frame, "Dataset view (JSON)", + overrides.getOrDefault("bdq.dataset.view", defaults.datasetView())); String[] configuredRecordFilters = new String[] {defaults.recordFilter().toPropertyString()}; JTextArea recordFilterSummary = new JTextArea(4, 40); recordFilterSummary.setEditable(false); @@ -317,8 +328,13 @@ private static JFrame createFrame(AppConfig defaults, Map overri recordFilterRow.add(new JLabel("Record filters"), BorderLayout.WEST); JPanel recordFilterButtons = new JPanel(new FlowLayout(FlowLayout.LEFT, 8, 0)); JButton buildRecordFilters = new JButton("Build Record Filters..."); + JButton buildDatasetView = new JButton("Build Dataset View..."); JButton clearRecordFilters = new JButton("Clear Filters"); + JButton clearDatasetView = new JButton("Clear View"); clearRecordFilters.setEnabled(!configuredRecordFilters[0].isBlank()); + clearDatasetView.setEnabled(!datasetView.field().getText().isBlank()); + recordFilterButtons.add(buildDatasetView); + recordFilterButtons.add(clearDatasetView); recordFilterButtons.add(buildRecordFilters); recordFilterButtons.add(clearRecordFilters); recordFilterRow.add(recordFilterButtons, BorderLayout.CENTER); @@ -485,6 +501,27 @@ private static JFrame createFrame(AppConfig defaults, Map overri recordFilterSummary, clearRecordFilters, buildRecordFilters)); + buildDatasetView.addActionListener(e -> loadDatasetViewDialog(frame, dataset.field().getText().trim(), datasetView.field())); + clearDatasetView.addActionListener(e -> { + datasetView.field().setText(""); + clearDatasetView.setEnabled(false); + }); + datasetView.field().getDocument().addDocumentListener(new javax.swing.event.DocumentListener() { + @Override + public void insertUpdate(javax.swing.event.DocumentEvent e) { + clearDatasetView.setEnabled(!datasetView.field().getText().isBlank()); + } + + @Override + public void removeUpdate(javax.swing.event.DocumentEvent e) { + clearDatasetView.setEnabled(!datasetView.field().getText().isBlank()); + } + + @Override + public void changedUpdate(javax.swing.event.DocumentEvent e) { + clearDatasetView.setEnabled(!datasetView.field().getText().isBlank()); + } + }); clearRecordFilters.addActionListener(e -> { configuredRecordFilters[0] = ""; updateRecordFilterSummary(recordFilterSummary, configuredRecordFilters[0]); @@ -536,6 +573,7 @@ protected PreflightState doInBackground() { selectedUseCaseId(useCaseChoice), configuredRecordFilters[0], datasetTable.getText().trim(), + datasetView.field().getText().trim(), useCaseSource.getText().trim(), testDefinitionsSource.getText().trim(), additionalTestDefinitions.getText().trim(), @@ -1395,6 +1433,7 @@ private static WorkbenchFacade createFacade( * form * @param datasetTable which of the dataset's tables to run against, named by location, * resource name or Darwin Core row type; blank to let the ingestor choose + * @param datasetView optional dataset-view JSON file path * @param useCaseSource use case RDF file/URL field value * @param testDefinitionsSource primary test definitions file/URL field value * @param additionalTestDefinitions comma-separated extra test definition files/URLs @@ -1411,6 +1450,7 @@ private static AppConfig buildConfig( String selectedUseCaseId, String recordFilters, String datasetTable, + String datasetView, String useCaseSource, String testDefinitionsSource, String additionalTestDefinitions, @@ -1446,8 +1486,60 @@ private static AppConfig buildConfig( parseThreads(threads), dedupEnabled, RecordFilterSpec.parse(recordFilters), - datasetTable); - } + datasetTable, + datasetView); + } + + /** + * Backward-compatible build-config helper used by reflection-based tests. + * + *

Delegates to the dataset-view-aware overload with an empty dataset view path. + * + * @param dataset dataset file path field value + * @param selectedUseCaseId ID of the use case chosen in the combo box + * @param recordFilters record-filter field value + * @param datasetTable selected dataset table name + * @param useCaseSource use case RDF file/URL field value + * @param testDefinitionsSource primary test definitions file/URL field value + * @param additionalTestDefinitions comma-separated extra test definition files/URLs + * @param ontologySource BDQ FFDQ ontology file/URL field value + * @param discoveryPackages comma-separated implementation discovery packages + * @param threads thread count field value + * @param dedupEnabled whether distinct-value execution is enabled for this run + * @param resolver resolves and caches remote/local resource paths + * @param defaults fallback values used when a field is blank + * @return the assembled configuration + */ + private static AppConfig buildConfig( + String dataset, + String selectedUseCaseId, + String recordFilters, + String datasetTable, + String useCaseSource, + String testDefinitionsSource, + String additionalTestDefinitions, + String ontologySource, + String discoveryPackages, + String threads, + boolean dedupEnabled, + CachedResourceResolver resolver, + AppConfig defaults) { + return buildConfig( + dataset, + selectedUseCaseId, + recordFilters, + datasetTable, + "", + useCaseSource, + testDefinitionsSource, + additionalTestDefinitions, + ontologySource, + discoveryPackages, + threads, + dedupEnabled, + resolver, + defaults); + } /** * Parses the thread-count field value, rejecting non-numeric or non-positive values. @@ -1785,6 +1877,193 @@ protected void done() { worker.execute(); } + /** + * Loads relational schema metadata and helps the user create/save a dataset view file. + * + * @param frame owner frame + * @param datasetPath selected dataset path + * @param datasetViewField dataset-view path field to update + */ + private static void loadDatasetViewDialog(JFrame frame, String datasetPath, JTextField datasetViewField) { + if (datasetPath == null || datasetPath.isBlank()) { + JOptionPane.showMessageDialog( + frame, + "Select a dataset before building a dataset view.", + "Dataset required", + JOptionPane.WARNING_MESSAGE); + return; + } + Path path = Path.of(datasetPath); + if (!Files.exists(path)) { + JOptionPane.showMessageDialog( + frame, + "Dataset input not found: " + datasetPath, + "Dataset required", + JOptionPane.ERROR_MESSAGE); + return; + } + DatasetSchemaInspector.DatasetSchemaOverview overview = new DatasetSchemaInspector().inspect(path); + if (overview.tables().size() <= 1) { + JOptionPane.showMessageDialog( + frame, + overview.describeTables() + ".\nBuild Dataset View is only needed when the dataset has related tables.", + "Dataset view not needed", + JOptionPane.INFORMATION_MESSAGE); + return; + } + SwingWorker worker = new SwingWorker<>() { + @Override + protected DatasetViewPreview doInBackground() { + RelationalDatasetIngestor ingestor = new RelationalDatasetIngestor(); + var relational = ingestor.ingest(path, ""); + DatasetSchema schema = relational.schema(); + DatasetView suggested = suggestDatasetView(schema); + var preview = new org.filteredpush.bdq_workbench.ingest.ViewFlattener().flatten(relational, suggested); + return new DatasetViewPreview(schema, suggested, preview); + } + + @Override + protected void done() { + try { + openDatasetViewDialog(frame, datasetViewField, get()); + } catch (Exception e) { + Throwable cause = e.getCause() == null ? e : e.getCause(); + JOptionPane.showMessageDialog( + frame, + "Unable to inspect dataset for dataset views: " + cause.getMessage(), + "Dataset view setup failed", + JOptionPane.ERROR_MESSAGE); + } + } + }; + worker.execute(); + } + + private static void openDatasetViewDialog(JFrame frame, JTextField datasetViewField, DatasetViewPreview previewData) { + DatasetViewIO io = new DatasetViewIO(); + DatasetSchema schema = previewData.schema(); + DatasetView suggested = previewData.suggested(); + var preview = previewData.preview(); + JTextArea details = new JTextArea(18, 80); + details.setEditable(false); + details.setLineWrap(true); + details.setWrapStyleWord(true); + StringBuilder builder = new StringBuilder(); + builder.append("Schema fingerprint: ").append(schema.schemaFingerprint()).append('\n'); + builder.append("Tables:\n"); + schema.tables().forEach(table -> builder.append(" - ") + .append(table.name()) + .append(" [") + .append(table.rowType()) + .append("] columns=") + .append(table.columns().size()) + .append('\n')); + builder.append("Relationships:\n"); + schema.relationships().forEach(relationship -> builder.append(" - ") + .append(relationship.fromTable()) + .append('.') + .append(relationship.fromColumn()) + .append(" -> ") + .append(relationship.toTable()) + .append('.') + .append(relationship.toColumn()) + .append('\n')); + builder.append("Suggested mappings:\n"); + suggested.mappings().forEach(mapping -> builder.append(" - ") + .append(mapping.term()) + .append(" <- ") + .append(mapping.sourceTable()) + .append('.') + .append(mapping.sourceColumn()) + .append('\n')); + builder.append("Preview rows: ").append(Math.min(5, preview.dataset().records().size())).append('\n'); + preview.dataset().records().stream().limit(5).forEach(row -> builder.append(" - ") + .append(row.id()) + .append(" => ") + .append(row.terms()) + .append('\n')); + if (!preview.diagnostics().isEmpty()) { + builder.append("Cardinality warnings:\n"); + preview.diagnostics().forEach(message -> builder.append(" - ").append(message).append('\n')); + } + details.setText(builder.toString()); + JButton save = new JButton("Save View..."); + JButton load = new JButton("Load View..."); + JPanel buttons = new JPanel(new FlowLayout(FlowLayout.RIGHT)); + buttons.add(load); + buttons.add(save); + JPanel panel = new JPanel(new BorderLayout(8, 8)); + panel.add(new JScrollPane(details), BorderLayout.CENTER); + panel.add(buttons, BorderLayout.SOUTH); + JDialog dialog = new JDialog(frame, "Build Dataset View", true); + dialog.setContentPane(panel); + dialog.pack(); + dialog.setLocationRelativeTo(frame); + + load.addActionListener(e -> { + String selected = chooseFile(frame, "Select dataset view JSON"); + if (selected == null) { + return; + } + try { + DatasetView view = io.load(Path.of(selected)); + io.validateCompatibility(view, schema); + datasetViewField.setText(selected); + dialog.dispose(); + } catch (AppException ex) { + JOptionPane.showMessageDialog( + frame, + "Unable to load dataset view: " + ex.getMessage(), + "Dataset view load failed", + JOptionPane.ERROR_MESSAGE); + } + }); + save.addActionListener(e -> { + String selected = chooseSaveFile(frame, "Save dataset view JSON", "bdq-dataset-view.json"); + if (selected == null) { + return; + } + io.save(Path.of(selected), suggested); + datasetViewField.setText(selected); + dialog.dispose(); + }); + dialog.setVisible(true); + } + + /** + * Creates a simple direct-mapping view suggestion from discovered schema. + */ + private static DatasetView suggestDatasetView(DatasetSchema schema) { + String grain = schema.tables().stream() + .filter(table -> "OCCURRENCE".equalsIgnoreCase(table.rowType())) + .findFirst() + .map(table -> table.name()) + .orElse(schema.tables().isEmpty() ? "core" : schema.tables().get(0).name()); + List joins = schema.relationships().stream() + .filter(relationship -> relationship.toTable().equals(grain)) + .map(relationship -> new DatasetViewJoin( + relationship.relationName(), + relationship.fromTable(), + DatasetViewCardinalityPolicy.REJECT)) + .toList(); + java.util.Set allowedTables = new java.util.LinkedHashSet<>(); + allowedTables.add(grain); + joins.forEach(join -> allowedTables.add(join.sourceTable())); + List requestedTerms = List.of( + "occurrenceID", "scientificName", "eventDate", "decimalLatitude", "decimalLongitude"); + List mappings = new ArrayList<>(); + for (String term : requestedTerms) { + String sourceTable = schema.tables().stream() + .filter(table -> allowedTables.contains(table.name())) + .filter(table -> table.columns().contains(term)) + .findFirst() + .map(table -> table.name()) + .orElse(grain); + mappings.add(new DatasetViewMapping(term, sourceTable, term)); + } + return new DatasetView(grain, schema.schemaFingerprint(), joins, mappings); + } + /** * Opens the interactive record-filter dialog for one already-profiled dataset. * @@ -1910,14 +2189,17 @@ private static RecordFilterRowWidgets addRecordFilterRow( } JTextField values = new JTextField(initialValues == null ? "" : initialValues); forceSingleLineControlHeight(fieldChoice, values.getPreferredSize().height); - JTextArea suggestionArea = new JTextArea(3, 30); + forceSingleLineControlHeight(values, values.getPreferredSize().height); + JTextArea suggestionArea = new JTextArea(8, 30); suggestionArea.setEditable(false); suggestionArea.setLineWrap(true); suggestionArea.setWrapStyleWord(true); suggestionArea.setBorder(BorderFactory.createEtchedBorder()); - lockTextAreaHeight(suggestionArea); installTextAreaClipboardSupport(suggestionArea); + JScrollPane suggestionScroll = new JScrollPane(suggestionArea); + lockTextAreaHeight(suggestionScroll, suggestionArea.getPreferredSize().height + 60); JButton remove = new JButton("Remove"); + forceSingleLineControlHeight(remove, values.getPreferredSize().height); remove.addActionListener(e -> { rowsPanel.remove(row); rowsPanel.revalidate(); @@ -1944,10 +2226,11 @@ private static RecordFilterRowWidgets addRecordFilterRow( stacked.setLayout(new BoxLayout(stacked, BoxLayout.Y_AXIS)); stacked.add(inputRow); stacked.add(valuesRow); - stacked.add(suggestionArea); + stacked.add(suggestionScroll); row.add(stacked, BorderLayout.CENTER); row.setBorder(BorderFactory.createEmptyBorder(4, 0, 4, 0)); + row.setMaximumSize(new Dimension(Integer.MAX_VALUE, row.getPreferredSize().height)); rowsPanel.add(row); return new RecordFilterRowWidgets(row, fieldChoice, values, unresolvedField, unresolvedMessage); } @@ -1977,7 +2260,7 @@ private static RecordFilterDatasetProfile profileRecordFilters(RecordDataset dat .thenComparing(Map.Entry::getKey, String.CASE_INSENSITIVE_ORDER)); distinctValueCounts.put(term, entries.size()); topValuesByTerm.put(term, entries.stream() - .limit(8) + .limit(RECORD_FILTER_SUGGESTION_LIMIT) .map(entry -> new RecordFilterValueOption(entry.getKey(), entry.getValue())) .toList()); }); @@ -2250,17 +2533,46 @@ private static void forceSingleLineControlHeight(JComboBox combo, int targetH } /** - * Fixes a text area's preferred/minimum/maximum height so dynamic wrapped text does not cause + * Forces a text field to keep a single-line control height. + * + * @param textField the text field to normalize + * @param targetHeight the desired control height in pixels + */ + private static void forceSingleLineControlHeight(JTextField textField, int targetHeight) { + Dimension preferred = textField.getPreferredSize(); + Dimension normalized = new Dimension(preferred.width, targetHeight); + textField.setPreferredSize(normalized); + textField.setMinimumSize(normalized); + textField.setMaximumSize(new Dimension(Integer.MAX_VALUE, targetHeight)); + } + + /** + * Forces a button to keep a single-line control height. + * + * @param button the button to normalize + * @param targetHeight the desired control height in pixels + */ + private static void forceSingleLineControlHeight(JButton button, int targetHeight) { + Dimension preferred = button.getPreferredSize(); + Dimension normalized = new Dimension(preferred.width, targetHeight); + button.setPreferredSize(normalized); + button.setMinimumSize(normalized); + button.setMaximumSize(normalized); + } + + /** + * Fixes a scroll pane's preferred/minimum/maximum height so dynamic wrapped text does not cause * surrounding filter rows to resize and scroll the selected controls out of view. * - * @param textArea the text area whose height should remain stable + * @param scrollPane the scroll pane whose height should remain stable + * @param targetHeight the desired control height in pixels */ - private static void lockTextAreaHeight(JTextArea textArea) { - Dimension preferred = textArea.getPreferredSize(); - Dimension normalized = new Dimension(Math.max(preferred.width, 320), preferred.height); - textArea.setPreferredSize(normalized); - textArea.setMinimumSize(normalized); - textArea.setMaximumSize(new Dimension(Integer.MAX_VALUE, normalized.height)); + private static void lockTextAreaHeight(JScrollPane scrollPane, int targetHeight) { + Dimension preferred = scrollPane.getPreferredSize(); + Dimension normalized = new Dimension(Math.max(preferred.width, 320), targetHeight); + scrollPane.setPreferredSize(normalized); + scrollPane.setMinimumSize(normalized); + scrollPane.setMaximumSize(new Dimension(Integer.MAX_VALUE, normalized.height)); } /** @@ -3399,6 +3711,13 @@ private record PickerField(JTextField field, JButton button) { private record RecordFilterValueOption(String value, long count) { } + /** Prepared schema/view/preview tuple for the dataset-view builder dialog. */ + private record DatasetViewPreview( + DatasetSchema schema, + DatasetView suggested, + org.filteredpush.bdq_workbench.ingest.ViewFlattenResult preview) { + } + /** Dataset-derived terms and value counts used to build record filters interactively. */ private record RecordFilterDatasetProfile( int recordCount, diff --git a/src/main/java/org/filteredpush/bdq_workbench/app/ConfigLoader.java b/src/main/java/org/filteredpush/bdq_workbench/app/ConfigLoader.java index 7c3439c..4a39a13 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/app/ConfigLoader.java +++ b/src/main/java/org/filteredpush/bdq_workbench/app/ConfigLoader.java @@ -123,7 +123,8 @@ private AppConfig load(Map overrides, CachedResourceResolver res parseThreadCount(getValue(defaults, overrides, "bdq.threads", "4")), parseBoolean(getValue(defaults, overrides, "bdq.execution.dedup", "true"), "bdq.execution.dedup"), RecordFilterSpec.parse(getValue(defaults, overrides, "bdq.record.filters", "")), - getValue(defaults, overrides, "bdq.dataset.table", "")); + getValue(defaults, overrides, "bdq.dataset.table", ""), + getValue(defaults, overrides, "bdq.dataset.view", "")); } /** diff --git a/src/main/java/org/filteredpush/bdq_workbench/app/WorkbenchFacade.java b/src/main/java/org/filteredpush/bdq_workbench/app/WorkbenchFacade.java index 794022d..306fa38 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/app/WorkbenchFacade.java +++ b/src/main/java/org/filteredpush/bdq_workbench/app/WorkbenchFacade.java @@ -159,7 +159,7 @@ public WorkbenchFacade( * @return the prepared run, ready for execution */ public PreparedRun prepare(AppConfig config) { - var ingestedDataset = ingestService.ingest(config.datasetPath(), config.datasetTable()); + var ingestedDataset = ingestService.ingest(config.datasetPath(), config.datasetTable(), config.datasetView()); RecordFilterSummary filterSummary = recordFilterService.apply(ingestedDataset, config.recordFilter()); var dataset = filterSummary.filteredDataset(); ExecutionPlan plan = policyResolverService.resolve(config.useCaseId()); diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/BuiltInDatasetViews.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/BuiltInDatasetViews.java new file mode 100644 index 0000000..9e50760 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/BuiltInDatasetViews.java @@ -0,0 +1,168 @@ +/** BuiltInDatasetViews.java + * + * Built-in reusable dataset view definitions. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.ingest; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import org.filteredpush.bdq_workbench.model.DatasetSchema; +import org.filteredpush.bdq_workbench.model.DatasetView; +import org.filteredpush.bdq_workbench.model.DatasetViewCardinalityPolicy; +import org.filteredpush.bdq_workbench.model.DatasetViewJoin; +import org.filteredpush.bdq_workbench.model.DatasetViewMapping; +import org.filteredpush.bdq_workbench.model.RelationshipSchema; +import org.filteredpush.bdq_workbench.model.TableSchema; + +/** + * Selector for built-in default dataset views. + */ +final class BuiltInDatasetViews { + + private BuiltInDatasetViews() { + } + + /** + * Picks an applicable built-in view for the given schema, if any. + * + * @param schema discovered schema + * @param diagnostics receives applicability diagnostics + * @return built-in view when applicable + */ + static Optional select(DatasetSchema schema, List diagnostics) { + Optional eventOccurrence = eventCoreOccurrenceExtension(schema); + if (eventOccurrence.isPresent()) { + return eventOccurrence; + } + Optional dataPackage = dwcDataPackageOccurrenceView(schema); + if (dataPackage.isPresent()) { + return dataPackage; + } + diagnostics.add("No built-in dataset view matched the dataset schema fingerprint " + + schema.schemaFingerprint() + "; falling back to flat ingest"); + return Optional.empty(); + } + + private static Optional dwcDataPackageOccurrenceView(DatasetSchema schema) { + TableSchema occurrence = findByRowType(schema, "OCCURRENCE"); + if (occurrence == null) { + return Optional.empty(); + } + if (schema.relationships().isEmpty()) { + return Optional.empty(); + } + List joins = new ArrayList<>(); + for (RelationshipSchema relation : schema.relationships()) { + if (relation.toTable().equals(occurrence.name())) { + joins.add(new DatasetViewJoin(relation.relationName(), relation.fromTable(), + DatasetViewCardinalityPolicy.FIRST_ROW)); + } + } + if (joins.isEmpty()) { + return Optional.empty(); + } + List mappings = defaultOccurrenceMappings(schema, occurrence.name(), joins); + return Optional.of(new DatasetView(occurrence.name(), schema.schemaFingerprint(), joins, mappings)); + } + + private static Optional eventCoreOccurrenceExtension(DatasetSchema schema) { + TableSchema event = findByRowType(schema, "EVENT"); + TableSchema occurrence = findByRowType(schema, "OCCURRENCE"); + if (event == null || occurrence == null) { + return Optional.empty(); + } + boolean hasEventToOccurrence = schema.relationships().stream() + .anyMatch(relation -> relation.toTable().equals(event.name()) + && relation.fromTable().equals(occurrence.name())); + if (!hasEventToOccurrence) { + return Optional.empty(); + } + List joins = List.of(new DatasetViewJoin( + occurrence.name(), + occurrence.name(), + DatasetViewCardinalityPolicy.FIRST_ROW)); + List mappings = defaultOccurrenceMappings(schema, occurrence.name(), joins); + return Optional.of(new DatasetView(event.name(), schema.schemaFingerprint(), joins, mappings)); + } + + /** + * Builds default occurrence-oriented mappings, preferring a joined source table when the + * occurrence table does not carry a mapped term. + * + * @param schema discovered dataset schema + * @param occurrenceTable name of the occurrence table + * @param joins joins included in the selected built-in view + * @return direct source mappings for common occurrence terms + */ + private static List defaultOccurrenceMappings( + DatasetSchema schema, + String occurrenceTable, + List joins) { + Set allowedSourceTables = new LinkedHashSet<>(); + allowedSourceTables.add(occurrenceTable); + joins.forEach(join -> allowedSourceTables.add(join.sourceTable())); + return List.of( + new DatasetViewMapping("occurrenceID", + resolveSourceTable(schema, allowedSourceTables, occurrenceTable, "occurrenceID"), + "occurrenceID"), + new DatasetViewMapping("scientificName", + resolveSourceTable(schema, allowedSourceTables, occurrenceTable, "scientificName"), + "scientificName"), + new DatasetViewMapping("eventDate", + resolveSourceTable(schema, allowedSourceTables, occurrenceTable, "eventDate"), + "eventDate"), + new DatasetViewMapping("decimalLatitude", + resolveSourceTable(schema, allowedSourceTables, occurrenceTable, "decimalLatitude"), + "decimalLatitude"), + new DatasetViewMapping("decimalLongitude", + resolveSourceTable(schema, allowedSourceTables, occurrenceTable, "decimalLongitude"), + "decimalLongitude")); + } + + /** + * Resolves which allowed table should source a mapped term. + * + * @param schema discovered dataset schema + * @param allowedSourceTables tables permitted by the built-in view + * @param fallbackTable table name used when no candidate table carries the mapped column + * @param columnName column/term being mapped + * @return table name to use as mapping source + */ + private static String resolveSourceTable( + DatasetSchema schema, + Set allowedSourceTables, + String fallbackTable, + String columnName) { + return schema.tables().stream() + .filter(table -> allowedSourceTables.contains(table.name())) + .filter(table -> table.columns().contains(columnName)) + .map(TableSchema::name) + .findFirst() + .orElse(fallbackTable); + } + + private static TableSchema findByRowType(DatasetSchema schema, String rowType) { + return schema.tables().stream() + .filter(table -> table.rowType().equalsIgnoreCase(rowType)) + .findFirst() + .orElse(null); + } +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageArchiveSupport.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageArchiveSupport.java new file mode 100644 index 0000000..5560162 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageArchiveSupport.java @@ -0,0 +1,160 @@ +/** DataPackageArchiveSupport.java + * + * Utilities for detecting and opening Data Package manifests stored in zip archives. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.ingest; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.FileSystemAlreadyExistsException; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +/** + * Utilities for Data Package manifests inside zip inputs. + */ +final class DataPackageArchiveSupport { + + /** Utility class; not instantiable. */ + private DataPackageArchiveSupport() { + } + + /** + * Reports whether a zip input contains a {@code datapackage.json} manifest. + * + * @param inputPath candidate dataset path + * @return {@code true} when the path is a zip containing a data package manifest + */ + static boolean isDataPackageArchive(Path inputPath) { + String fileName = inputPath.getFileName().toString().toLowerCase(); + if (!fileName.endsWith(".zip")) { + return false; + } + try (ZipFile zipFile = new ZipFile(inputPath.toFile())) { + return findManifestEntryName(zipFile).isPresent(); + } catch (IOException e) { + return false; + } + } + + /** + * Resolves the manifest path for either an unpacked data package or a zip-contained one. + * + *

For zip inputs this keeps the zip filesystem open while {@code reader} executes, so the + * caller may also read package data files referenced from the manifest. + * + * @param inputPath path to either {@code datapackage.json} or a zip containing it + * @param reader callback that consumes the resolved manifest path + * @param result type produced by the callback + * @return callback result + * @throws IOException if the manifest cannot be resolved or read + */ + static T withManifestPath(Path inputPath, ManifestPathReader reader) throws IOException { + if (!inputPath.getFileName().toString().toLowerCase().endsWith(".zip")) { + return reader.read(inputPath); + } + URI archiveUri = URI.create("jar:" + inputPath.toUri()); + FileSystem zipFs; + boolean shouldClose = false; + try { + zipFs = FileSystems.newFileSystem(archiveUri, Map.of()); + shouldClose = true; + } catch (FileSystemAlreadyExistsException alreadyOpen) { + zipFs = FileSystems.getFileSystem(archiveUri); + } + try { + Path manifestPath = findManifestPath(zipFs) + .orElseThrow(() -> new IOException("Zip input contains no datapackage.json manifest: " + inputPath)); + return reader.read(manifestPath); + } finally { + if (shouldClose) { + zipFs.close(); + } + } + } + + /** + * Reads one value from a resolved manifest path. + * + * @param callback result type + */ + @FunctionalInterface + interface ManifestPathReader { + /** + * Reads one value from the provided manifest path. + * + * @param manifestPath resolved path to the data package manifest + * @return callback result + * @throws IOException if reading fails + */ + T read(Path manifestPath) throws IOException; + } + + /** + * Finds a manifest entry path within a zip archive. + * + * @param zipFile zip file to inspect + * @return matching manifest entry name when present + */ + private static Optional findManifestEntryName(ZipFile zipFile) { + return zipFile.stream() + .filter(entry -> !entry.isDirectory()) + .map(ZipEntry::getName) + .filter(DataPackageArchiveSupport::isDataPackageManifestName) + .min(Comparator.comparingInt(String::length).thenComparing(String::compareTo)); + } + + /** + * Finds a manifest path within an opened zip filesystem. + * + * @param zipFs opened zip filesystem + * @return manifest path when present + * @throws IOException if directory traversal fails + */ + private static Optional findManifestPath(FileSystem zipFs) throws IOException { + try (Stream paths = Files.walk(zipFs.getPath("/"))) { + return paths + .filter(Files::isRegularFile) + .filter(path -> isDataPackageManifestName(path.toString().replace('\\', '/'))) + .min(Comparator + .comparingInt((Path path) -> path.toString().length()) + .thenComparing(path -> path.toString())); + } + } + + /** + * Reports whether a zip entry/path names {@code datapackage.json}. + * + * @param entryName candidate path or entry name + * @return {@code true} when the name resolves to a data package manifest + */ + private static boolean isDataPackageManifestName(String entryName) { + String normalized = entryName.toLowerCase(); + return normalized.equals("datapackage.json") + || normalized.endsWith("/datapackage.json") + || normalized.endsWith("\\datapackage.json"); + } +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageDialectParser.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageDialectParser.java index 1a843b3..868478e 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageDialectParser.java +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageDialectParser.java @@ -183,7 +183,8 @@ static DataPackageResourceMeta parseResource(ObjectMapper mapper, JsonNode resou dialect.path("skipInitialSpace").asBoolean(false), resolveHeaderLines(dialect), columnNames, - readPrimaryKey(resource.path("schema"))); + readPrimaryKey(resource.path("schema")), + readForeignKeys(resource.path("schema"))); LOG.debug("Parsed data package resource '{}': paths={}, encoding={}, delimiter={}, quoteChar={}," + " escapeChar={}, headerLines={}, schemaColumns={}", meta.label(), meta.paths(), meta.encoding(), describe(meta.delimiter()), describe(meta.quoteChar()), @@ -329,6 +330,50 @@ private static List readSchemaFieldNames(JsonNode schema) { return names; } + /** + * Reads single-column foreign-key declarations from a resource schema. + * + * @param schema the resource's {@code schema} property + * @return declared single-column foreign keys + */ + private static List readForeignKeys(JsonNode schema) { + List keys = new ArrayList<>(); + JsonNode foreignKeys = schema.path("foreignKeys"); + if (!foreignKeys.isArray()) { + return keys; + } + for (JsonNode key : foreignKeys) { + String field = firstText(key.path("fields")); + JsonNode reference = key.path("reference"); + String referenceField = firstText(reference.path("fields")); + if (field == null || referenceField == null) { + continue; + } + String referenceResource = text(reference, "resource"); + keys.add(new DataPackageForeignKey( + field.trim(), + referenceResource == null ? "" : referenceResource.trim(), + referenceField.trim())); + } + return keys; + } + + /** + * Reads a string value from either a text node or a single-element text array. + * + * @param node node to inspect + * @return extracted text, or {@code null} + */ + private static String firstText(JsonNode node) { + if (node.isTextual()) { + return node.asText(null); + } + if (node.isArray() && node.size() == 1 && node.get(0).isTextual()) { + return node.get(0).asText(null); + } + return null; + } + /** * Determines how many leading header lines the dialect declares. * diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageForeignKey.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageForeignKey.java new file mode 100644 index 0000000..7154db5 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageForeignKey.java @@ -0,0 +1,30 @@ +/** DataPackageForeignKey.java + * + * One foreign-key declaration from a Data Package table schema. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.ingest; + +/** + * Single-column foreign-key metadata. + * + * @param field local child-table column + * @param referenceResource referenced resource name (blank means same table) + * @param referenceField referenced parent-table column + */ +public record DataPackageForeignKey(String field, String referenceResource, String referenceField) { +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageIngestor.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageIngestor.java index cab0556..a2e7d65 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageIngestor.java +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageIngestor.java @@ -69,7 +69,8 @@ public class DataPackageIngestor { /** * Ingests a Darwin Core Data Package into canonical records. * - * @param dataPackagePath path to the {@code datapackage.json} manifest file + * @param dataPackagePath path to the {@code datapackage.json} manifest file, or to a zip that + * contains one * @return the dataset parsed from the manifest's first resource * @throws AppException if the manifest has no resources, or the resource cannot be read */ @@ -81,7 +82,8 @@ public RecordDataset ingest(Path dataPackagePath) { * Ingests a Darwin Core Data Package into canonical records, optionally naming which of the * package's resources to read. * - * @param dataPackagePath path to the {@code datapackage.json} manifest file + * @param dataPackagePath path to the {@code datapackage.json} manifest file, or to a zip that + * contains one * @param requestedTable the name, file name or row type of the resource to read; blank to * select one automatically * @return the dataset parsed from the selected resource @@ -90,26 +92,41 @@ public RecordDataset ingest(Path dataPackagePath) { */ public RecordDataset ingest(Path dataPackagePath, String requestedTable) { try { - JsonNode root = mapper.readTree(Files.newBufferedReader(dataPackagePath)); - JsonNode resources = root.path("resources"); - if (!resources.isArray() || resources.isEmpty()) { - throw new AppException("Data package does not include resources"); - } - Path packageDir = dataPackagePath.toAbsolutePath().getParent(); - List> tables = - DataPackageDialectParser.parseResources(mapper, root, packageDir); - if (tables.isEmpty()) { - throw new AppException("Data package declares no resource with a readable data file path" - + " (inline data and remote resource URLs are not supported): " + dataPackagePath); - } - CoreTableCandidate selected = - CoreTableSelector.select(tables, requestedTable).selected(); - return ingestResource(dataPackagePath, selected); + return DataPackageArchiveSupport.withManifestPath(dataPackagePath, + manifestPath -> ingestManifest(manifestPath, dataPackagePath, requestedTable)); } catch (IOException e) { throw new AppException("Failed to ingest Darwin Core Data Package from " + dataPackagePath, e); } } + /** + * Ingests a data package manifest already resolved to a readable filesystem path. + * + * @param manifestPath resolved path to {@code datapackage.json} + * @param sourcePath original user-supplied dataset path, used in diagnostics + * @param requestedTable the name, file name or row type of the resource to read; blank to + * select one automatically + * @return the dataset parsed from the selected resource + * @throws IOException if the manifest cannot be read + */ + private RecordDataset ingestManifest(Path manifestPath, Path sourcePath, String requestedTable) throws IOException { + JsonNode root = mapper.readTree(Files.newBufferedReader(manifestPath)); + JsonNode resources = root.path("resources"); + if (!resources.isArray() || resources.isEmpty()) { + throw new AppException("Data package does not include resources"); + } + Path packageDir = manifestPath.toAbsolutePath().getParent(); + List> tables = + DataPackageDialectParser.parseResources(mapper, root, packageDir); + if (tables.isEmpty()) { + throw new AppException("Data package declares no resource with a readable data file path" + + " (inline data and remote resource URLs are not supported): " + sourcePath); + } + CoreTableCandidate selected = + CoreTableSelector.select(tables, requestedTable).selected(); + return ingestResource(sourcePath, selected); + } + /** * Parses every data file of a resource into canonical records. * diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageResourceMeta.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageResourceMeta.java index 734265b..2577d69 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageResourceMeta.java +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/DataPackageResourceMeta.java @@ -48,6 +48,7 @@ * @param columnNames column names from the resource's table schema, empty when it declares none * @param idColumn the single-column primary key the resource's table schema declares, or * {@code ""} when it declares none or declares a composite one + * @param foreignKeys single-column foreign keys the table schema declares */ public record DataPackageResourceMeta( String name, @@ -62,7 +63,8 @@ public record DataPackageResourceMeta( boolean skipInitialSpace, int headerLines, List columnNames, - String idColumn) { + String idColumn, + List foreignKeys) { /** * Canonical constructor; copies the collection components defensively. @@ -73,6 +75,7 @@ public record DataPackageResourceMeta( paths = List.copyOf(paths); columnNames = List.copyOf(columnNames); idColumn = idColumn == null ? "" : idColumn; + foreignKeys = List.copyOf(foreignKeys); } /** diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/DatasetSchemaInspector.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/DatasetSchemaInspector.java new file mode 100644 index 0000000..cfe3ea9 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/DatasetSchemaInspector.java @@ -0,0 +1,166 @@ +/** DatasetSchemaInspector.java + * + * Lightweight schema inspection for dataset-view setup. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.ingest; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import org.filteredpush.bdq_workbench.app.AppException; + +/** + * Inspects dataset table metadata without reading table rows. + */ +public class DatasetSchemaInspector { + private final ObjectMapper mapper = new ObjectMapper(); + + /** + * Inspects available dataset tables for UI decisions. + * + * @param inputPath dataset input path + * @return metadata-only table summary + */ + public DatasetSchemaOverview inspect(Path inputPath) { + String fileName = inputPath.getFileName().toString().toLowerCase(); + if (fileName.endsWith(".zip")) { + if (DataPackageArchiveSupport.isDataPackageArchive(inputPath)) { + return inspectDataPackage(inputPath); + } + return inspectDwcArchive(inputPath); + } + if (fileName.endsWith(".json") || fileName.endsWith("datapackage")) { + return inspectDataPackage(inputPath); + } + throw new AppException("Unsupported dataset input: " + inputPath); + } + + private DatasetSchemaOverview inspectDwcArchive(Path inputPath) { + try (ZipFile zipFile = new ZipFile(inputPath.toFile())) { + List> tables = DwcArchiveMetaParser.parseTables(zipFile); + if (!tables.isEmpty()) { + return new DatasetSchemaOverview(tables.stream() + .map(table -> new DatasetTableSummary( + table.label(), + table.rowType(), + table.rowTypeEvidence(), + table.declaredCore())) + .toList()); + } + return new DatasetSchemaOverview(List.of(new DatasetTableSummary( + resolveConventionalCoreEntryName(zipFile), + DatasetRowType.OCCURRENCE, + "conventional default", + true))); + } catch (IOException e) { + throw new AppException("Failed to inspect DwC-A " + inputPath, e); + } + } + + private DatasetSchemaOverview inspectDataPackage(Path inputPath) { + try { + return DataPackageArchiveSupport.withManifestPath(inputPath, manifestPath -> { + JsonNode root = mapper.readTree(Files.newBufferedReader(manifestPath)); + Path packageDir = manifestPath.toAbsolutePath().getParent(); + List> tables = + DataPackageDialectParser.parseResources(mapper, root, packageDir); + return new DatasetSchemaOverview(tables.stream() + .map(table -> new DatasetTableSummary( + table.label(), + table.rowType(), + table.rowTypeEvidence(), + table.declaredCore())) + .toList()); + }); + } catch (IOException e) { + throw new AppException("Failed to inspect data package " + inputPath, e); + } + } + + private String resolveConventionalCoreEntryName(ZipFile zipFile) { + ZipEntry occurrence = zipFile.getEntry("occurrence.txt"); + if (occurrence != null) { + return occurrence.getName(); + } + return zipFile.stream() + .filter(entry -> !entry.isDirectory() && entry.getName().endsWith(".txt")) + .map(ZipEntry::getName) + .findFirst() + .orElse("occurrence.txt"); + } + + /** + * Metadata-only dataset summary. + * + * @param tables offered tables + */ + public record DatasetSchemaOverview(List tables) { + + /** + * Canonical constructor; copies list defensively. + */ + public DatasetSchemaOverview { + tables = List.copyOf(tables); + } + + /** + * Renders a short table list for user-facing messages. + * + * @return one-line table summary + */ + public String describeTables() { + if (tables.isEmpty()) { + return "Dataset offers no readable tables"; + } + if (tables.size() == 1) { + return "Dataset offers one table, " + tables.get(0).describe(); + } + return "Dataset offers " + tables.size() + " tables"; + } + } + + /** + * Summary of one offered table. + * + * @param label table label + * @param rowType inferred row type + * @param rowTypeEvidence evidence for inferred row type + * @param declaredCore whether this table is the declared core + */ + public record DatasetTableSummary( + String label, + DatasetRowType rowType, + String rowTypeEvidence, + boolean declaredCore) { + + /** + * Renders one table as a short human-readable descriptor. + * + * @return rendered table descriptor + */ + public String describe() { + String declared = declaredCore ? ", declared core" : ""; + return label + " [rowType=" + rowType.name() + " (" + rowTypeEvidence + ")" + declared + "]"; + } + } +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/DatasetViewIO.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/DatasetViewIO.java new file mode 100644 index 0000000..68bef92 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/DatasetViewIO.java @@ -0,0 +1,75 @@ +/** DatasetViewIO.java + * + * JSON load/save helpers for standalone DatasetView files. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.ingest; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.nio.file.Path; +import org.filteredpush.bdq_workbench.app.AppException; +import org.filteredpush.bdq_workbench.model.DatasetSchema; +import org.filteredpush.bdq_workbench.model.DatasetView; + +/** + * Loads and saves standalone dataset view JSON files. + */ +public class DatasetViewIO { + private final ObjectMapper mapper = new ObjectMapper(); + + /** + * Reads a dataset view JSON file. + * + * @param path view file path + * @return parsed dataset view + */ + public DatasetView load(Path path) { + try { + return mapper.readValue(path.toFile(), DatasetView.class); + } catch (IOException e) { + throw new AppException("Unable to read dataset view file " + path, e); + } + } + + /** + * Writes a dataset view JSON file. + * + * @param path view file path + * @param view view to persist + */ + public void save(Path path, DatasetView view) { + try { + mapper.writerWithDefaultPrettyPrinter().writeValue(path.toFile(), view); + } catch (IOException e) { + throw new AppException("Unable to save dataset view file " + path, e); + } + } + + /** + * Verifies a view is compatible with the current schema fingerprint. + * + * @param view loaded view + * @param schema current schema + */ + public void validateCompatibility(DatasetView view, DatasetSchema schema) { + if (!view.schemaFingerprint().equals(schema.schemaFingerprint())) { + throw new AppException("Dataset view fingerprint does not match dataset schema fingerprint: view=" + + view.schemaFingerprint() + ", dataset=" + schema.schemaFingerprint()); + } + } +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/DefaultIngestService.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/DefaultIngestService.java index d05036e..3300261 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/ingest/DefaultIngestService.java +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/DefaultIngestService.java @@ -20,26 +20,38 @@ package org.filteredpush.bdq_workbench.ingest; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; import org.filteredpush.bdq_workbench.app.AppException; +import org.filteredpush.bdq_workbench.model.DatasetView; import org.filteredpush.bdq_workbench.model.RecordDataset; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Dispatches ingestion based on source format. * - *

Inspects the input path's file name and delegates to {@link DwcArchiveIngestor} for - * {@code .zip} Darwin Core Archives or {@link DataPackageIngestor} for {@code .json}/ - * {@code datapackage} Darwin Core Data Packages. + *

Inspects the input path and delegates to {@link DwcArchiveIngestor} for Darwin Core + * Archives, and to {@link DataPackageIngestor} for data package manifests + * ({@code .json}/{@code datapackage}) and zipped data packages containing + * {@code datapackage.json}. */ public class DefaultIngestService implements IngestService { + private static final Logger LOG = LoggerFactory.getLogger(DefaultIngestService.class); private final DwcArchiveIngestor dwcArchiveIngestor; private final DataPackageIngestor dataPackageIngestor; + private final RelationalDatasetIngestor relationalDatasetIngestor; + private final DatasetViewIO datasetViewIO; + private final ViewFlattener viewFlattener; /** * Creates a service with default {@link DwcArchiveIngestor} and {@link DataPackageIngestor} * instances. */ public DefaultIngestService() { - this(new DwcArchiveIngestor(), new DataPackageIngestor()); + this(new DwcArchiveIngestor(), new DataPackageIngestor(), + new RelationalDatasetIngestor(), new DatasetViewIO(), new ViewFlattener()); } /** @@ -49,8 +61,24 @@ public DefaultIngestService() { * @param dataPackageIngestor ingestor used for {@code .json}/{@code datapackage} inputs */ public DefaultIngestService(DwcArchiveIngestor dwcArchiveIngestor, DataPackageIngestor dataPackageIngestor) { + this(dwcArchiveIngestor, dataPackageIngestor, + new RelationalDatasetIngestor(), new DatasetViewIO(), new ViewFlattener()); + } + + /** + * Creates a service wired to explicit flat and relational ingestion components. + */ + public DefaultIngestService( + DwcArchiveIngestor dwcArchiveIngestor, + DataPackageIngestor dataPackageIngestor, + RelationalDatasetIngestor relationalDatasetIngestor, + DatasetViewIO datasetViewIO, + ViewFlattener viewFlattener) { this.dwcArchiveIngestor = dwcArchiveIngestor; this.dataPackageIngestor = dataPackageIngestor; + this.relationalDatasetIngestor = relationalDatasetIngestor; + this.datasetViewIO = datasetViewIO; + this.viewFlattener = viewFlattener; } /** @@ -80,8 +108,23 @@ public RecordDataset ingest(Path inputPath) { */ @Override public RecordDataset ingest(Path inputPath, String requestedTable) { + return ingest(inputPath, requestedTable, ""); + } + + @Override + public RecordDataset ingest(Path inputPath, String requestedTable, String datasetView) { + if (datasetView != null && !datasetView.isBlank()) { + return ingestThroughView(inputPath, requestedTable, datasetView); + } + return ingestWithOptionalBuiltInView(inputPath, requestedTable); + } + + private RecordDataset ingestFlat(Path inputPath, String requestedTable) { String fileName = inputPath.getFileName().toString().toLowerCase(); if (fileName.endsWith(".zip")) { + if (DataPackageArchiveSupport.isDataPackageArchive(inputPath)) { + return dataPackageIngestor.ingest(inputPath, requestedTable); + } return dwcArchiveIngestor.ingest(inputPath, requestedTable); } if (fileName.endsWith(".json") || fileName.endsWith("datapackage")) { @@ -89,4 +132,38 @@ public RecordDataset ingest(Path inputPath, String requestedTable) { } throw new AppException("Unsupported dataset input: " + inputPath); } + + private RecordDataset ingestThroughView(Path inputPath, String requestedTable, String datasetViewPath) { + RelationalIngestResult relational = relationalDatasetIngestor.ingest(inputPath, requestedTable); + DatasetView view = datasetViewIO.load(Path.of(datasetViewPath)); + datasetViewIO.validateCompatibility(view, relational.schema()); + ViewFlattenResult flattened = viewFlattener.flatten(relational, view); + logDiagnostics(relational.diagnostics(), flattened.diagnostics()); + return flattened.dataset(); + } + + private RecordDataset ingestWithOptionalBuiltInView(Path inputPath, String requestedTable) { + RelationalIngestResult relational = relationalDatasetIngestor.ingest(inputPath, requestedTable); + if (relational.graphs().isEmpty()) { + return ingestFlat(inputPath, requestedTable); + } + List diagnostics = new ArrayList<>(); + Optional builtIn = BuiltInDatasetViews.select(relational.schema(), diagnostics); + if (builtIn.isEmpty()) { + logDiagnostics(relational.diagnostics(), diagnostics); + List rows = relational.graphs().stream() + .map(org.filteredpush.bdq_workbench.model.RecordGraph::core) + .toList(); + return new RecordDataset(rows); + } + ViewFlattenResult flattened = viewFlattener.flatten(relational, builtIn.get()); + logDiagnostics(relational.diagnostics(), diagnostics, flattened.diagnostics()); + return flattened.dataset(); + } + + private void logDiagnostics(List... groups) { + for (List group : groups) { + group.forEach(message -> LOG.warn("Dataset view diagnostic: {}", message)); + } + } } diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/DwcArchiveCoreMeta.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/DwcArchiveCoreMeta.java index c9c2e03..7f7b169 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/ingest/DwcArchiveCoreMeta.java +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/DwcArchiveCoreMeta.java @@ -47,6 +47,8 @@ * @param columnNames column names by zero-based column index, with gaps filled by placeholders * @param idColumn the name of the column the archive declares as this table's record * identifier, or {@code ""} when it declares none + * @param coreIdColumn the name of the extension column declaring the related core record + * identifier, or {@code ""} when this table is a core or declares none * @param constantTerms term values declared in {@code meta.xml} as defaults for columns that are * absent from the data files, applied to every record */ @@ -59,6 +61,7 @@ public record DwcArchiveCoreMeta( int ignoreHeaderLines, List columnNames, String idColumn, + String coreIdColumn, Map constantTerms) { /** @@ -69,6 +72,7 @@ public record DwcArchiveCoreMeta( locations = List.copyOf(locations); columnNames = List.copyOf(columnNames); idColumn = idColumn == null ? "" : idColumn; + coreIdColumn = coreIdColumn == null ? "" : coreIdColumn; constantTerms = Map.copyOf(new LinkedHashMap<>(constantTerms)); } } diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/DwcArchiveMetaParser.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/DwcArchiveMetaParser.java index a3b58a5..e35c24a 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/ingest/DwcArchiveMetaParser.java +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/DwcArchiveMetaParser.java @@ -229,6 +229,10 @@ private static Optional buildCoreMeta(Element core) { if (idIndex >= 0) { namesByIndex.putIfAbsent(idIndex, "id"); } + int coreIdIndex = parseInt(attribute(firstChildElement(core, "coreid"), "index"), -1); + if (coreIdIndex >= 0) { + namesByIndex.putIfAbsent(coreIdIndex, "coreid"); + } DwcArchiveCoreMeta meta = new DwcArchiveCoreMeta( attributeOrDefault(core, "rowType", ""), locations, @@ -238,6 +242,7 @@ private static Optional buildCoreMeta(Element core) { Math.max(0, parseInt(attribute(core, "ignoreHeaderLines"), 0)), toColumnNames(namesByIndex), idIndex >= 0 ? namesByIndex.getOrDefault(idIndex, "") : "", + coreIdIndex >= 0 ? namesByIndex.getOrDefault(coreIdIndex, "") : "", constantTerms); LOG.debug("Parsed meta.xml table: rowType={}, locations={}, encoding={}, delimiter={}, enclosedBy={}, " + "ignoreHeaderLines={}, columns={}, constantTerms={}", diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/IngestService.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/IngestService.java index 900e1c3..019c587 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/ingest/IngestService.java +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/IngestService.java @@ -56,4 +56,16 @@ public interface IngestService { default RecordDataset ingest(Path inputPath, String requestedTable) { return ingest(inputPath); } + + /** + * Ingests the dataset at the given input path, optionally applying a dataset-view definition. + * + * @param inputPath path to dataset input file + * @param requestedTable optional requested table name + * @param datasetView optional dataset view JSON path + * @return ingested (possibly view-flattened) dataset + */ + default RecordDataset ingest(Path inputPath, String requestedTable, String datasetView) { + return ingest(inputPath, requestedTable); + } } diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/RelationalDatasetIngestor.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/RelationalDatasetIngestor.java new file mode 100644 index 0000000..8b91326 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/RelationalDatasetIngestor.java @@ -0,0 +1,225 @@ +/** RelationalDatasetIngestor.java + * + * Builds RecordGraph structures from DwC-A/Data Package inputs. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.ingest; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipFile; +import org.filteredpush.bdq_workbench.app.AppException; +import org.filteredpush.bdq_workbench.model.CanonicalRecord; +import org.filteredpush.bdq_workbench.model.DatasetSchema; +import org.filteredpush.bdq_workbench.model.RecordGraph; +import org.filteredpush.bdq_workbench.model.RelationshipSchema; +import org.filteredpush.bdq_workbench.model.SourceCell; +import org.filteredpush.bdq_workbench.model.TableSchema; + +/** + * Relational ingestion for non-flat datasets. + */ +public class RelationalDatasetIngestor { + private final DwcArchiveIngestor dwcArchiveIngestor = new DwcArchiveIngestor(); + private final DataPackageIngestor dataPackageIngestor = new DataPackageIngestor(); + private final ObjectMapper mapper = new ObjectMapper(); + + /** + * Ingests a dataset and emits relational graphs plus schema metadata. + * + * @param inputPath dataset path + * @param requestedTable optional preferred core/grain table + * @return relational ingest output + */ + public RelationalIngestResult ingest(Path inputPath, String requestedTable) { + String fileName = inputPath.getFileName().toString().toLowerCase(); + if (fileName.endsWith(".zip")) { + if (DataPackageArchiveSupport.isDataPackageArchive(inputPath)) { + return ingestDataPackage(inputPath, requestedTable); + } + return ingestDwcArchive(inputPath, requestedTable); + } + if (fileName.endsWith(".json") || fileName.endsWith("datapackage")) { + return ingestDataPackage(inputPath, requestedTable); + } + throw new AppException("Unsupported dataset input: " + inputPath); + } + + private RelationalIngestResult ingestDwcArchive(Path inputPath, String requestedTable) { + try (ZipFile zipFile = new ZipFile(inputPath.toFile())) { + List> tables = DwcArchiveMetaParser.parseTables(zipFile); + if (tables.isEmpty()) { + return new RelationalIngestResult(List.of(), new DatasetSchema(List.of(), List.of(), ""), List.of()); + } + CoreTableCandidate selected = CoreTableSelector.select(tables, requestedTable).selected(); + Map> rowsByTable = new LinkedHashMap<>(); + for (CoreTableCandidate table : tables) { + List rows = dwcArchiveIngestor.ingest(inputPath, table.label()).records().stream() + .map(row -> withTableProvenance(row, table.label())) + .toList(); + rowsByTable.put(table.label(), rows); + } + List relationships = tables.stream() + .filter(table -> !table.descriptor().coreIdColumn().isBlank()) + .map(table -> new RelationshipSchema( + table.label(), + table.descriptor().coreIdColumn(), + selected.label(), + selected.descriptor().idColumn().isBlank() + ? selected.rowType().identifierTerm() + : selected.descriptor().idColumn(), + table.label())) + .toList(); + return assembleResult(selected.label(), rowsByTable, tables.stream() + .map(table -> new TableSchema( + table.label(), + table.label(), + table.rowType().name(), + table.descriptor().idColumn().isBlank() + ? table.rowType().identifierTerm() + : table.descriptor().idColumn(), + table.descriptor().columnNames())) + .toList(), relationships); + } catch (IOException e) { + throw new AppException("Failed relational ingest for DwC-A " + inputPath, e); + } + } + + private RelationalIngestResult ingestDataPackage(Path inputPath, String requestedTable) { + try { + return DataPackageArchiveSupport.withManifestPath(inputPath, + manifestPath -> relationalFromDataPackageManifest(manifestPath, inputPath, requestedTable)); + } catch (IOException e) { + throw new AppException("Failed relational ingest for Data Package " + inputPath, e); + } + } + + /** + * Builds a relational ingest result from a resolved Data Package manifest. + * + * @param manifestPath resolved path to {@code datapackage.json} + * @param sourcePath original user-supplied dataset path + * @param requestedTable optional preferred core/grain table + * @return relational ingest output + * @throws IOException if the manifest cannot be read + */ + private RelationalIngestResult relationalFromDataPackageManifest(Path manifestPath, Path sourcePath, + String requestedTable) throws IOException { + JsonNode root = mapper.readTree(Files.newBufferedReader(manifestPath)); + Path packageDir = manifestPath.toAbsolutePath().getParent(); + List> tables = + DataPackageDialectParser.parseResources(mapper, root, packageDir); + if (tables.isEmpty()) { + return new RelationalIngestResult(List.of(), new DatasetSchema(List.of(), List.of(), ""), List.of()); + } + CoreTableCandidate selected = CoreTableSelector.select(tables, requestedTable).selected(); + Map> rowsByTable = new LinkedHashMap<>(); + for (CoreTableCandidate table : tables) { + List rows = dataPackageIngestor.ingest(sourcePath, table.label()).records().stream() + .map(row -> withTableProvenance(row, table.label())) + .toList(); + rowsByTable.put(table.label(), rows); + } + List relationships = new ArrayList<>(); + for (CoreTableCandidate table : tables) { + for (DataPackageForeignKey key : table.descriptor().foreignKeys()) { + String referencedTableLabel = resolveReferencedTableLabel(tables, table, key); + if (referencedTableLabel == null || !referencedTableLabel.equals(selected.label())) { + continue; + } + relationships.add(new RelationshipSchema( + table.label(), + key.field(), + referencedTableLabel, + key.referenceField(), + table.label())); + } + } + return assembleResult(selected.label(), rowsByTable, tables.stream() + .map(table -> new TableSchema( + table.label(), + table.label(), + table.rowType().name(), + table.descriptor().idColumn().isBlank() + ? table.rowType().identifierTerm() + : table.descriptor().idColumn(), + table.descriptor().columnNames())) + .toList(), relationships); + } + + private String resolveReferencedTableLabel(List> tables, + CoreTableCandidate source, + DataPackageForeignKey key) { + if (key.referenceResource().isBlank()) { + return source.label(); + } + return tables.stream() + .filter(candidate -> candidate.descriptor().name().equalsIgnoreCase(key.referenceResource())) + .map(CoreTableCandidate::label) + .findFirst() + .orElse(null); + } + + private RelationalIngestResult assembleResult(String coreTable, Map> rowsByTable, + List tables, List relationships) { + List diagnostics = new ArrayList<>(); + Map>> relatedByTableByCoreId = new LinkedHashMap<>(); + for (RelationshipSchema relation : relationships) { + Map> byCore = new LinkedHashMap<>(); + for (CanonicalRecord related : rowsByTable.getOrDefault(relation.fromTable(), List.of())) { + String key = related.terms().getOrDefault(relation.fromColumn(), ""); + byCore.computeIfAbsent(key, ignored -> new ArrayList<>()).add(related); + } + relatedByTableByCoreId.put(relation.relationName(), byCore); + } + List graphs = new ArrayList<>(); + for (CanonicalRecord core : rowsByTable.getOrDefault(coreTable, List.of())) { + Map> relatedByRelation = new LinkedHashMap<>(); + for (RelationshipSchema relation : relationships) { + String coreValue = core.terms().getOrDefault(relation.toColumn(), ""); + if (coreValue.isBlank()) { + continue; + } + List related = relatedByTableByCoreId + .getOrDefault(relation.relationName(), Map.of()) + .getOrDefault(coreValue, List.of()); + if (related.isEmpty()) { + diagnostics.add("No related rows found for relation " + relation.relationName() + + " and core record " + core.id()); + } + relatedByRelation.put(relation.relationName(), related); + } + graphs.add(new RecordGraph(core, relatedByRelation)); + } + String fingerprint = SchemaFingerprint.of(tables, relationships); + return new RelationalIngestResult(graphs, new DatasetSchema(tables, relationships, fingerprint), diagnostics); + } + + private CanonicalRecord withTableProvenance(CanonicalRecord row, String tableName) { + Map> provenance = new LinkedHashMap<>(); + row.terms().forEach((term, value) -> provenance.put(term, List.of( + new SourceCell(tableName, tableName, row.id(), term, term)))); + return new CanonicalRecord(row.id(), row.terms(), provenance); + } +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/RelationalIngestResult.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/RelationalIngestResult.java new file mode 100644 index 0000000..1a63ee1 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/RelationalIngestResult.java @@ -0,0 +1,45 @@ +/** RelationalIngestResult.java + * + * Result of relational dataset ingest plus schema discovery. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.ingest; + +import java.util.List; +import org.filteredpush.bdq_workbench.model.DatasetSchema; +import org.filteredpush.bdq_workbench.model.RecordGraph; + +/** + * Relational ingest output. + * + * @param graphs core-record graphs in deterministic row order + * @param schema discovered schema metadata and fingerprint + * @param diagnostics non-fatal ingest diagnostics + */ +public record RelationalIngestResult( + List graphs, + DatasetSchema schema, + List diagnostics) { + + /** + * Canonical constructor; copies list components defensively. + */ + public RelationalIngestResult { + graphs = List.copyOf(graphs); + diagnostics = List.copyOf(diagnostics); + } +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/SchemaFingerprint.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/SchemaFingerprint.java new file mode 100644 index 0000000..0cb03b0 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/SchemaFingerprint.java @@ -0,0 +1,79 @@ +/** SchemaFingerprint.java + * + * Deterministic schema fingerprint helper. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.ingest; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Comparator; +import java.util.HexFormat; +import java.util.List; +import java.util.stream.Collectors; +import org.filteredpush.bdq_workbench.model.RelationshipSchema; +import org.filteredpush.bdq_workbench.model.TableSchema; + +/** + * Computes a stable schema-shape fingerprint. + */ +final class SchemaFingerprint { + + private SchemaFingerprint() { + } + + /** + * Builds a fingerprint from table names, row types and column-name sets. + * + * @param tables discovered tables + * @return fingerprint hash + */ + static String of(List tables, List relationships) { + String tablePart = tables.stream() + .sorted(Comparator.comparing(TableSchema::name)) + .map(table -> table.name().toLowerCase() + + "|" + + table.rowType().toLowerCase() + + "|" + + table.columns().stream().map(String::toLowerCase).sorted().collect(Collectors.joining(","))) + .collect(Collectors.joining("||")); + String relationshipPart = relationships.stream() + .sorted(Comparator.comparing(RelationshipSchema::relationName) + .thenComparing(RelationshipSchema::fromTable) + .thenComparing(RelationshipSchema::fromColumn) + .thenComparing(RelationshipSchema::toTable) + .thenComparing(RelationshipSchema::toColumn)) + .map(relationship -> relationship.relationName().toLowerCase() + + "|" + + relationship.fromTable().toLowerCase() + + "." + + relationship.fromColumn().toLowerCase() + + "->" + + relationship.toTable().toLowerCase() + + "." + + relationship.toColumn().toLowerCase()) + .collect(Collectors.joining("||")); + String canonical = tablePart + "##" + relationshipPart; + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + return HexFormat.of().formatHex(digest.digest(canonical.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 not available", e); + } + } +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/ViewFlattenResult.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/ViewFlattenResult.java new file mode 100644 index 0000000..c86d4bd --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/ViewFlattenResult.java @@ -0,0 +1,39 @@ +/** ViewFlattenResult.java + * + * Flattening output with diagnostics. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.ingest; + +import java.util.List; +import org.filteredpush.bdq_workbench.model.RecordDataset; + +/** + * Flattened dataset and diagnostics. + * + * @param dataset flattened dataset + * @param diagnostics non-fatal flattening diagnostics + */ +public record ViewFlattenResult(RecordDataset dataset, List diagnostics) { + + /** + * Canonical constructor; copies diagnostics defensively. + */ + public ViewFlattenResult { + diagnostics = List.copyOf(diagnostics); + } +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/ingest/ViewFlattener.java b/src/main/java/org/filteredpush/bdq_workbench/ingest/ViewFlattener.java new file mode 100644 index 0000000..495e37a --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/ingest/ViewFlattener.java @@ -0,0 +1,134 @@ +/** ViewFlattener.java + * + * Flattens relational record graphs with a reusable DatasetView. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.ingest; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.filteredpush.bdq_workbench.model.CanonicalRecord; +import org.filteredpush.bdq_workbench.model.DatasetView; +import org.filteredpush.bdq_workbench.model.DatasetViewCardinalityPolicy; +import org.filteredpush.bdq_workbench.model.DatasetViewJoin; +import org.filteredpush.bdq_workbench.model.DatasetViewMapping; +import org.filteredpush.bdq_workbench.model.RecordDataset; +import org.filteredpush.bdq_workbench.model.RecordGraph; +import org.filteredpush.bdq_workbench.model.SourceCell; + +/** + * Applies a dataset view to relational graphs and emits flat canonical records. + */ +public class ViewFlattener { + + /** + * Flattens relational graphs according to a reusable dataset view. + * + * @param relational relational ingest result + * @param view view definition to apply + * @return flattened dataset and non-fatal diagnostics + */ + public ViewFlattenResult flatten(RelationalIngestResult relational, DatasetView view) { + List diagnostics = new ArrayList<>(); + List flattened = new ArrayList<>(); + for (RecordGraph graph : relational.graphs()) { + Map terms = new LinkedHashMap<>(); + Map> provenance = new LinkedHashMap<>(); + for (DatasetViewMapping mapping : view.mappings()) { + ValueSelection selected = selectValue(graph, mapping, view.grainTable(), view.joins(), diagnostics); + terms.put(mapping.term(), selected.value()); + if (!selected.cells().isEmpty()) { + provenance.put(mapping.term(), selected.cells()); + } + } + flattened.add(new CanonicalRecord(graph.core().id(), terms, provenance)); + } + return new ViewFlattenResult(new RecordDataset(flattened), diagnostics); + } + + /** + * Resolves one mapping from either the core row or one related relation. + */ + private ValueSelection selectValue(RecordGraph graph, DatasetViewMapping mapping, String grainTable, + List joins, + List diagnostics) { + if (mapping.sourceTable().equalsIgnoreCase(grainTable)) { + String value = graph.core().terms().getOrDefault(mapping.sourceColumn(), ""); + return new ValueSelection(value, sourceCells(graph.core(), grainTable, mapping.sourceColumn(), mapping.term())); + } + DatasetViewJoin join = joins.stream() + .filter(candidate -> candidate.sourceTable().equalsIgnoreCase(mapping.sourceTable())) + .findFirst() + .orElse(null); + if (join == null) { + diagnostics.add("View mapping for term " + mapping.term() + " references source table " + + mapping.sourceTable() + " but the view has no join for it"); + return ValueSelection.empty(); + } + List related = graph.relatedByRelation().getOrDefault(join.relationName(), List.of()); + if (related.isEmpty()) { + return ValueSelection.empty(); + } + List selections = related.stream() + .map(row -> new ValueSelection( + row.terms().getOrDefault(mapping.sourceColumn(), ""), + sourceCells(row, join.sourceTable(), mapping.sourceColumn(), mapping.term()))) + .toList(); + if (related.size() > 1 && join.cardinalityPolicy() == DatasetViewCardinalityPolicy.REJECT) { + diagnostics.add("Cardinality conflict for relation " + join.relationName() + " on record " + + graph.core().id() + " while mapping term " + mapping.term()); + return ValueSelection.empty(); + } + if (join.cardinalityPolicy() == DatasetViewCardinalityPolicy.FIRST_ROW) { + return selections.get(0); + } + if (join.cardinalityPolicy() == DatasetViewCardinalityPolicy.AGGREGATE) { + List included = selections.stream() + .filter(selection -> !selection.value().isBlank()) + .toList(); + String aggregated = included.stream() + .map(ValueSelection::value) + .collect(Collectors.joining(" | ")); + List cells = included.stream().flatMap(selection -> selection.cells().stream()).toList(); + return new ValueSelection(aggregated, cells); + } + return selections.get(0); + } + + /** + * Builds source-cell provenance for one value read from one row. + */ + private List sourceCells(CanonicalRecord row, String sourceTable, String sourceColumn, String term) { + List existing = row.provenanceByTerm().get(sourceColumn); + if (existing != null && !existing.isEmpty()) { + return existing.stream() + .map(cell -> new SourceCell(cell.table(), cell.sourceLocation(), cell.rowRef(), sourceColumn, term)) + .toList(); + } + return List.of(new SourceCell(sourceTable, sourceTable, row.id(), sourceColumn, term)); + } + + private record ValueSelection(String value, List cells) { + + private static ValueSelection empty() { + return new ValueSelection("", List.of()); + } + } +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/model/CanonicalRecord.java b/src/main/java/org/filteredpush/bdq_workbench/model/CanonicalRecord.java index 889d18c..e375457 100644 --- a/src/main/java/org/filteredpush/bdq_workbench/model/CanonicalRecord.java +++ b/src/main/java/org/filteredpush/bdq_workbench/model/CanonicalRecord.java @@ -20,6 +20,8 @@ package org.filteredpush.bdq_workbench.model; import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; /** @@ -30,8 +32,9 @@ * can update term values in place as a run progresses. */ public final class CanonicalRecord { - private final String id; - private final Map terms; + private final String id; + private final Map terms; + private final Map> provenanceByTerm; /** * Creates a canonical record, copying the supplied terms into a new mutable map. @@ -39,35 +42,58 @@ public final class CanonicalRecord { * @param id the record's identifier * @param terms the record's initial Darwin Core term values, keyed by term name */ - public CanonicalRecord(String id, Map terms) { - this.id = id; - this.terms = new HashMap<>(terms); - } + public CanonicalRecord(String id, Map terms) { + this(id, terms, Map.of()); + } + + /** + * Creates a canonical record with value-level source provenance. + * + * @param id the record's identifier + * @param terms the record's Darwin Core term values, keyed by term name + * @param provenanceByTerm provenance cells for each term value + */ + public CanonicalRecord(String id, Map terms, Map> provenanceByTerm) { + this.id = id; + this.terms = new HashMap<>(terms); + Map> copy = new LinkedHashMap<>(); + provenanceByTerm.forEach((term, cells) -> copy.put(term, List.copyOf(cells))); + this.provenanceByTerm = Map.copyOf(copy); + } /** * Returns this record's identifier. * * @return the record ID */ - public String id() { - return id; - } + public String id() { + return id; + } /** * Returns this record's Darwin Core term values. * * @return the mutable map of term name to value backing this record */ - public Map terms() { - return terms; - } + public Map terms() { + return terms; + } + + /** + * Returns source provenance for this record's term values. + * + * @return immutable map of term name to one or more source cells + */ + public Map> provenanceByTerm() { + return provenanceByTerm; + } /** * Creates an independent copy of this record, with its own copy of the terms map. * * @return a new {@code CanonicalRecord} with the same ID and a copy of the current term values */ - public CanonicalRecord copy() { - return new CanonicalRecord(id, terms); - } + public CanonicalRecord copy() { + return new CanonicalRecord(id, terms, provenanceByTerm); + } } diff --git a/src/main/java/org/filteredpush/bdq_workbench/model/DatasetSchema.java b/src/main/java/org/filteredpush/bdq_workbench/model/DatasetSchema.java new file mode 100644 index 0000000..1657b44 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/model/DatasetSchema.java @@ -0,0 +1,43 @@ +/** DatasetSchema.java + * + * Relational table/relationship metadata discovered during ingest. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.model; + +import java.util.List; + +/** + * Discovered schema used by dataset views. + * + * @param tables discovered tables/resources + * @param relationships discovered table relationships + * @param schemaFingerprint deterministic schema-shape fingerprint + */ +public record DatasetSchema( + List tables, + List relationships, + String schemaFingerprint) { + + /** + * Canonical constructor; copies list components defensively. + */ + public DatasetSchema { + tables = List.copyOf(tables); + relationships = List.copyOf(relationships); + } +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/model/DatasetView.java b/src/main/java/org/filteredpush/bdq_workbench/model/DatasetView.java new file mode 100644 index 0000000..4733819 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/model/DatasetView.java @@ -0,0 +1,45 @@ +/** DatasetView.java + * + * Standalone JSON-serializable dataset view definition. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.model; + +import java.util.List; + +/** + * Reusable flattening view. + * + * @param grainTable table whose rows define flattened output records + * @param schemaFingerprint required schema fingerprint compatibility key + * @param joins ordered joins to include while flattening + * @param mappings direct term-to-column mappings + */ +public record DatasetView( + String grainTable, + String schemaFingerprint, + List joins, + List mappings) { + + /** + * Canonical constructor; copies list components defensively. + */ + public DatasetView { + joins = List.copyOf(joins); + mappings = List.copyOf(mappings); + } +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/model/DatasetViewCardinalityPolicy.java b/src/main/java/org/filteredpush/bdq_workbench/model/DatasetViewCardinalityPolicy.java new file mode 100644 index 0000000..2850094 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/model/DatasetViewCardinalityPolicy.java @@ -0,0 +1,34 @@ +/** DatasetViewCardinalityPolicy.java + * + * Cardinality handling choices when flattening related rows. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.model; + +/** + * Per-join policy for handling multiple related rows. + */ +public enum DatasetViewCardinalityPolicy { + /** Concatenate all matching row values in deterministic related-row order using {@code " | "}. */ + AGGREGATE, + + /** Use only the first matching row value in deterministic related-row order. */ + FIRST_ROW, + + /** Keep flattening but emit a diagnostic when multiple related rows are present. */ + REJECT +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/model/DatasetViewJoin.java b/src/main/java/org/filteredpush/bdq_workbench/model/DatasetViewJoin.java new file mode 100644 index 0000000..305fc15 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/model/DatasetViewJoin.java @@ -0,0 +1,33 @@ +/** DatasetViewJoin.java + * + * Join definition for dataset view flattening. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.model; + +/** + * One ordered relationship in a dataset view. + * + * @param relationName relation key from {@link RecordGraph#relatedByRelation()} + * @param sourceTable source table name for this join + * @param cardinalityPolicy multiple-row handling policy + */ +public record DatasetViewJoin( + String relationName, + String sourceTable, + DatasetViewCardinalityPolicy cardinalityPolicy) { +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/model/DatasetViewMapping.java b/src/main/java/org/filteredpush/bdq_workbench/model/DatasetViewMapping.java new file mode 100644 index 0000000..b873b0c --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/model/DatasetViewMapping.java @@ -0,0 +1,30 @@ +/** DatasetViewMapping.java + * + * Direct term-to-column mapping in a dataset view. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.model; + +/** + * One direct term mapping in a dataset view. + * + * @param term Darwin Core term in the flattened output + * @param sourceTable source table name + * @param sourceColumn source table column + */ +public record DatasetViewMapping(String term, String sourceTable, String sourceColumn) { +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/model/RecordGraph.java b/src/main/java/org/filteredpush/bdq_workbench/model/RecordGraph.java new file mode 100644 index 0000000..90e9587 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/model/RecordGraph.java @@ -0,0 +1,42 @@ +/** RecordGraph.java + * + * Non-flat record model with a core record and related rows. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.model; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * One relationally-connected record graph. + * + * @param core the core record for the graph + * @param relatedByRelation related subrecords keyed by relation/table name + */ +public record RecordGraph(CanonicalRecord core, Map> relatedByRelation) { + + /** + * Canonical constructor; copies relation lists defensively. + */ + public RecordGraph { + Map> copy = new LinkedHashMap<>(); + relatedByRelation.forEach((relation, records) -> copy.put(relation, List.copyOf(records))); + relatedByRelation = Map.copyOf(copy); + } +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/model/RelationshipSchema.java b/src/main/java/org/filteredpush/bdq_workbench/model/RelationshipSchema.java new file mode 100644 index 0000000..9462f54 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/model/RelationshipSchema.java @@ -0,0 +1,37 @@ +/** RelationshipSchema.java + * + * Relationship metadata between two tables. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.model; + +/** + * One directed table relationship. + * + * @param fromTable related/child table + * @param fromColumn related-table foreign-key column + * @param toTable referenced/parent table + * @param toColumn referenced-table identifier column + * @param relationName relation key used in {@link RecordGraph#relatedByRelation()} + */ +public record RelationshipSchema( + String fromTable, + String fromColumn, + String toTable, + String toColumn, + String relationName) { +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/model/SourceCell.java b/src/main/java/org/filteredpush/bdq_workbench/model/SourceCell.java new file mode 100644 index 0000000..c029dfd --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/model/SourceCell.java @@ -0,0 +1,37 @@ +/** SourceCell.java + * + * Provenance for one value in a flattened record. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.model; + +/** + * Source location of a flattened term value. + * + * @param table the source table/resource label + * @param sourceLocation the source file or manifest path for the table + * @param rowRef deterministic source row reference within the table + * @param column the source column name + * @param term the flattened Darwin Core term that took this value + */ +public record SourceCell( + String table, + String sourceLocation, + String rowRef, + String column, + String term) { +} diff --git a/src/main/java/org/filteredpush/bdq_workbench/model/TableSchema.java b/src/main/java/org/filteredpush/bdq_workbench/model/TableSchema.java new file mode 100644 index 0000000..883dd47 --- /dev/null +++ b/src/main/java/org/filteredpush/bdq_workbench/model/TableSchema.java @@ -0,0 +1,46 @@ +/** TableSchema.java + * + * Table discovery metadata. + * + * Copyright 2026 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.filteredpush.bdq_workbench.model; + +import java.util.List; + +/** + * One discovered table/resource shape. + * + * @param name stable table name used in view definitions + * @param label human-readable label + * @param rowType detected Darwin Core row type name + * @param identifierColumn row identifier column, if known + * @param columns declared column names + */ +public record TableSchema( + String name, + String label, + String rowType, + String identifierColumn, + List columns) { + + /** + * Canonical constructor; copies columns defensively. + */ + public TableSchema { + columns = List.copyOf(columns); + } +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 5923315..cceb18a 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -5,6 +5,7 @@ bdq.usecase.file= bdq.rdf.files= bdq.dataset=dataset.zip bdq.dataset.table= +bdq.dataset.view= bdq.usecase.id= bdq.discovery.packages=org.filteredpush bdq.threads=4 diff --git a/src/test/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchApplicationTest.java b/src/test/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchApplicationTest.java index b554802..3c1b66a 100644 --- a/src/test/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchApplicationTest.java +++ b/src/test/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchApplicationTest.java @@ -131,7 +131,7 @@ void helpDocumentsTheGuiFlag() { int exitCode = BdqWorkbenchApplication.run(new String[] {"--help"}, printStream(out), printStream(err)); assertThat(exitCode).isZero(); - assertThat(out.toString()).contains("--gui").contains("--dataset-table "); + assertThat(out.toString()).contains("--gui").contains("--dataset-table ").contains("--dataset-view "); } /** diff --git a/src/test/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGuiTest.java b/src/test/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGuiTest.java index f9d40e8..061370d 100644 --- a/src/test/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGuiTest.java +++ b/src/test/java/org/filteredpush/bdq_workbench/app/BdqWorkbenchGuiTest.java @@ -6,6 +6,7 @@ import java.lang.reflect.Method; import java.nio.file.Path; import java.time.Instant; +import java.util.ArrayList; import java.util.List; import java.util.Map; import javax.swing.JButton; @@ -713,6 +714,30 @@ void recordFilterProfileSuggestionsSummarizeCommonDatasetValues() throws Excepti assertThat(suggestions).contains(" - Mexico (1)"); } + @Test + void recordFilterProfileSuggestionsShowTwentyValuesThenSummarizeRemainder() throws Exception { + Method profileHelper = BdqWorkbenchGui.class.getDeclaredMethod("profileRecordFilters", RecordDataset.class); + profileHelper.setAccessible(true); + List records = new ArrayList<>(); + for (int i = 1; i <= 21; i++) { + records.add(new CanonicalRecord("r" + i, Map.of("dwc:country", "v" + i))); + } + Object profile = profileHelper.invoke(null, new RecordDataset(records)); + Method suggestionHelper = BdqWorkbenchGui.class.getDeclaredMethod( + "renderRecordFilterValueSuggestions", + profile.getClass(), + String.class, + String.class); + suggestionHelper.setAccessible(true); + + String suggestions = (String) suggestionHelper.invoke(null, profile, "dwc:country", null); + + assertThat(suggestions).contains(" - v1 (1)"); + assertThat(suggestions).contains(" - v21 (1)"); + assertThat(suggestions).doesNotContain(" - v9 (1)"); + assertThat(suggestions).contains("... and 1 more distinct value(s)"); + } + @Test void recordFilterSuggestionsWarnWhenSavedFieldIsMissingFromCurrentDataset() throws Exception { Method profileHelper = BdqWorkbenchGui.class.getDeclaredMethod("profileRecordFilters", RecordDataset.class); diff --git a/src/test/java/org/filteredpush/bdq_workbench/app/ConfigLoaderTest.java b/src/test/java/org/filteredpush/bdq_workbench/app/ConfigLoaderTest.java index 39bd58c..00c27b6 100644 --- a/src/test/java/org/filteredpush/bdq_workbench/app/ConfigLoaderTest.java +++ b/src/test/java/org/filteredpush/bdq_workbench/app/ConfigLoaderTest.java @@ -27,4 +27,13 @@ void rejectsBlankRecordFilterValues() { .isInstanceOf(AppException.class) .hasMessageContaining("values must not be blank"); } + + @Test + void loadsDatasetViewPathFromOverrides() { + AppConfig config = new ConfigLoader().load(Map.of( + "bdq.dataset", "dataset.zip", + "bdq.dataset.view", "/tmp/view.json")); + + assertThat(config.datasetView()).isEqualTo("/tmp/view.json"); + } } diff --git a/src/test/java/org/filteredpush/bdq_workbench/ingest/DataPackageDialectIngestTest.java b/src/test/java/org/filteredpush/bdq_workbench/ingest/DataPackageDialectIngestTest.java index 8fbd9a8..f089d54 100644 --- a/src/test/java/org/filteredpush/bdq_workbench/ingest/DataPackageDialectIngestTest.java +++ b/src/test/java/org/filteredpush/bdq_workbench/ingest/DataPackageDialectIngestTest.java @@ -3,10 +3,16 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import com.fasterxml.jackson.databind.ObjectMapper; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import org.filteredpush.bdq_workbench.app.AppException; import org.filteredpush.bdq_workbench.model.RecordDataset; import org.junit.jupiter.api.Test; @@ -247,6 +253,123 @@ void resourcePathEscapingThePackageDirectoryIsRejected(@TempDir Path tempDir) th .hasMessageContaining("no resource with a readable data file path"); } + @Test + void schemaForeignKeysAreParsed(@TempDir Path tempDir) throws Exception { + writeFile(tempDir, "occurrence.csv", StandardCharsets.UTF_8, "occurrenceID\nocc-1\n"); + writeFile(tempDir, "identification.csv", StandardCharsets.UTF_8, "identificationID,occurrenceID\nid-1,occ-1\n"); + Path manifest = writeManifest(tempDir, """ + { + "resources": [ + { + "name": "occurrence", + "path": "occurrence.csv", + "schema": { "fields": [ { "name": "occurrenceID" } ], "primaryKey": "occurrenceID" } + }, + { + "name": "identification", + "path": "identification.csv", + "schema": { + "fields": [ { "name": "identificationID" }, { "name": "occurrenceID" } ], + "foreignKeys": [ + { "fields": "occurrenceID", "reference": { "resource": "occurrence", "fields": "occurrenceID" } } + ] + } + } + ] + } + """); + + var root = new ObjectMapper().readTree(Files.newBufferedReader(manifest)); + List> tables = + DataPackageDialectParser.parseResources(new ObjectMapper(), root, tempDir); + + assertThat(tables).hasSize(2); + assertThat(tables.get(1).descriptor().foreignKeys()) + .singleElement() + .satisfies(key -> { + assertThat(key.field()).isEqualTo("occurrenceID"); + assertThat(key.referenceResource()).isEqualTo("occurrence"); + assertThat(key.referenceField()).isEqualTo("occurrenceID"); + }); + } + + @Test + void relationalIngestBuildsCoreGraphsFromForeignKeys(@TempDir Path tempDir) throws Exception { + writeFile(tempDir, "occurrence.csv", StandardCharsets.UTF_8, + "occurrenceID,eventDate\nocc-1,2020-01-01\n"); + writeFile(tempDir, "identification.csv", StandardCharsets.UTF_8, + "identificationID,occurrenceID,scientificName\nid-1,occ-1,Abies balsamea\n"); + Path manifest = writeManifest(tempDir, """ + { + "resources": [ + { + "name": "occurrence", + "path": "occurrence.csv", + "schema": { + "fields": [ { "name": "occurrenceID" }, { "name": "eventDate" } ], + "primaryKey": "occurrenceID" + } + }, + { + "name": "identification", + "path": "identification.csv", + "schema": { + "fields": [ { "name": "identificationID" }, { "name": "occurrenceID" }, { "name": "scientificName" } ], + "foreignKeys": [ + { "fields": "occurrenceID", "reference": { "resource": "occurrence", "fields": "occurrenceID" } } + ] + } + } + ] + } + """); + + RelationalIngestResult relational = new RelationalDatasetIngestor().ingest(manifest, "occurrence"); + + assertThat(relational.graphs()).hasSize(1); + assertThat(relational.graphs().get(0).relatedByRelation()).containsKey("identification"); + assertThat(relational.graphs().get(0).relatedByRelation().get("identification")).hasSize(1); + } + + @Test + void defaultIngestServiceReadsZippedDataPackageWithMultipleTables(@TempDir Path tempDir) throws Exception { + Map entries = new LinkedHashMap<>(); + entries.put("datapackage.json", """ + { + "resources": [ + { + "name": "occurrence", + "path": "occurrence.csv", + "schema": { + "fields": [ { "name": "occurrenceID" }, { "name": "eventDate" } ], + "primaryKey": "occurrenceID" + } + }, + { + "name": "identification", + "path": "identification.csv", + "schema": { + "fields": [ { "name": "identificationID" }, { "name": "occurrenceID" }, { "name": "scientificName" } ], + "foreignKeys": [ + { "fields": "occurrenceID", "reference": { "resource": "occurrence", "fields": "occurrenceID" } } + ] + } + } + ] + } + """.getBytes(StandardCharsets.UTF_8)); + entries.put("occurrence.csv", "occurrenceID,eventDate\nocc-1,2020-01-01\n".getBytes(StandardCharsets.UTF_8)); + entries.put("identification.csv", + "identificationID,occurrenceID,scientificName\nid-1,occ-1,Abies balsamea\n".getBytes(StandardCharsets.UTF_8)); + Path archive = Files.createTempFile(tempDir, "datapackage", ".zip"); + writeZipArchive(archive, entries); + + RecordDataset dataset = new DefaultIngestService().ingest(archive, ""); + + assertThat(dataset.records()).hasSize(1); + assertThat(dataset.records().get(0).terms()).containsEntry("scientificName", "Abies balsamea"); + } + /** * Writes a {@code datapackage.json} manifest into a directory. * @@ -274,4 +397,21 @@ private Path writeFile(Path packageDir, String fileName, Charset encoding, Strin Files.writeString(file, content, encoding); return file; } + + /** + * Writes a zip archive from named byte entries. + * + * @param archivePath path of the zip file to create + * @param entries ordered map of entry names to contents + * @throws Exception if the archive cannot be written + */ + private void writeZipArchive(Path archivePath, Map entries) throws Exception { + try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(archivePath), StandardCharsets.UTF_8)) { + for (Map.Entry entry : entries.entrySet()) { + zip.putNextEntry(new ZipEntry(entry.getKey())); + zip.write(entry.getValue()); + zip.closeEntry(); + } + } + } } diff --git a/src/test/java/org/filteredpush/bdq_workbench/ingest/DatasetSchemaInspectorTest.java b/src/test/java/org/filteredpush/bdq_workbench/ingest/DatasetSchemaInspectorTest.java new file mode 100644 index 0000000..c337726 --- /dev/null +++ b/src/test/java/org/filteredpush/bdq_workbench/ingest/DatasetSchemaInspectorTest.java @@ -0,0 +1,68 @@ +package org.filteredpush.bdq_workbench.ingest; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class DatasetSchemaInspectorTest { + + @Test + void dwcaWithOneDeclaredCoreReportsSingleTable(@TempDir Path tempDir) throws Exception { + String meta = """ + + + + occurrence.txt + + + + + """; + Map entries = new LinkedHashMap<>(); + entries.put("meta.xml", meta.getBytes(StandardCharsets.UTF_8)); + entries.put("occurrence.txt", "occurrenceID\nocc-1\n".getBytes(StandardCharsets.UTF_8)); + Path archive = Files.createTempFile(tempDir, "dataset", ".zip"); + try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(archive), StandardCharsets.UTF_8)) { + for (Map.Entry entry : entries.entrySet()) { + zip.putNextEntry(new ZipEntry(entry.getKey())); + zip.write(entry.getValue()); + zip.closeEntry(); + } + } + + DatasetSchemaInspector.DatasetSchemaOverview overview = new DatasetSchemaInspector().inspect(archive); + + assertThat(overview.tables()).hasSize(1); + assertThat(overview.describeTables()).contains("Dataset offers one table, occurrence.txt"); + assertThat(overview.tables().get(0).describe()).contains("declared core"); + } + + @Test + void datapackageWithTwoResourcesReportsTwoTables(@TempDir Path tempDir) throws Exception { + Files.writeString(tempDir.resolve("occurrence.csv"), "occurrenceID\nocc-1\n", StandardCharsets.UTF_8); + Files.writeString(tempDir.resolve("event.csv"), "eventID\nevt-1\n", StandardCharsets.UTF_8); + Files.writeString(tempDir.resolve("datapackage.json"), """ + { + "resources": [ + { "name": "occurrence", "path": "occurrence.csv" }, + { "name": "event", "path": "event.csv" } + ] + } + """, StandardCharsets.UTF_8); + + DatasetSchemaInspector.DatasetSchemaOverview overview = + new DatasetSchemaInspector().inspect(tempDir.resolve("datapackage.json")); + + assertThat(overview.tables()).hasSize(2); + assertThat(overview.describeTables()).isEqualTo("Dataset offers 2 tables"); + } +} diff --git a/src/test/java/org/filteredpush/bdq_workbench/ingest/DatasetViewSchemaTest.java b/src/test/java/org/filteredpush/bdq_workbench/ingest/DatasetViewSchemaTest.java new file mode 100644 index 0000000..3b56f35 --- /dev/null +++ b/src/test/java/org/filteredpush/bdq_workbench/ingest/DatasetViewSchemaTest.java @@ -0,0 +1,124 @@ +package org.filteredpush.bdq_workbench.ingest; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.nio.file.Path; +import java.util.List; +import org.filteredpush.bdq_workbench.app.AppException; +import org.filteredpush.bdq_workbench.model.DatasetSchema; +import org.filteredpush.bdq_workbench.model.DatasetView; +import org.filteredpush.bdq_workbench.model.DatasetViewCardinalityPolicy; +import org.filteredpush.bdq_workbench.model.DatasetViewJoin; +import org.filteredpush.bdq_workbench.model.DatasetViewMapping; +import org.filteredpush.bdq_workbench.model.RelationshipSchema; +import org.filteredpush.bdq_workbench.model.TableSchema; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class DatasetViewSchemaTest { + + @Test + void schemaFingerprintIsStableForEquivalentShapes() { + List first = List.of( + new TableSchema("occurrence", "occurrence", "OCCURRENCE", "occurrenceID", + List.of("occurrenceID", "scientificName")), + new TableSchema("identification", "identification", "OTHER", "identificationID", + List.of("scientificName", "occurrenceID"))); + List second = List.of( + new TableSchema("identification", "identification", "OTHER", "identificationID", + List.of("occurrenceID", "scientificName")), + new TableSchema("occurrence", "occurrence", "OCCURRENCE", "occurrenceID", + List.of("scientificName", "occurrenceID"))); + + assertThat(SchemaFingerprint.of(first, List.of())).isEqualTo(SchemaFingerprint.of(second, List.of())); + } + + @Test + void datasetViewRoundTripsToJsonAndValidatesFingerprint(@TempDir Path tempDir) { + DatasetView view = new DatasetView( + "occurrence", + "fingerprint-1", + List.of(new DatasetViewJoin("identification", "identification", DatasetViewCardinalityPolicy.FIRST_ROW)), + List.of(new DatasetViewMapping("scientificName", "identification", "scientificName"))); + DatasetViewIO io = new DatasetViewIO(); + Path file = tempDir.resolve("view.json"); + + io.save(file, view); + DatasetView loaded = io.load(file); + + assertThat(loaded).isEqualTo(view); + + DatasetSchema compatible = new DatasetSchema(List.of(), List.of(), "fingerprint-1"); + io.validateCompatibility(loaded, compatible); + + DatasetSchema incompatible = new DatasetSchema(List.of(), List.of(), "fingerprint-2"); + assertThatThrownBy(() -> io.validateCompatibility(loaded, incompatible)) + .isInstanceOf(AppException.class) + .hasMessageContaining("fingerprint does not match"); + } + + @Test + void builtInViewIsOnlyAppliedWhenRelationshipsArePresent() { + DatasetSchema withoutRelationship = new DatasetSchema( + List.of( + new TableSchema("event", "event", "EVENT", "eventID", List.of("eventID")), + new TableSchema("occurrence", "occurrence", "OCCURRENCE", "occurrenceID", List.of("occurrenceID"))), + List.of(), + "fp"); + DatasetSchema withRelationship = new DatasetSchema( + withoutRelationship.tables(), + List.of(new RelationshipSchema("occurrence", "coreid", "event", "eventID", "occurrence")), + "fp"); + + assertThat(BuiltInDatasetViews.select(withoutRelationship, new java.util.ArrayList<>())).isEmpty(); + assertThat(BuiltInDatasetViews.select(withRelationship, new java.util.ArrayList<>())).isPresent(); + } + + @Test + void builtInDataPackageViewSourcesScientificNameFromJoinedTableWhenOccurrenceLacksColumn() { + DatasetSchema schema = new DatasetSchema( + List.of( + new TableSchema("occurrence", "occurrence", "OCCURRENCE", "occurrenceID", + List.of("occurrenceID", "eventDate", "decimalLatitude", "decimalLongitude")), + new TableSchema("identification", "identification", "OTHER", "identificationID", + List.of("occurrenceID", "scientificName"))), + List.of(new RelationshipSchema( + "identification", + "occurrenceID", + "occurrence", + "occurrenceID", + "identification")), + "fp"); + + DatasetView view = BuiltInDatasetViews.select(schema, new java.util.ArrayList<>()).orElseThrow(); + + assertThat(view.mappings()) + .anySatisfy(mapping -> { + assertThat(mapping.term()).isEqualTo("scientificName"); + assertThat(mapping.sourceTable()).isEqualTo("identification"); + assertThat(mapping.sourceColumn()).isEqualTo("scientificName"); + }); + } + + @Test + void viewLoadFailureIsReportedAsAppException(@TempDir Path tempDir) { + DatasetViewIO io = new DatasetViewIO(); + Path missing = tempDir.resolve("missing-view.json"); + + assertThatThrownBy(() -> io.load(missing)) + .isInstanceOf(AppException.class) + .hasMessageContaining("Unable to read dataset view file"); + } + + @Test + void viewSaveFailureIsReportedAsAppException(@TempDir Path tempDir) { + DatasetViewIO io = new DatasetViewIO(); + DatasetView view = new DatasetView("occurrence", "fp", List.of(), List.of()); + Path invalid = tempDir.resolve("missing").resolve("view.json"); + + assertThatThrownBy(() -> io.save(invalid, view)) + .isInstanceOf(AppException.class) + .hasMessageContaining("Unable to save dataset view file"); + } +} diff --git a/src/test/java/org/filteredpush/bdq_workbench/ingest/DwcArchiveMetaIngestTest.java b/src/test/java/org/filteredpush/bdq_workbench/ingest/DwcArchiveMetaIngestTest.java index c6b97d5..c9165a0 100644 --- a/src/test/java/org/filteredpush/bdq_workbench/ingest/DwcArchiveMetaIngestTest.java +++ b/src/test/java/org/filteredpush/bdq_workbench/ingest/DwcArchiveMetaIngestTest.java @@ -7,8 +7,10 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; import java.util.zip.ZipOutputStream; import org.filteredpush.bdq_workbench.model.RecordDataset; import org.junit.jupiter.api.Test; @@ -193,6 +195,45 @@ void metaXmlNamingAMissingCoreFileFallsBackToConvention(@TempDir Path tempDir) t assertThat(dataset.records().get(0).terms()).containsEntry("country", "Canada"); } + @Test + void extensionCoreIdColumnIsParsedFromMetaXml(@TempDir Path tempDir) throws Exception { + String meta = """ + + + + event.txt + + + + + occurrence.txt + + + + + """; + Map entries = new LinkedHashMap<>(); + entries.put("meta.xml", meta.getBytes(StandardCharsets.UTF_8)); + entries.put("event.txt", "eventID\nevt-1\n".getBytes(StandardCharsets.UTF_8)); + entries.put("occurrence.txt", "coreid\toccurrenceID\nevt-1\tocc-1\n".getBytes(StandardCharsets.UTF_8)); + Path archive = Files.createTempFile(tempDir, "dataset", ".zip"); + try (ZipOutputStream zip = new ZipOutputStream(Files.newOutputStream(archive), StandardCharsets.UTF_8)) { + for (Map.Entry entry : entries.entrySet()) { + zip.putNextEntry(new ZipEntry(entry.getKey())); + zip.write(entry.getValue()); + zip.closeEntry(); + } + } + + try (ZipFile zipFile = new ZipFile(archive.toFile())) { + List> tables = DwcArchiveMetaParser.parseTables(zipFile); + assertThat(tables).hasSize(2); + assertThat(tables.get(1).descriptor().coreIdColumn()).isEqualTo("coreid"); + } + } + /** * Writes a single-core Darwin Core Archive to a temporary zip file. * diff --git a/src/test/java/org/filteredpush/bdq_workbench/ingest/ViewFlattenerTest.java b/src/test/java/org/filteredpush/bdq_workbench/ingest/ViewFlattenerTest.java new file mode 100644 index 0000000..17a4a8c --- /dev/null +++ b/src/test/java/org/filteredpush/bdq_workbench/ingest/ViewFlattenerTest.java @@ -0,0 +1,131 @@ +package org.filteredpush.bdq_workbench.ingest; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import java.util.Map; +import org.filteredpush.bdq_workbench.model.CanonicalRecord; +import org.filteredpush.bdq_workbench.model.DatasetSchema; +import org.filteredpush.bdq_workbench.model.DatasetView; +import org.filteredpush.bdq_workbench.model.DatasetViewCardinalityPolicy; +import org.filteredpush.bdq_workbench.model.DatasetViewJoin; +import org.filteredpush.bdq_workbench.model.DatasetViewMapping; +import org.filteredpush.bdq_workbench.model.RecordGraph; +import org.filteredpush.bdq_workbench.model.SourceCell; +import org.junit.jupiter.api.Test; + +class ViewFlattenerTest { + + @Test + void aggregatePolicyConcatenatesDeterministicallyAndRetainsProvenance() { + CanonicalRecord core = new CanonicalRecord("occ-1", Map.of("occurrenceID", "occ-1")); + CanonicalRecord idA = new CanonicalRecord("id-1", Map.of("scientificName", "Abies")); + CanonicalRecord idB = new CanonicalRecord("id-2", Map.of("scientificName", "Picea")); + RecordGraph graph = new RecordGraph(core, Map.of("identification", List.of(idA, idB))); + RelationalIngestResult relational = new RelationalIngestResult( + List.of(graph), + new DatasetSchema(List.of(), List.of(), "fp"), + List.of()); + DatasetView view = new DatasetView( + "core", + "fp", + List.of(new DatasetViewJoin("identification", "identification", DatasetViewCardinalityPolicy.AGGREGATE)), + List.of(new DatasetViewMapping("scientificName", "identification", "scientificName"))); + + ViewFlattenResult flattened = new ViewFlattener().flatten(relational, view); + + assertThat(flattened.dataset().records()).hasSize(1); + assertThat(flattened.dataset().records().get(0).terms().get("scientificName")) + .isEqualTo("Abies | Picea"); + assertThat(flattened.dataset().records().get(0).provenanceByTerm().get("scientificName")) + .extracting(SourceCell::rowRef) + .containsExactly("id-1", "id-2"); + } + + @Test + void firstRowPolicyUsesFirstRowDeterministically() { + CanonicalRecord core = new CanonicalRecord("occ-1", Map.of("occurrenceID", "occ-1")); + CanonicalRecord idA = new CanonicalRecord("id-1", Map.of("scientificName", "Abies")); + CanonicalRecord idB = new CanonicalRecord("id-2", Map.of("scientificName", "Picea")); + RecordGraph graph = new RecordGraph(core, Map.of("identification", List.of(idA, idB))); + RelationalIngestResult relational = new RelationalIngestResult( + List.of(graph), + new DatasetSchema(List.of(), List.of(), "fp"), + List.of()); + DatasetView view = new DatasetView( + "core", + "fp", + List.of(new DatasetViewJoin("identification", "identification", DatasetViewCardinalityPolicy.FIRST_ROW)), + List.of(new DatasetViewMapping("scientificName", "identification", "scientificName"))); + + ViewFlattenResult flattened = new ViewFlattener().flatten(relational, view); + + assertThat(flattened.dataset().records().get(0).terms().get("scientificName")).isEqualTo("Abies"); + } + + @Test + void rejectPolicyAddsDiagnosticInsteadOfCrashing() { + CanonicalRecord core = new CanonicalRecord("occ-1", Map.of("occurrenceID", "occ-1")); + CanonicalRecord idA = new CanonicalRecord("id-1", Map.of("scientificName", "Abies")); + CanonicalRecord idB = new CanonicalRecord("id-2", Map.of("scientificName", "Picea")); + RecordGraph graph = new RecordGraph(core, Map.of("identification", List.of(idA, idB))); + RelationalIngestResult relational = new RelationalIngestResult( + List.of(graph), + new DatasetSchema(List.of(), List.of(), "fp"), + List.of()); + DatasetView view = new DatasetView( + "core", + "fp", + List.of(new DatasetViewJoin("identification", "identification", DatasetViewCardinalityPolicy.REJECT)), + List.of(new DatasetViewMapping("scientificName", "identification", "scientificName"))); + + ViewFlattenResult flattened = new ViewFlattener().flatten(relational, view); + + assertThat(flattened.dataset().records().get(0).terms().get("scientificName")).isEmpty(); + assertThat(flattened.diagnostics()).anyMatch(message -> message.contains("Cardinality conflict")); + } + + @Test + void missingJoinAddsDiagnostic() { + CanonicalRecord core = new CanonicalRecord("occ-1", Map.of("occurrenceID", "occ-1")); + RecordGraph graph = new RecordGraph(core, Map.of()); + RelationalIngestResult relational = new RelationalIngestResult( + List.of(graph), + new DatasetSchema(List.of(), List.of(), "fp"), + List.of()); + DatasetView view = new DatasetView( + "occurrence", + "fp", + List.of(), + List.of(new DatasetViewMapping("scientificName", "identification", "scientificName"))); + + ViewFlattenResult flattened = new ViewFlattener().flatten(relational, view); + + assertThat(flattened.diagnostics()).anyMatch(message -> message.contains("no join")); + } + + @Test + void firstRowPolicyKeepsFirstRowEvenWhenBlankAndUsesItsProvenance() { + CanonicalRecord core = new CanonicalRecord("occ-1", Map.of("occurrenceID", "occ-1")); + CanonicalRecord idA = new CanonicalRecord("id-1", Map.of("scientificName", "")); + CanonicalRecord idB = new CanonicalRecord("id-2", Map.of("scientificName", "Picea")); + RecordGraph graph = new RecordGraph(core, Map.of("identification", List.of(idA, idB))); + RelationalIngestResult relational = new RelationalIngestResult( + List.of(graph), + new DatasetSchema(List.of(), List.of(), "fp"), + List.of()); + DatasetView view = new DatasetView( + "occurrence", + "fp", + List.of(new DatasetViewJoin("identification", "identification", DatasetViewCardinalityPolicy.FIRST_ROW)), + List.of(new DatasetViewMapping("scientificName", "identification", "scientificName"))); + + ViewFlattenResult flattened = new ViewFlattener().flatten(relational, view); + + assertThat(flattened.dataset().records().get(0).terms().get("scientificName")).isEmpty(); + assertThat(flattened.dataset().records().get(0).provenanceByTerm().get("scientificName")) + .singleElement() + .extracting(SourceCell::rowRef) + .isEqualTo("id-1"); + } +}