From 17212c9125596038be85ad0a257b2e79d57391da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martynas=20Jusevi=C4=8Dius?= Date: Sun, 27 Sep 2026 17:03:04 +0200 Subject: [PATCH 1/2] The app installs with the ldh CLI. install.sh makes the dataspace public with ldh admin make-public and pushes the root and container documents with a single ldh push, so update-folder.sh goes and import-ns.sh reaches the ontology document through ldh patch, post and clear instead of the platform's bin/ scripts; .ldhignore keeps ns.ttl out of the push, which installs separately. Certificates are the PKCS12 keystore the CLI reads rather than the cert.pem the scripts fed curl, and the option naming follows it: -c, not the -f the CLI no longer has. bin/make-public.sh goes with them - it wrote the same two authorizations straight to the admin store to avoid needing a certificate, which ldh admin make-public now does through the API, so make public calls that too. Co-Authored-By: Claude Opus 5 (1M context) --- app/.ldhignore | 3 +- app/import-ns.sh | 46 +++++++++++++++--------------- app/install.sh | 36 +++++++++--------------- app/update-folder.sh | 63 ----------------------------------------- bin/make-public.sh | 67 -------------------------------------------- 5 files changed, 38 insertions(+), 177 deletions(-) delete mode 100755 app/update-folder.sh delete mode 100755 bin/make-public.sh diff --git a/app/.ldhignore b/app/.ldhignore index e278be7..b742d1b 100644 --- a/app/.ldhignore +++ b/app/.ldhignore @@ -1,2 +1,3 @@ -root.ttl ns.ttl +*.ru +*.sh diff --git a/app/import-ns.sh b/app/import-ns.sh index 8a3a805..c180910 100755 --- a/app/import-ns.sh +++ b/app/import-ns.sh @@ -1,25 +1,24 @@ #!/usr/bin/env bash -# Installs/updates the LTLOD namespace ontology (ns.ttl) into the admin -# dataspace's ontologies/namespace/ document, which LinkedDataHub serves at -# {base}ns. Mirrors LinkedDataHub-Apps demo/northwind-traders: -# 1. PATCH-reset the ontology document (drop everything except the document -# resource and its foaf:primaryTopic), -# 2. POST ns.ttl with a prepended @base <{base}ns> directive so its : prefix -# (<#>) resolves to the end-user namespace, +# Installs/updates the LTLOD namespace ontology (ns.ttl) into the admin dataspace's +# ontologies/namespace/ document, which LinkedDataHub serves at {base}ns. Mirrors +# LinkedDataHub-Apps demo/northwind-traders: +# 1. PATCH-reset the ontology document (drop everything except the document resource and its +# foaf:primaryTopic), +# 2. POST ns.ttl with a prepended @base <{base}ns> directive so its : prefix (<#>) resolves to the +# end-user namespace, # 3. clear the ontology from server memory so it reloads fresh. -# Requires LinkedDataHub's bin/ subdirs on $PATH (patch.sh, post.sh, -# clear-ontology.sh) — `make install` in the root Makefile sets this up. +# Requires `ldh` on $PATH — `make install` in the root Makefile sets this up. set -euo pipefail if [ "$#" -ne 3 ] && [ "$#" -ne 4 ]; then - echo "Usage: $0" '$base $cert_pem_file $cert_password [$proxy]' >&2 - echo "Example: $0" 'https://localhost:4443/ ./ssl/owner/cert.pem Password https://localhost:5443/' >&2 + echo "Usage: $0" '$base $cert_file $cert_password [$proxy]' >&2 + echo "Example: $0" 'https://localhost:4443/ ./ssl/owner/keystore.p12 Password https://localhost:5443/' >&2 echo "Note: special characters such as $ need to be escaped in passwords!" >&2 exit 1 fi base="$1" -cert_pem_file=$(realpath "$2") +cert_file=$(realpath "$2") cert_password="$3" proxy="${4:-$base}" @@ -31,26 +30,27 @@ admin_uri() { admin_base=$(admin_uri "$base") admin_proxy=$(admin_uri "$proxy") +ontology_doc="${admin_base}ontologies/namespace/" -printf "\n### Resetting namespace ontology document: %sontologies/namespace/\n" "$admin_base" -{ echo "BASE <${admin_base}ontologies/namespace/>"; cat "$app_dir/patch-ontology.ru"; } | patch.sh \ - -f "$cert_pem_file" \ +printf "\n### Resetting namespace ontology document: %s\n" "$ontology_doc" +{ echo "BASE <${ontology_doc}>"; cat "$app_dir/patch-ontology.ru"; } | ldh patch \ + -f "$cert_file" \ -p "$cert_password" \ --proxy "$admin_proxy" \ - "${admin_base}ontologies/namespace/" + "$ontology_doc" printf "\n### Appending ns.ttl to the namespace ontology\n" -{ echo "@base <${base}ns> ."; cat "$app_dir/ns.ttl"; } | post.sh \ - -f "$cert_pem_file" \ +{ echo "@base <${base}ns> ."; cat "$app_dir/ns.ttl"; } | ldh post \ + -f "$cert_file" \ -p "$cert_password" \ --proxy "$admin_proxy" \ - --content-type "text/turtle" \ - "${admin_base}ontologies/namespace/" + -t text/turtle \ + "$ontology_doc" printf "\n### Clearing ontology from server memory: %sns#\n" "$base" -clear-ontology.sh \ - -f "$cert_pem_file" \ - -p "$cert_password" \ +ldh admin clear ontology \ -b "$admin_base" \ + -f "$cert_file" \ + -p "$cert_password" \ --proxy "$admin_proxy" \ --ontology "${base}ns#" diff --git a/app/install.sh b/app/install.sh index 2a11452..d161eab 100755 --- a/app/install.sh +++ b/app/install.sh @@ -1,41 +1,31 @@ #!/usr/bin/env bash -# Sets up the LTLOD dataspace on a running LinkedDataHub instance via the LDH -# CLI: makes it publicly readable, then creates/updates the container -# scaffolding (root document + containers + taxonomy scheme containers) and the -# namespace ontology (ns.ttl with 1:N entity views). Requires LinkedDataHub's -# bin/ subdirs and Jena's bin/ (for `turtle`) on $PATH — `make install` in the -# root Makefile sets this up. The public grant equals `make public` -# (bin/make-public.sh) but goes through LDH's HTTP API, so it also works -# against remote instances; both are idempotent. +# Sets up the LTLOD dataspace on a running LinkedDataHub instance with the ldh CLI: makes it +# publicly readable, pushes the container scaffolding (root document + containers + taxonomy scheme +# containers) and installs the namespace ontology (ns.ttl with 1:N entity views). Requires `ldh` +# on $PATH — `make install` in the root Makefile sets this up from ../LinkedDataHub/cli. The public +# grant is the same `ldh admin make-public` that `make public` runs, so either works against a +# remote instance; both are idempotent, and PUT replaces each document. set -euo pipefail if [ "$#" -ne 3 ] && [ "$#" -ne 4 ]; then - echo "Usage: $0" '$base $cert_pem_file $cert_password [$proxy]' >&2 - echo "Example: $0" 'https://localhost:4443/ ./ssl/owner/cert.pem Password https://localhost:5443/' >&2 + echo "Usage: $0" '$base $cert_file $cert_password [$proxy]' >&2 + echo "Example: $0" 'https://localhost:4443/ ./ssl/owner/keystore.p12 Password https://localhost:5443/' >&2 echo "Note: special characters such as $ need to be escaped in passwords!" >&2 exit 1 fi base="$1" -cert_pem_file=$(realpath "$2") +cert_file=$(realpath "$2") cert_password="$3" proxy="${4:-$base}" app_dir="$(cd "$(dirname "$0")" && pwd)" printf "\n### Creating authorization to make the dataspace public\n" -make-public.sh -b "$base" -f "$cert_pem_file" -p "$cert_password" --proxy "$proxy" +ldh admin make-public -b "$base" -c "$cert_file" -p "$cert_password" --proxy "$proxy" -printf "\n### Updating root document: %s\n" "$base" -turtle --base="$base" < "$app_dir/root.ttl" | put.sh \ - -f "$cert_pem_file" \ - -p "$cert_password" \ - --proxy "$proxy" \ - -t "application/n-triples" \ - "$base" - -printf "\n### Updating container documents\n" -"$app_dir/update-folder.sh" "$base" "$cert_pem_file" "$cert_password" "$app_dir" "$app_dir" "$proxy" +printf "\n### Pushing root and container documents\n" +ldh push -b "$base" -c "$cert_file" -p "$cert_password" --proxy "$proxy" --dir "$app_dir" "$base" printf "\n### Updating namespace ontology\n" -"$app_dir/import-ns.sh" "$base" "$cert_pem_file" "$cert_password" "$proxy" +"$app_dir/import-ns.sh" "$base" "$cert_file" "$cert_password" "$proxy" diff --git a/app/update-folder.sh b/app/update-folder.sh deleted file mode 100755 index 8c3a3df..0000000 --- a/app/update-folder.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash -# Recursively PUT every *.ttl in a folder tree as a LinkedDataHub document: -# /foo.ttl -> ${base}foo/ (path = file path minus $pwd prefix and -# extension, plus trailing slash). Simplified from linkeddatahub.com's -# update-folder.sh: TTL documents only, no file uploads. A folder's .ttl files -# are processed before its subdirectories, so parent containers always exist -# before their children (taxonomies.ttl before taxonomies/*.ttl). -set -e - -if [ "$#" -ne 5 ] && [ "$#" -ne 6 ]; then - echo "Usage: $0" '$base $cert_pem_file $cert_password $pwd $abs_folder [$proxy]' >&2 - echo "Example: $0" 'https://localhost:4443/ ./ssl/owner/cert.pem Password /folder /folder [https://localhost:5443/]' >&2 - echo "Note: special characters such as $ need to be escaped in passwords!" >&2 - exit 1 -fi - -base="$1" -cert_pem_file="$2" -cert_password="$3" -pwd="$4" -folder="$5" -proxy="${6:-$base}" - -ldhignore_file="$folder/.ldhignore" - -is_ldhignored() { - local name - name="$(basename "$1")" - [[ -f "$ldhignore_file" ]] || return 1 - while IFS= read -r pattern || [[ -n "$pattern" ]]; do - [[ -z "$pattern" || "$pattern" == \#* ]] && continue - pattern="${pattern%/}" - [[ "$name" == $pattern ]] && return 0 - done < "$ldhignore_file" - return 1 -} - -for ttl_file in "$folder"/*.ttl; do - if [[ -f "$ttl_file" ]]; then - if git check-ignore -q "$ttl_file" 2>/dev/null || is_ldhignored "$ttl_file"; then - printf "Skipping %s\n" "$ttl_file" - continue - fi - path="${ttl_file%.*}" # strip extension - path="${path#*$pwd/}" # strip leading $pwd/ - path="${path}/" # add trailing slash - printf "\n### Updating %s\n" "${base}${path}" - cat "$ttl_file" | turtle --base="${base}${path}" | put.sh \ - -f "$cert_pem_file" \ - -p "$cert_password" \ - --proxy "$proxy" \ - -t "application/n-triples" \ - "${base}${path}" - fi -done - -while IFS= read -r subdir; do - if git check-ignore -q "$subdir" 2>/dev/null || is_ldhignored "$subdir"; then - printf "Skipping %s\n" "$subdir" - continue - fi - "$(dirname "$0")/update-folder.sh" "$base" "$cert_pem_file" "$cert_password" "$pwd" "$subdir" "$proxy" -done < <(find "$folder" -mindepth 1 -maxdepth 1 -type d -not -name '.*' -not -name 'target') diff --git a/bin/make-public.sh b/bin/make-public.sh deleted file mode 100755 index c6487f9..0000000 --- a/bin/make-public.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash -# Makes all documents of the end-user application publicly readable. -# Direct-to-triplestore equivalent of LinkedDataHub CLI's make-public.sh: -# runs the same SPARQL update against fuseki-admin from inside the docker -# network, so no owner WebID certificate or published ports are needed. -set -e - -if [ "$#" -ne 1 ]; then - echo "Usage: $0" '$env_file' >&2 - echo "Example: $0 .env" >&2 - exit 1 -fi - -env_file="$1" - -function envProp { - local expectedKey=$1 - while IFS='=' read -r k v; do - if [ -n "$k" ] && [ "$k" == "$expectedKey" ] ; then - echo "$v"; - break; - fi - done < "$env_file" -} - -if [ "$(envProp "HTTPS_PORT")" = 443 ]; then - base_uri="$(envProp "PROTOCOL")://$(envProp "HOST")$(envProp "ABS_PATH")" -else - base_uri="$(envProp "PROTOCOL")://$(envProp "HOST"):$(envProp "HTTPS_PORT")$(envProp "ABS_PATH")" -fi -admin_base_uri=$(echo "$base_uri" | sed 's|://|://admin.|') - -printf "### Granting public access on: %s\n" "$base_uri" - -docker compose exec -T linkeddatahub curl -s -f -X POST \ - -H "Content-Type: application/sparql-update" \ - --data-binary @- \ - http://varnish-admin/ds/ < -PREFIX def: -PREFIX dh: -PREFIX nfo: -PREFIX foaf: - -INSERT DATA -{ - GRAPH <${admin_base_uri}acl/authorizations/public/> - { - <${admin_base_uri}acl/authorizations/public/#this> acl:accessToClass def:Root, dh:Container, dh:Item, nfo:FileDataObject ; - acl:accessTo <${base_uri}sparql> . - - <${admin_base_uri}acl/authorizations/public/#sparql-post> a acl:Authorization ; - acl:accessTo <${base_uri}sparql> ; - acl:mode acl:Append ; - acl:agentClass foaf:Agent, acl:AuthenticatedAgent . # hacky way to allow queries over POST - } -} -EOF - -# the update bypassed LDH, so cached ACL lookups must be dropped -docker compose restart varnish-admin - -# wait until varnish-admin accepts connections again — LDH returns 500s on ACL -# lookups while it is down -until docker compose exec -T linkeddatahub curl -s -o /dev/null http://varnish-admin/; do - sleep 1 -done From ba08c97b97f3ae327daf4d7539a933657439099b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martynas=20Jusevi=C4=8Dius?= Date: Sun, 27 Sep 2026 17:03:35 +0200 Subject: [PATCH 2/2] docker-compose.yml, Makefile and bin/server-cert-gen.sh are byte-identical copies of LinkedDataHub's, and every delta lives in an override or an include. One Fuseki serves both dataspace roles as its own dataset: the mount widens from ./fuseki/ to ./fuseki and each location gains the role, so the TDB2 folders stay where fuseki-admin and fuseki-end-user wrote them and no data moves - the role used to be encoded in the per-container mount, which one server cannot do. system.trig follows to fuseki:3030/admin/ and /end-user/, and both Varnish backends to the one host. The copy also brings the egress forward proxy that confines the store's SPARQL SERVICE and LOAD to public addresses, the sef-compiler the platform calls to compose package stylesheets, the ./sef, ./packages and ./settings mounts, Memento-Datetime in the four CORS expose-header lines, and the ?query= ban that invalidates cached SPARQL results after a write. The Makefile keeps only what every deployment shares; make/config.mk carries this repo's settings and make/local.mk its interactive install. BASE_URI is derived from .env without ABS_PATH, which had exactly one working value and is gone from .env and from bin/server-cert-gen.sh, re-copied from the platform. `make sef` had resolved the platform image with an unanchored grep that matched the sef-compiler's just as readily, in an order Compose does not guarantee, and it now refuses up front when the pinned image is neither built nor pullable instead of failing four commands later. `make load` clears every dataset's tdb.lock rather than only the end-user one: stopping the single server leaves a PID-1 lock in each, and any of them blocks the restart. Co-Authored-By: Claude Opus 5 (1M context) --- .env | 1 - .gitignore | 3 + CLAUDE.md | 30 ++-- Makefile | 286 +++++++++++++++++++++--------------- README.md | 8 +- bin/server-cert-gen.sh | 14 +- config/fuseki/config.ttl | 36 ++++- config/system.trig | 12 +- docker-compose.override.yml | 10 +- docker-compose.yml | 86 +++++++---- make/config.mk | 15 ++ make/local.mk | 41 ++++++ 12 files changed, 360 insertions(+), 182 deletions(-) create mode 100644 make/config.mk create mode 100644 make/local.mk diff --git a/.env b/.env index 9ce6827..ac565db 100644 --- a/.env +++ b/.env @@ -5,7 +5,6 @@ HTTP_PORT=81 HTTPS_PORT=4443 HTTPS_CLIENT_CERT_PORT=5443 HOST=localhost -ABS_PATH=/ OWNER_MBOX=martynas@atomgraph.com OWNER_GIVEN_NAME=Martynas diff --git a/.gitignore b/.gitignore index 9082b2f..d8f3126 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,9 @@ files/client.xsl.sef.json /secrets/ /fuseki/ /uploads/ +/sef/ +/packages/ +/settings/ /datasets/owner/ /datasets/secretary/ docker-compose.override.yml diff --git a/CLAUDE.md b/CLAUDE.md index fc29dd0..c715282 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -33,7 +33,7 @@ make install # set up the dataspace via LDH CLI: make it publi # cert / password / proxy, defaults = the local stack (Enter×4 # or `printf '\n\n\n\n' | make install`); enter another Base URL # + owner cert to install onto any LDH instance -make load # bulk-load datasets/current/*/*.trig into fuseki-end-user TDB2; +make load # bulk-load datasets/current/*/*.trig into the end-user TDB2 dataset; # resolves the base-relative TriG against BASE_URI (.env) via riot # before tdb2.tdbloader; ends with `make public` (anonymous read) make down / make drop # stop stack / wipe LDH runtime state (never datasets/current/) @@ -217,6 +217,12 @@ since the rows differ in the image cell. ## Gotchas +- **`$ldt:base` no longer exists in LDH** — use `lds:base()` (`xs:anyURI`, the dataspace base) in + `files/layout.xsl` and `files/client.xsl`. LinkedDataHub `ffc28c0f4` and Web-Client `d7cac4638` + (2026-09-12) removed the param with no back-compat shim, so an app stylesheet still referencing it + fails to compile (`XPST0008` -> "Too many errors") and EVERY end-user page 500s. `xmlns:lds` must + be declared on the stylesheet. Both stylesheet trees define `lds:base()`, so the same call works + server-side (`server.xsl`) and client-side (`client/functions.xsl`). - **BINDs inside OPTIONAL are evaluated bottom-up**: a BIND referencing an outer variable (e.g. `?graph`) silently unbinds and drops triples. Keep only triple patterns inside OPTIONAL; do URI construction after it, guarded with @@ -254,12 +260,14 @@ since the rows differ in the image cell. `BASE_URI` (from `.env`, passed as `-e BASE_URI=…`) with `riotcmd.riot --base=… --output=nquads`, then loads the N-Quads (tdb2.tdbloader has no `--base`). Load is append-only — clean rebuild: `make down && rm -rf fuseki/end-user && - make up && make load`. It stops fuseki-end-user first and removes the stale - `tdb.lock` (lock PIDs are container-relative), then restarts the Varnish caches. + make up && make load`. It stops fuseki first — one server now holds both roles, so + the admin store goes down with it — and removes the stale `tdb.lock` (lock PIDs are + container-relative), then restarts the Varnish caches. - **`docker-compose.yml` is a verbatim mirror of `../LinkedDataHub/docker-compose.yml`** - (only `build: .` → `image: atomgraph/linkeddatahub:5.6.0`, because LTLOD pulls the - published image and `make sef`/`make up` grep that line). Every LTLOD-specific delta - lives in the **committed** `docker-compose.override.yml` (compose auto-merges it): + — a byte-identical copy, so `diff ../LinkedDataHub/docker-compose.yml docker-compose.yml` + prints nothing. Every LTLOD-specific delta lives in the **committed** + `docker-compose.override.yml` (compose auto-merges it), the `image:` pin that stands in + for the base's `build: .` included — this repo has no Dockerfile: the runtime image pin, `TZ="Europe/Vilnius"`, `ENABLE_WEBID_SIGNUP=false`, and the `tdb-loader` bulk-load service (`profiles: [ load ]`, so `make up` skips it and `make load` starts it via `docker compose run`). Re-sync from upstream by re-copying LDH's @@ -299,8 +307,8 @@ since the rows differ in the image cell. adds `dh:Item` + `sioc:has_container` otherwise, plus `dct:created`/ `acl:owner`) — never put sioc triples in `app/*.ttl`. `make install` is idempotent: PUT replaces the whole named graph. -- **Public read access is class-based**: `make public` (direct-to-fuseki) and - `make install` (LDH CLI `make-public.sh`, works remotely) grant the same +- **Public read access is class-based**: both `make public` and `make install` run + `ldh admin make-public` through LDH's HTTP API, granting `acl:accessToClass def:Root, dh:Container, dh:Item, nfo:FileDataObject` — ETL documents match because mappings type them `dh:Item`/`dh:Container`. (Untyped docs would also pass: LDH's ACL query leaves `$Type` unbound when @@ -310,9 +318,9 @@ since the rows differ in the image cell. LDH stacks, but ports 81/4443/5443 still clash — one stack at a time. - **502 on all public endpoints after restarting backend containers** (fuseki, varnish): nginx resolves upstream container IPs at startup — restart nginx - too. `fuseki-end-user` can be OOM-killed (exit 137) under memory pressure - when other Docker workloads run; `docker compose up -d fuseki-end-user` - revives it (LDH health recovers on its own). + too. `fuseki` can be OOM-killed (exit 137) under memory pressure when other + Docker workloads run; `docker compose up -d fuseki` revives it (LDH health + recovers on its own). ## Verification diff --git a/Makefile b/Makefile index 22ecd75..d6bcb4a 100644 --- a/Makefile +++ b/Makefile @@ -1,148 +1,200 @@ +# Canonical LinkedDataHub deployment Makefile. +# +# Byte-identical across LinkedDataHub and every LDH-based deployment repo, the same convention +# docker-compose.yml follows. Do not edit it per repo - re-sync with +# +# cp ../LinkedDataHub/Makefile Makefile +# +# and keep everything repo-specific in the two optional includes: +# +# make/config.mk variables: which scripts to call, what `drop` wipes, what `sef` stages +# make/local.mk targets this repo adds - or replaces, by naming them in LOCAL_TARGETS +# +# `diff ../LinkedDataHub/Makefile Makefile` must print nothing. + SHELL := /bin/bash -.PHONY: up down stop logs cert secrets install load public drop sef +-include make/config.mk + +# This deployment's settings, the same file docker compose reads. Optional, so targets that need +# no base URI still run before it is written; the URI variables below are lazily expanded and say +# so when it is missing. +-include .env + +COMPOSE ?= docker compose +CERT_GEN ?= ./bin/server-cert-gen.sh +LDH_HOME ?= ../LinkedDataHub +LDH ?= $(LDH_HOME)/cli/bin/ldh +OWNER_CERT ?= ssl/owner/keystore.p12 +OWNER_PASSWORD_FILE ?= secrets/owner_cert_password.txt +LOGS_SERVICE ?= linkeddatahub +SEF_ENTRY ?= files/client.xsl +SEF_OUT ?= files/client.xsl.sef.json +SEF_EXTRA ?= +DROP_PATHS ?= datasets fuseki ssl uploads sef packages settings +VALIDATE_PATHS ?= . +LOAD_STAGING ?= +HTTPS_CLIENT_CERT_PORT ?= 5443 +PROJECT ?= $(or $(COMPOSE_PROJECT_NAME),$(notdir $(CURDIR))) + +# A dataspace serves its documents from the root of its origin, so the base URI is that root and +# has no path component. Lazily expanded (`=`, not `:=`), so the error fires only when a target +# actually needs a URI. +ORIGIN = $(if $(and $(PROTOCOL),$(HOST)),$(PROTOCOL)://$(HOST)$(if $(filter-out 443,$(HTTPS_PORT)),:$(HTTPS_PORT)),$(error .env is missing or incomplete: PROTOCOL and HOST are required)) +BASE_URI = $(ORIGIN)/ +PROXY_URI = $(PROTOCOL)://$(HOST):$(HTTPS_CLIENT_CERT_PORT)/ + +TARGETS := up down stop restart ps logs cert secrets sef public load validate drop +COMPOSE_TARGETS := up down stop restart ps logs + +.PHONY: $(TARGETS) $(LOCAL_TARGETS) + +# Treat goals that are not targets as arguments for docker compose rather than as make goals, so +# `make up -- --build -d` and `make up nginx` work. +ifneq (,$(filter $(COMPOSE_TARGETS),$(MAKECMDGOALS))) +COMPOSE_ARGS := $(filter-out $(TARGETS) $(LOCAL_TARGETS),$(MAKECMDGOALS)) +$(eval $(COMPOSE_ARGS):;@:) +endif -# LDH CLI checkout (provides put.sh etc.); Jena provides the `turtle` command -LDH_HOME ?= ../LinkedDataHub -include etl/config.mk # for JENA_HOME (its BASE is unused here) -include .env # PROTOCOL/HOST/HTTPS_PORT/HTTPS_CLIENT_CERT_PORT/ABS_PATH +# --- stack ------------------------------------------------------------------- -ifeq ($(HTTPS_PORT),443) -BASE_URI := $(PROTOCOL)://$(HOST)$(ABS_PATH) -else -BASE_URI := $(PROTOCOL)://$(HOST):$(HTTPS_PORT)$(ABS_PATH) -endif -PROXY_URI := $(PROTOCOL)://$(HOST):$(HTTPS_CLIENT_CERT_PORT)$(ABS_PATH) +up: secrets cert + $(COMPOSE) up $(ARGS) $(COMPOSE_ARGS) + +down: + $(COMPOSE) down $(ARGS) $(COMPOSE_ARGS) + +stop: + $(COMPOSE) stop $(ARGS) $(COMPOSE_ARGS) + +restart: + $(COMPOSE) restart $(ARGS) $(COMPOSE_ARGS) + +ps: + $(COMPOSE) ps $(ARGS) $(COMPOSE_ARGS) + +# Follows LOGS_SERVICE unless the command line names other services. +logs: + $(COMPOSE) logs -f $(ARGS) $(or $(COMPOSE_ARGS),$(LOGS_SERVICE)) + +# --- first-run bootstrap ------------------------------------------------------ SECRET_FILES := secrets/owner_cert_password.txt \ secrets/secretary_cert_password.txt \ secrets/client_truststore_password.txt +secrets: $(SECRET_FILES) + secrets/%.txt: @mkdir -p secrets openssl rand -base64 24 > $@ -secrets: $(SECRET_FILES) - -ssl/server/server.crt: - ./bin/server-cert-gen.sh .env nginx ssl - +# Generate the server SSL certificate from .env. A file target, so `make up` does not regenerate +# it on every start. cert: ssl/server/server.crt -# Compile the client-side XSLT override (files/client.xsl) to a Saxon-JS SEF. -# Copies the deployed LDH image's ROOT/static tree into a temp dir so the -# stylesheet's `../com/atomgraph/linkeddatahub/xsl/client.xsl` import resolves, -# canonicalizes the source, then compiles with xslt3-he. Run once before the -# first `make up` (the compose mount needs the file to exist) and after any edit -# to files/client.xsl; then recreate the container to reload. Requires Node/npx -# (xslt3-he) and xmlstarlet. -# Resolve the EFFECTIVE image from the merged compose config (base + -# docker-compose.override.yml), NOT the base docker-compose.yml alone: an image -# pin in the override (e.g. a dev build) must be honoured, else the SEF compiles -# against a different base stylesheet tree than the runtime serves and CSR -# diverges from SSR (e.g. gsp:asWKT suppression that works server-side but not -# client-side). +ssl/server/server.crt: + $(CERT_GEN) .env nginx ssl + +# --- client stylesheet -------------------------------------------------------- + +ifeq ($(filter sef,$(LOCAL_TARGETS)),) +ifneq ($(wildcard $(SEF_ENTRY)),) +# Compile this deployment's client.xsl override to a Saxon-JS SEF. Stages the deployed image's +# ROOT/static tree in a temp dir so the stylesheet's ../com/atomgraph/linkeddatahub/xsl/client.xsl +# import resolves, canonicalizes every stylesheet there (the platform build inlines XML entities +# the same way), then compiles. Run it before the first `make up` - the compose mount needs the +# file to exist - and after every edit, then recreate the container to reload it. +# +# Resolve the EFFECTIVE image from the merged compose config, never from docker-compose.yml +# alone: an image pin in an override has to win, or the SEF compiles against a different +# stylesheet tree than the runtime serves and CSR diverges from SSR. The pattern is anchored on +# the tag (or digest) separator because `--images ` ignores the service filter and lists +# every image, so a bare `linkeddatahub` also matches the sef-compiler's, in an order Compose +# does not guarantee. sef: - @LDH_IMAGE=$$(docker compose config --images linkeddatahub | grep -m1 'linkeddatahub'); \ + @set -e; \ + LDH_IMAGE=$$($(COMPOSE) config --images linkeddatahub | grep -m1 -E 'linkeddatahub[:@]'); \ + [ -n "$$LDH_IMAGE" ] || { echo "ERROR: no linkeddatahub image in the merged compose config" >&2; exit 1; }; \ echo "Using LDH image: $$LDH_IMAGE"; \ + docker image inspect "$$LDH_IMAGE" >/dev/null 2>&1 || docker pull "$$LDH_IMAGE" >/dev/null 2>&1 || \ + { echo "ERROR: $$LDH_IMAGE is neither built locally nor pullable - fix the image pin in docker-compose.override.yml" >&2; exit 1; }; \ TMP_DIR=$$(mktemp -d); \ - docker create --name ltlod-sef-tmp "$$LDH_IMAGE" >/dev/null; \ - docker cp ltlod-sef-tmp:/usr/local/tomcat/webapps/ROOT/static "$$TMP_DIR/"; \ - docker rm ltlod-sef-tmp >/dev/null; \ + trap 'rm -rf "$$TMP_DIR"; docker rm -f $(PROJECT)-sef-tmp >/dev/null 2>&1 || true' EXIT; \ + docker create --name $(PROJECT)-sef-tmp "$$LDH_IMAGE" >/dev/null; \ + docker cp $(PROJECT)-sef-tmp:/usr/local/tomcat/webapps/ROOT/static "$$TMP_DIR/"; \ + docker rm $(PROJECT)-sef-tmp >/dev/null; \ find "$$TMP_DIR/static" -name '*.xsl' -print0 | while IFS= read -r -d '' f; do xmlstarlet c14n "$$f" > "$$f.tmp" 2>/dev/null && mv "$$f.tmp" "$$f" || rm -f "$$f.tmp"; done; \ - mkdir -p "$$TMP_DIR/static/files" && xmlstarlet c14n ./files/client.xsl > "$$TMP_DIR/static/files/client.xsl"; \ - xmlstarlet c14n ./files/overrides.xsl > "$$TMP_DIR/static/files/overrides.xsl"; \ - npx xslt3-he -t -xsl:"$$TMP_DIR/static/files/client.xsl" -export:"$$TMP_DIR/client.xsl.sef.json" -nogo -ns:##html5 -relocate:on; \ - if [ $$? -ne 0 ] || [ ! -s "$$TMP_DIR/client.xsl.sef.json" ]; then \ - rm -rf "$$TMP_DIR"; \ - echo "SEF compile FAILED - files/client.xsl.sef.json left unchanged" >&2; \ - exit 1; \ - fi; \ - mv "$$TMP_DIR/client.xsl.sef.json" ./files/client.xsl.sef.json; \ - rm -rf "$$TMP_DIR"; \ - echo "Wrote files/client.xsl.sef.json" + mkdir -p "$$TMP_DIR/static/files"; \ + for f in $(SEF_ENTRY) $(SEF_EXTRA); do xmlstarlet c14n "./$$f" > "$$TMP_DIR/static/files/$$(basename "$$f")"; done; \ + npx xslt3-he -t -xsl:"$$TMP_DIR/static/files/$$(basename $(SEF_ENTRY))" -export:"$$TMP_DIR/out.sef.json" -nogo -ns:##html5 -relocate:on || \ + { echo "SEF compile FAILED - $(SEF_OUT) left unchanged" >&2; exit 1; }; \ + [ -s "$$TMP_DIR/out.sef.json" ] || { echo "SEF compile produced nothing - $(SEF_OUT) left unchanged" >&2; exit 1; }; \ + mv "$$TMP_DIR/out.sef.json" $(SEF_OUT); \ + echo "Wrote $(SEF_OUT)" +else +sef: + @echo "ERROR: $(SEF_ENTRY) not found - this deployment overrides no client stylesheet" >&2; exit 1 +endif +endif -up: secrets cert - docker compose up - @echo "LinkedDataHub starting — first boot takes ~1-2 min (self-signed cert)." - @echo "URL: https://localhost:4443/" +# --- app ---------------------------------------------------------------------- -down: - docker compose down +# `install` is deliberately absent: every deployment installs its own app structure its own +# way, so each defines `install` (and any install-prod) in make/local.mk. -stop: - docker compose stop +ifeq ($(filter public,$(LOCAL_TARGETS)),) +# Grant anonymous read on every end-user document. Idempotent - the CLI PATCHes one authorization. +public: + @[ -x "$(LDH)" ] || { echo "ERROR: ldh CLI not found at $(LDH) - run 'make cli' in $(LDH_HOME)"; exit 1; } + @[ -f $(OWNER_PASSWORD_FILE) ] || { echo "ERROR: $(OWNER_PASSWORD_FILE) not found - run 'make secrets' and install first"; exit 1; } + LDH_BASE="$(BASE_URI)" \ + LDH_CERT_FILE="$(OWNER_CERT)" \ + LDH_CERT_PASSWORD="$$(cat $(OWNER_PASSWORD_FILE))" \ + LDH_PROXY="$(PROXY_URI)" \ + $(LDH) admin make-public -logs: - docker compose logs -f linkeddatahub - -# Install the app structure (root + containers + taxonomy schemes + the -# namespace ontology with 1:N views) onto a LinkedDataHub instance via LDH CLI -# PUTs. Interactive, LinkedDataHub-Apps style: prompts for the target instance -# with defaults from the local docker-compose stack (.env, ssl/, secrets/) — -# press Enter to install locally, or enter another Base URL + owner cert to -# install on any LDH instance. Re-running is safe (PUT replaces). Local order: -# make up -> make install -> make load. -install: - @[ -d "$(LDH_HOME)/bin" ] || \ - { echo "ERROR: LDH CLI not found — clone https://github.com/AtomGraph/LinkedDataHub to $(LDH_HOME) or pass LDH_HOME=…"; exit 1; } - @read -p "Enter Base URL [$(BASE_URI)]: " BASE_URL; \ - BASE_URL=$${BASE_URL:-$(BASE_URI)}; \ - read -p "Enter Certificate Path [ssl/owner/cert.pem]: " CERT_PATH; \ - CERT_PATH=$${CERT_PATH:-ssl/owner/cert.pem}; \ - [ -f "$$CERT_PATH" ] || { echo "ERROR: certificate not found: $$CERT_PATH"; exit 1; }; \ - PW_DEFAULT=""; \ - [ -f secrets/owner_cert_password.txt ] && PW_DEFAULT="$$(cat secrets/owner_cert_password.txt)"; \ - if [ -n "$$PW_DEFAULT" ]; then \ - read -r -s -p "Enter Certificate Password [from secrets/owner_cert_password.txt]: " PASSWORD; \ - else \ - read -r -s -p "Enter Certificate Password (required): " PASSWORD; \ - fi; \ - echo ""; \ - PASSWORD=$${PASSWORD:-$$PW_DEFAULT}; \ - if [ -z "$$PASSWORD" ]; then echo "Password cannot be empty. Aborting."; exit 1; fi; \ - PROXY_DEFAULT=""; \ - [ "$$BASE_URL" = "$(BASE_URI)" ] && PROXY_DEFAULT="$(PROXY_URI)"; \ - read -p "Enter Proxy URL (optional) [$$PROXY_DEFAULT]: " PROXY_URL; \ - PROXY_URL=$${PROXY_URL:-$$PROXY_DEFAULT}; \ - if [ "$$BASE_URL" = "$(BASE_URI)" ] && [ -n "$$(docker compose ps -q linkeddatahub 2>/dev/null)" ]; then \ - echo "Waiting for LinkedDataHub health (first-boot seeding must finish)..."; \ - until [ "$$(docker inspect -f '{{.State.Health.Status}}' $$(docker compose ps -q linkeddatahub))" = "healthy" ]; do \ - sleep 5; echo " ...waiting"; \ - done; \ - fi; \ - export PATH="$$(find "$$(cd $(LDH_HOME) && pwd)/bin" -type d | tr '\n' ':')$(JENA_HOME)/bin:$$PATH"; \ - if [ -n "$$PROXY_URL" ]; then \ - ./app/install.sh "$$BASE_URL" "$$CERT_PATH" "$$PASSWORD" "$$PROXY_URL"; \ - else \ - ./app/install.sh "$$BASE_URL" "$$CERT_PATH" "$$PASSWORD"; \ - fi - -# Bulk-load datasets/current/*/*.trig into the end-user TDB2 store. The committed -# TriG is base-relative; the loader resolves it against BASE_URI (from .env) so -# the same files load at whatever base this deployment uses — no per-base regen. -# APPEND-ONLY: clean rebuild = `make down && rm -rf fuseki/end-user && make up && make load`. +endif + +ifneq ($(LOAD_STAGING),) +# Bulk-load $(LOAD_STAGING)/*/*.trig straight into the end-user TDB2 dataset, bypassing the HTTP +# API - minutes instead of hours for millions of quads. The committed TriG is base-relative, so +# the loader resolves it against BASE_URI and the same files load at whatever base this +# deployment uses. APPEND-ONLY: for a clean rebuild delete fuseki/end-user first. One Fuseki +# serves both roles, so stopping it for the load takes the admin store down with it - and its +# stop leaves a PID-1 tdb.lock in every dataset, each of which would block the restart, so all +# of them are cleared and not just the one being loaded. load: - @ls datasets/current/*/*.trig >/dev/null 2>&1 || \ - { echo "ERROR: no TriG files under datasets/current/ — run 'make -C etl' first."; exit 1; } - @[ -n "$$(docker compose ps -q fuseki-end-user)" ] || \ - { echo "ERROR: fuseki-end-user container not found — run 'make up' first."; exit 1; } + @ls $(LOAD_STAGING)/*/*.trig >/dev/null 2>&1 || \ + { echo "ERROR: no TriG files under $(LOAD_STAGING)/ - run 'make -C etl' first."; exit 1; } + @[ -n "$$($(COMPOSE) ps -q fuseki)" ] || \ + { echo "ERROR: fuseki container not found - run 'make up' first."; exit 1; } @echo "Waiting for LinkedDataHub health (first-boot seeding must finish)..." - @until [ "$$(docker inspect -f '{{.State.Health.Status}}' $$(docker compose ps -q linkeddatahub))" = "healthy" ]; do \ + @until [ "$$(docker inspect -f '{{.State.Health.Status}}' $$($(COMPOSE) ps -q linkeddatahub))" = "healthy" ]; do \ sleep 5; echo " ...waiting"; \ done - docker compose stop fuseki-end-user - rm -f fuseki/end-user/DB2/tdb.lock - docker compose run --rm -e BASE_URI="$(BASE_URI)" tdb-loader - docker compose up -d fuseki-end-user - docker compose restart varnish-end-user varnish-frontend + $(COMPOSE) stop fuseki + rm -f fuseki/*/DB2/tdb.lock + $(COMPOSE) run --rm -e BASE_URI="$(BASE_URI)" tdb-loader + $(COMPOSE) up -d fuseki + $(COMPOSE) restart varnish-end-user varnish-frontend $(MAKE) public +else +load: + @echo "ERROR: this deployment has no bulk loader (set LOAD_STAGING in make/config.mk)" >&2; exit 1 +endif -# Grant anonymous read access (idempotent; equivalent of LDH CLI make-public.sh) -public: - ./bin/make-public.sh .env +# --- housekeeping ------------------------------------------------------------- -# Wipes LDH runtime state. NEVER touches datasets/current/. +# Parse every RDF file under VALIDATE_PATHS. Needs Jena's riot on PATH ($JENA_HOME/bin). +validate: + @find $(VALIDATE_PATHS) \( -name '*.ttl' -o -name '*.trig' \) -type f -print0 | xargs -0 riot --validate + +# Stop the stack, remove its volumes and wipe this deployment's local state - irreversible. +# Stops first on purpose: deleting the directories under a running Fuseki leaves it writing into +# paths that no longer exist. drop: - @read -p "Delete fuseki/, ssl/, secrets/, uploads/, datasets/{owner,secretary}? [y/N] " ans && \ - [ "$$ans" = "y" ] && { docker compose down -v; sudo rm -rf datasets/owner datasets/secretary fuseki ssl secrets uploads; } || echo "Aborted." + @read -p "Stop the stack and delete $(DROP_PATHS)? [y/N] " ans && [ "$$ans" = "y" ] || { echo "Aborted."; exit 0; }; \ + $(COMPOSE) down -v && sudo rm -rf $(DROP_PATHS) + +-include make/local.mk diff --git a/README.md b/README.md index 0257656..c451d11 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,7 @@ dokumentų URI visada sutampa su LDH adresu. Perkurti prieš krovimą nebūtina Duomenų struktūra kuriama dviem lygiais: - **Karkasas** (`make install`): šakninis dokumentas, konteineriai ir taksonomijų - schemos iš `app/` katalogo dokumentas po dokumento **per LDH CLI** (`put.sh`, + schemos iš `app/` katalogo **per LDH CLI** (`ldh push`, kaip [LinkedDataHub-Apps](https://github.com/AtomGraph/LinkedDataHub-Apps) projektuose) — taip dokumentai gauna `ldh:ChildrenView` bloką, dėl kurio konteinerių puslapiai rodo vaikų sąrašus. Asmenų konteineris @@ -196,12 +196,12 @@ Duomenų struktūra kuriama dviem lygiais: repozitorijos (`../LinkedDataHub`, keičiama per `make install LDH_HOME=…`). - **Duomenys** (`make load`): ETL rinkiniai — vien `dh:Item` dokumentai su `sioc:has_container` nuorodomis į karkasą — rašomi **tiesiogiai į - `fuseki-end-user` TDB2 saugyklą** (santykinės URI pirma išsprendžiamos pagal + `fuseki` serverio `end-user` TDB2 saugyklą** (santykinės URI pirma išsprendžiamos pagal `.env` bazę su `riot`, tada `tdb2.tdbloader` per vienkartinį `tdb-loader` konteinerį), ne po vieną dokumentą per HTTP: ~1 mln. ketvertų užsikrauna per kelias minutes. Pabaigoje suteikiama vieša skaitymo prieiga - (`make public` — LDH CLI `make-public.sh` atitikmuo, vykdomas tiesiogiai per - `fuseki-admin` konteinerių tinkle). + (`make public` — `ldh admin make-public`, kuri per LDH HTTP API + suteikia `acl:accessToClass` teises). Triplestore prievadai **neatveriami į host'ą** — SPARQL užklausos teikiamos per LDH: . Krovimas yra *append-only*: pakartotinis `make load` tik papildo saugyklą; švariam perkrovimui: diff --git a/bin/server-cert-gen.sh b/bin/server-cert-gen.sh index 93975ce..ce0707b 100755 --- a/bin/server-cert-gen.sh +++ b/bin/server-cert-gen.sh @@ -41,22 +41,20 @@ if [ -z "$(envProp "HOST")" ]; then echo "Configuration is incomplete: HOST is missing" exit 1 fi -if [ -z "$(envProp "ABS_PATH")" ]; then - echo "Configuration is incomplete: ABS_PATH is missing" - exit 1 -fi + +# a dataspace serves its documents from the root of its origin, so its base URI is the origin's root if [ "$(envProp "PROTOCOL")" = "https" ]; then if [ "$(envProp "HTTPS_PORT")" = 443 ]; then - base_uri="$(envProp "PROTOCOL")://$(envProp "HOST")$(envProp "ABS_PATH")" + base_uri="$(envProp "PROTOCOL")://$(envProp "HOST")/" else - base_uri="$(envProp "PROTOCOL")://$(envProp "HOST"):$(envProp "HTTPS_PORT")$(envProp "ABS_PATH")" + base_uri="$(envProp "PROTOCOL")://$(envProp "HOST"):$(envProp "HTTPS_PORT")/" fi else if [ "$(envProp "HTTP_PORT")" = 80 ]; then - base_uri="$(envProp "PROTOCOL")://$(envProp "HOST")$(envProp "ABS_PATH")" + base_uri="$(envProp "PROTOCOL")://$(envProp "HOST")/" else - base_uri="$(envProp "PROTOCOL")://$(envProp "HOST"):$(envProp "HTTP_PORT")$(envProp "ABS_PATH")" + base_uri="$(envProp "PROTOCOL")://$(envProp "HOST"):$(envProp "HTTP_PORT")/" fi fi diff --git a/config/fuseki/config.ttl b/config/fuseki/config.ttl index f4c5961..18764b0 100644 --- a/config/fuseki/config.ttl +++ b/config/fuseki/config.ttl @@ -6,12 +6,32 @@ PREFIX tdb2: [] a fuseki:Server . -<#service> a fuseki:Service ; - fuseki:name "ds" ; - fuseki:endpoint [ fuseki:operation fuseki:query; ] ; - fuseki:endpoint [ fuseki:operation fuseki:update;] ; - fuseki:endpoint [ fuseki:operation fuseki:gsp-rw; ] ; - fuseki:dataset <#dataset> . +# One dataset per dataspace role, named after the dataspace's origin with the deployment host dropped and the role +# appended. This deployment has only the root dataspace, so the names come out as plain end-user and admin. Each +# dataset is stored in its own folder under ./fuseki//DB2 - the folders the former fuseki-end-user and +# fuseki-admin containers already wrote, each of which mounted ./fuseki/ as /fuseki/databases and kept its +# data in DB2 there, so the switch to one server moves no data. -<#dataset> a tdb2:DatasetTDB2 ; - tdb2:location "/fuseki/databases/DB2" . \ No newline at end of file +# Queries are cancelled after 60 s, the Varnish backends' first_byte_timeout, past which no proxied client is still +# waiting for the answer. The timeout sits on the query endpoint: Fuseki honours ja:context there and on the +# dataset, but silently ignores it on the fuseki:Service. + +<#end-user> a fuseki:Service ; + fuseki:name "end-user" ; + fuseki:endpoint [ fuseki:operation fuseki:query ; ja:context [ ja:cxtName "arq:queryTimeout" ; ja:cxtValue "60000" ] ] ; + fuseki:endpoint [ fuseki:operation fuseki:update ] ; + fuseki:endpoint [ fuseki:operation fuseki:gsp-rw ] ; + fuseki:dataset <#end-user-dataset> . + +<#end-user-dataset> a tdb2:DatasetTDB2 ; + tdb2:location "/fuseki/databases/end-user/DB2" . + +<#admin> a fuseki:Service ; + fuseki:name "admin" ; + fuseki:endpoint [ fuseki:operation fuseki:query ; ja:context [ ja:cxtName "arq:queryTimeout" ; ja:cxtValue "60000" ] ] ; + fuseki:endpoint [ fuseki:operation fuseki:update ] ; + fuseki:endpoint [ fuseki:operation fuseki:gsp-rw ] ; + fuseki:dataset <#admin-dataset> . + +<#admin-dataset> a tdb2:DatasetTDB2 ; + tdb2:location "/fuseki/databases/admin/DB2" . diff --git a/config/system.trig b/config/system.trig index e0db930..1a4a45e 100644 --- a/config/system.trig +++ b/config/system.trig @@ -22,9 +22,9 @@ a sd:Service ; dct:title "LTLOD admin service" ; sd:supportedLanguage sd:SPARQL11Query, sd:SPARQL11Update ; - sd:endpoint ; - a:graphStore ; - a:quadStore . + sd:endpoint ; + a:graphStore ; + a:quadStore . } @@ -44,8 +44,8 @@ a sd:Service ; dct:title "LTLOD service" ; sd:supportedLanguage sd:SPARQL11Query, sd:SPARQL11Update ; - sd:endpoint ; - a:graphStore ; - a:quadStore . + sd:endpoint ; + a:graphStore ; + a:quadStore . } diff --git a/docker-compose.override.yml b/docker-compose.override.yml index b888f89..65e6a25 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -1,6 +1,6 @@ services: linkeddatahub: - # image: atomgraph/linkeddatahub:fe7106d14e022a9eb709fbe1b3beb60496f7d177 + image: atomgraph/linkeddatahub:latest environment: - TZ="Europe/Vilnius" - ENABLE_WEBID_SIGNUP=false # disable public WebID signup @@ -16,7 +16,8 @@ services: - ./files/overrides.xsl:/usr/local/tomcat/webapps/ROOT/static/xsl/overrides.xsl:ro - ./files/client.xsl:/usr/local/tomcat/webapps/ROOT/static/lt/linkeddata/xsl/client.xsl:ro - ./files/client.xsl.sef.json:/usr/local/tomcat/webapps/ROOT/static/lt/linkeddata/xsl/client.xsl.sef.json:ro - # one-off bulk loader: `make load` runs it with fuseki-end-user stopped. + # one-off bulk loader: `make load` runs it with fuseki stopped. It mounts ./fuseki/end-user + # itself, so its --loc /fuseki/databases/DB2 still writes the end-user dataset's TDB2 folder. # The fuseki image bundles the full Jena CLI inside the fuseki-server jar. # Committed TriG is base-relative with no @base; resolve it against the # deployment base (BASE_URI, injected by `make load` from .env) with riot @@ -37,3 +38,8 @@ services: --quiet --base="$${BASE_URI:?BASE_URI not set}" --syntax=trig --output=nquads /staging/*/*.trig > /tmp/staging.nq; exec "$${JAVA_HOME}/bin/java" -Xmx3g -cp "/fuseki/$${FUSEKI_JAR}" tdb2.tdbloader --loc /fuseki/databases/DB2 /tmp/staging.nq + # The base defines the service; this repo has no Dockerfile, so its `build:` cannot apply. Pin an + # image built from the same source tree as the platform image above, so the sef-compiler's copy of + # the static tree matches what is served. + sef-compiler: + image: atomgraph/linkeddatahub-sef-compiler:core-5.0.4-treefix diff --git a/docker-compose.yml b/docker-compose.yml index 138139b..ff9d76b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,11 +36,12 @@ services: - ./ssl/server:/etc/nginx/ssl:ro linkeddatahub: user: root # otherwise the ldh user does not have permissions to the mounted folder which is owner by root - image: atomgraph/linkeddatahub:5.6.0 + build: . mem_limit: 2048m depends_on: - - fuseki-admin - - fuseki-end-user + - fuseki + - sef-compiler + - egress environment: # - JPDA_ADDRESS=*:8000 # debugger host - performance hit when enabled - CATALINA_OPTS=-XX:+UseContainerSupport -XX:MaxRAMPercentage=75 --add-exports java.base/sun.security.tools.keytool=ALL-UNNAMED # heap will use up to 75% of container's RAM @@ -53,7 +54,6 @@ services: - BACKEND_PROXY_END_USER=http://varnish-end-user/ - PROTOCOL=${PROTOCOL} - HOST=${HOST} - - ABS_PATH=${ABS_PATH} - HTTP_SCHEME=https - HTTP_PORT=7070 - HTTP_PROXY_NAME=${HOST} @@ -65,6 +65,8 @@ services: - SIGN_UP_CERT_VALIDITY=180 - MAX_CONTENT_LENGTH=${MAX_CONTENT_LENGTH:-2097152} - ALLOW_INTERNAL_URLS=${ALLOW_INTERNAL_URLS:-} + - EGRESS_PROXY=${EGRESS_PROXY:-egress:3128} # the platform's own SPARQL SERVICE (PATCH, imports) goes through egress too + - CONNECTION_REQUEST_TIMEOUT=${CONNECTION_REQUEST_TIMEOUT:-} - NOTIFICATION_ADDRESS=LinkedDataHub - MAIL_SMTP_HOST=email-server @@ -101,35 +103,52 @@ services: - ./datasets/owner:/var/linkeddatahub/datasets/owner - ./datasets/secretary:/var/linkeddatahub/datasets/secretary - ./uploads:/var/www/linkeddatahub/uploads + - ./sef:/var/www/linkeddatahub/sef + - ./packages:/var/www/linkeddatahub/packages + - ./settings:/var/www/linkeddatahub/settings - ./config/dev.log4j.properties:/usr/local/tomcat/webapps/ROOT/WEB-INF/classes/log4j.properties:ro - ./config/dataspaces.trig:/var/linkeddatahub/datasets/dataspaces.trig - ./config/system.trig:/var/linkeddatahub/datasets/system.trig - fuseki-admin: + sef-compiler: + # composes each dataspace's client stylesheet with its package stylesheets and compiles + # the result to a SEF. Built from the same Dockerfile as the platform, so its copy of the + # static tree is the deployed one by construction + build: + context: . + target: sef-compiler + mem_limit: 3072m # the compile peaks around 1.5 GB; it must not share the platform's limit + restart: on-failure + expose: + - 8080 # internal only - the platform is the sole client + fuseki: # one server; every dataspace role is its own dataset in config/fuseki/config.ttl, stored under ./fuseki/ image: atomgraph/fuseki:6.1.0 user: root # otherwise fuseki user does not have permissions to the mounted folder which is owner by root - mem_limit: 1536m # leave headroom above heap for TDB mmap/native memory + mem_limit: 4608m # the former fuseki-admin (1536m) and fuseki-end-user (3072m) combined restart: on-failure + depends_on: + - egress environment: - - JAVA_OPTIONS=-Xmx768m -Xms768m + - JAVA_OPTIONS=-Xmx2304m -Xms2304m # the former heaps combined: 768m admin + 1536m end-user + # every outbound request (SPARQL SERVICE, LOAD) goes through egress, which only lets it reach public addresses. + # JAVA_TOOL_OPTIONS rather than JAVA_OPTIONS, which an override setting the heap would replace. The empty + # nonProxyHosts is load-bearing: by default the JVM bypasses the proxy for localhost, and + # SERVICE would read a sibling dataset in this same server + - JAVA_TOOL_OPTIONS=-Dhttp.proxyHost=${EGRESS_PROXY_HOST:-egress} -Dhttp.proxyPort=3128 -Dhttps.proxyHost=${EGRESS_PROXY_HOST:-egress} -Dhttps.proxyPort=3128 -Dhttp.nonProxyHosts= expose: - 3030 volumes: - ./config/fuseki/config.ttl:/fuseki/config.ttl:ro - - ./fuseki/admin:/fuseki/databases + - ./fuseki:/fuseki/databases command: [ "--config", "/fuseki/config.ttl" ] - fuseki-end-user: - image: atomgraph/fuseki:6.1.0 - user: root # otherwise the fuseki user does not have permissions to the mounted folder which is owner by root - mem_limit: 3072m # leave headroom above heap for TDB mmap/native memory + egress: # forward proxy for the triplestores' outbound requests: refuses loopback, private and link-local destinations + image: ubuntu/squid:6.6-24.04_beta + mem_limit: 256m restart: on-failure - environment: - - JAVA_OPTIONS=-Xmx1536m -Xms1536m + configs: + - source: egress_squid_conf + target: /etc/squid/squid.conf expose: - - 3030 - volumes: - - ./config/fuseki/config.ttl:/fuseki/config.ttl:ro - - ./fuseki/end-user:/fuseki/databases - command: [ "--config", "/fuseki/config.ttl" ] + - 3128 varnish-frontend: image: varnish:7.7.3 user: root # otherwise varnish user does not have permissions to the mounted folder which is owner by root @@ -216,13 +235,13 @@ configs: add_header Access-Control-Allow-Origin "*" always; add_header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, PATCH, HEAD, OPTIONS" always; add_header Access-Control-Allow-Headers "Accept, Content-Type, Authorization" always; - add_header Access-Control-Expose-Headers "Link, Content-Location, Location" always; + add_header Access-Control-Expose-Headers "Link, Content-Location, Location, Memento-Datetime" always; if ($$request_method = OPTIONS) { add_header Access-Control-Allow-Origin "*"; add_header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, PATCH, HEAD, OPTIONS"; add_header Access-Control-Allow-Headers "Accept, Content-Type, Authorization"; - add_header Access-Control-Expose-Headers "Link, Content-Location, Location"; + add_header Access-Control-Expose-Headers "Link, Content-Location, Location, Memento-Datetime"; add_header Access-Control-Max-Age "1728000"; return 204; } @@ -288,13 +307,13 @@ configs: add_header Access-Control-Allow-Origin "*" always; add_header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, PATCH, HEAD, OPTIONS" always; add_header Access-Control-Allow-Headers "Accept, Content-Type, Authorization" always; - add_header Access-Control-Expose-Headers "Link, Content-Location, Location" always; + add_header Access-Control-Expose-Headers "Link, Content-Location, Location, Memento-Datetime" always; if ($$request_method = OPTIONS) { add_header Access-Control-Allow-Origin "*"; add_header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, PATCH, HEAD, OPTIONS"; add_header Access-Control-Allow-Headers "Accept, Content-Type, Authorization"; - add_header Access-Control-Expose-Headers "Link, Content-Location, Location"; + add_header Access-Control-Expose-Headers "Link, Content-Location, Location, Memento-Datetime"; add_header Access-Control-Max-Age "1728000"; return 204; } @@ -462,7 +481,7 @@ configs: import xkey; backend default { - .host = "${VARNISH_ADMIN_BACKEND_HOST:-fuseki-admin}"; + .host = "${VARNISH_ADMIN_BACKEND_HOST:-fuseki}"; .port = "${VARNISH_ADMIN_BACKEND_PORT:-3030}"; .first_byte_timeout = 60s; } @@ -529,6 +548,10 @@ configs: if ((beresp.status == 200 || beresp.status == 201 || beresp.status == 204) && bereq.method ~ "POST|PUT|DELETE|PATCH") { set beresp.http.X-LinkedDataHub = "Banned"; ban("req.url == " + bereq.url + " && req.http.host == " + bereq.http.host); + /* SPARQL query results depend on dataset state — any write invalidates every cached query. xkey below + covers only the responses LDH stamps (ontology CONSTRUCTs); a client SELECT carries no key, so + without this it keeps its pre-write answer for the whole 24h TTL. */ + ban("req.url ~ \?(query|default-graph-uri|named-graph-uri)="); } /* promote outbound surrogate-key hint from LDH into the indexed xkey header on the cached response */ @@ -545,7 +568,7 @@ configs: import std; backend default { - .host = "${VARNISH_END_USER_BACKEND_HOST:-fuseki-end-user}"; + .host = "${VARNISH_END_USER_BACKEND_HOST:-fuseki}"; .port = "${VARNISH_END_USER_BACKEND_PORT:-3030}"; .first_byte_timeout = 60s; } @@ -606,3 +629,16 @@ configs: return (deliver); } + egress_squid_conf: + content: | + # The triplestores reach the outside world only through here, so a SPARQL SERVICE or LOAD can federate with public + # endpoints but cannot reach this deployment's own services - the admin store, Varnish, the platform - or the + # host's and cloud's internal addresses. dst resolves the host where the connection is made, so every redirect + # hop and every DNS answer is checked, not just the IRI written in the query. + http_port 3128 + + acl internal dst 0.0.0.0/8 10.0.0.0/8 100.64.0.0/10 127.0.0.0/8 169.254.0.0/16 172.16.0.0/12 192.168.0.0/16 ::1 fc00::/7 fe80::/10 + http_access deny internal + http_access allow all + + cache deny all diff --git a/make/config.mk b/make/config.mk new file mode 100644 index 0000000..4f35697 --- /dev/null +++ b/make/config.mk @@ -0,0 +1,15 @@ +# Settings for the canonical Makefile. + +# install is this deployment's own: the canonical Makefile carries no install target +LOCAL_TARGETS := install + +# files/client.xsl imports files/overrides.xsl, so both are staged for the SEF compile +SEF_EXTRA := files/overrides.xsl + +# `make load` bulk-loads the ETL output straight into the end-user dataset +LOAD_STAGING := datasets/current + +# never wipe datasets/current - that is ETL output, not LDH runtime state +DROP_PATHS := datasets/owner datasets/secretary fuseki ssl secrets uploads sef packages settings + +include etl/config.mk # for JENA_HOME (its BASE is unused here) diff --git a/make/local.mk b/make/local.mk new file mode 100644 index 0000000..504e34d --- /dev/null +++ b/make/local.mk @@ -0,0 +1,41 @@ +# Deployment-specific targets. + +# Install the app structure (root + containers + the namespace ontology and its views) onto a +# LinkedDataHub instance through the ldh CLI. Interactive, with defaults from the local stack +# (.env, ssl/, secrets/): press Enter to install locally, or give another base URL and owner +# certificate to install on any LDH instance. Re-running is safe (PUT replaces). +# Local order: make up -> make install -> make load. +install: + @[ -x "$(LDH)" ] && [ -f "$(LDH_HOME)/cli/target/ldh.jar" ] || \ + { echo "ERROR: ldh CLI not found - clone https://github.com/AtomGraph/LinkedDataHub to $(LDH_HOME) (or pass LDH_HOME=...) and run 'make cli' there"; exit 1; } + @read -p "Enter Base URL [$(BASE_URI)]: " BASE_URL; \ + BASE_URL=$${BASE_URL:-$(BASE_URI)}; \ + read -p "Enter Certificate Path [$(OWNER_CERT)]: " CERT_PATH; \ + CERT_PATH=$${CERT_PATH:-$(OWNER_CERT)}; \ + [ -f "$$CERT_PATH" ] || { echo "ERROR: certificate not found: $$CERT_PATH"; exit 1; }; \ + PW_DEFAULT=""; \ + [ -f $(OWNER_PASSWORD_FILE) ] && PW_DEFAULT="$$(cat $(OWNER_PASSWORD_FILE))"; \ + if [ -n "$$PW_DEFAULT" ]; then \ + read -r -s -p "Enter Certificate Password [from $(OWNER_PASSWORD_FILE)]: " PASSWORD; \ + else \ + read -r -s -p "Enter Certificate Password (required): " PASSWORD; \ + fi; \ + echo ""; \ + PASSWORD=$${PASSWORD:-$$PW_DEFAULT}; \ + if [ -z "$$PASSWORD" ]; then echo "Password cannot be empty. Aborting."; exit 1; fi; \ + PROXY_DEFAULT=""; \ + [ "$$BASE_URL" = "$(BASE_URI)" ] && PROXY_DEFAULT="$(PROXY_URI)"; \ + read -p "Enter Proxy URL (optional) [$$PROXY_DEFAULT]: " PROXY_URL; \ + PROXY_URL=$${PROXY_URL:-$$PROXY_DEFAULT}; \ + if [ "$$BASE_URL" = "$(BASE_URI)" ] && [ -n "$$($(COMPOSE) ps -q linkeddatahub 2>/dev/null)" ]; then \ + echo "Waiting for LinkedDataHub health (first-boot seeding must finish)..."; \ + until [ "$$(docker inspect -f '{{.State.Health.Status}}' $$($(COMPOSE) ps -q linkeddatahub))" = "healthy" ]; do \ + sleep 5; echo " ...waiting"; \ + done; \ + fi; \ + export PATH="$$(cd $(LDH_HOME) && pwd)/cli/bin:$$PATH"; \ + if [ -n "$$PROXY_URL" ]; then \ + ./app/install.sh "$$BASE_URL" "$$CERT_PATH" "$$PASSWORD" "$$PROXY_URL"; \ + else \ + ./app/install.sh "$$BASE_URL" "$$CERT_PATH" "$$PASSWORD"; \ + fi