From 001de25862616525feba71589e5b71fceb07e15b Mon Sep 17 00:00:00 2001 From: Juraj Roka <95219754+jr-rk@users.noreply.github.com> Date: Tue, 11 Aug 2026 16:02:46 +0200 Subject: [PATCH 1/2] feat(ssr): add configurable crawler policy Crawlers enumerating Discovery facet links produced an unbounded set of distinct URLs; the URL-keyed SSR cache never hit and the render workers saturated, so the site answered HTTP 504 for everyone. - robots.txt: always disallow scoped community/collection search and every facet URL (?f. / &f.). Upstream "Disallow: /search" only matches paths that start with /search, so these stayed crawlable. - SSR: exclude the scoped-search paths from rendering via universal.excludePathPatterns (config.yml, config.example.yml, and the three build-time environment defaults) so they serve a CSR shell. - New robots config section (disallowHandle / disallowBrowse / disallowBitstreams), all default false, switchable per instance via config.yml or DSPACE_ROBOTS_* env vars with no template edit. Ports and generalizes mendelu 29b88bf36b (adapted from 9.x ssr. to this branch's universal. key), TUL 0eaea26586, ZCU-PUB 11eb4d0f6e, VSB-TUO acae3b4532. Refs dataquest-dev/dspace-customers#902 Co-Authored-By: Claude Fable 5 --- config/config.example.yml | 20 ++++++++++++++++ config/config.yml | 17 ++++++++++++++ server.ts | 8 +++++-- src/config/app-config.interface.ts | 2 ++ src/config/default-app-config.ts | 10 ++++++++ src/config/robots-config.interface.ts | 27 ++++++++++++++++++++++ src/environments/environment.production.ts | 9 ++++++++ src/environments/environment.test.ts | 14 +++++++++++ src/environments/environment.ts | 9 ++++++++ src/robots.txt.ejs | 21 +++++++++++++++++ 10 files changed, 135 insertions(+), 2 deletions(-) create mode 100644 src/config/robots-config.interface.ts diff --git a/config/config.example.yml b/config/config.example.yml index 8b56711c7d2..16368b4c105 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -31,6 +31,14 @@ universal: flag: "i" - pattern: "^/collections/[a-f0-9-]{36}/browse(/.*)?$" flag: "i" + # Scoped search inside a community/collection is the most expensive route, and its facet + # links are effectively unbounded; crawlers walking them saturated SSR in production + # (HTTP 504). Serve these as a plain CSR shell. src/robots.txt.ejs asks well-behaved + # crawlers to stay out of the same paths; this handles the ones that ignore robots.txt. + - pattern: "^/communities/[a-f0-9-]{36}/search(/.*)?$" + flag: "i" + - pattern: "^/collections/[a-f0-9-]{36}/search(/.*)?$" + flag: "i" - pattern: "^/browse/" - pattern: "^/search$" - pattern: "^/community-list$" @@ -60,6 +68,18 @@ universal: # Disable this setting to avoid URL replacement during SSR. In this the state is not transferred to avoid security issues. replaceRestUrl: true +# Optional per-instance blocks of the served /robots.txt (src/robots.txt.ejs). +# The Discovery facet-trap rules are always emitted; these switch the extra blocks. +# All default false because each de-indexes content some instances intentionally +# expose (a blanket /handle block breaks instances relying on handle redirects). +robots: + # Disallow: /handle + disallowHandle: false + # Disallow: /browse + disallowBrowse: false + # Disallow: /bitstream/ and /bitstreams/ + disallowBitstreams: false + # The REST API server settings # NOTE: these settings define which (publicly available) REST API to use. They are usually # 'synced' with the 'dspace.server.url' setting in your backend's local.cfg. diff --git a/config/config.yml b/config/config.yml index 9257bd2b09d..bcf6a3dd01c 100644 --- a/config/config.yml +++ b/config/config.yml @@ -19,6 +19,14 @@ universal: flag: "i" - pattern: "^/collections/[a-f0-9-]{36}/browse(/.*)?$" flag: "i" + # Scoped search inside a community/collection is the most expensive route, and its facet + # links are effectively unbounded; crawlers walking them saturated SSR in production + # (HTTP 504). Serve these as a plain CSR shell. src/robots.txt.ejs asks well-behaved + # crawlers to stay out of the same paths; this handles the ones that ignore robots.txt. + - pattern: "^/communities/[a-f0-9-]{36}/search(/.*)?$" + flag: "i" + - pattern: "^/collections/[a-f0-9-]{36}/search(/.*)?$" + flag: "i" - pattern: "^/browse/" - pattern: "^/search$" - pattern: "^/community-list$" @@ -38,6 +46,15 @@ universal: # If set to false the component will not be included in the HTML returned from the server side rendering. enableBrowseComponent: false +# Optional per-instance blocks of the served /robots.txt (src/robots.txt.ejs). +# The Discovery facet-trap rules are always emitted; these switch the extra blocks. +# All default false because each de-indexes content some instances intentionally +# expose (a blanket /handle block breaks instances relying on handle redirects). +robots: + disallowHandle: false + disallowBrowse: false + disallowBitstreams: false + # Caching settings cache: # NOTE: how long should objects be cached for by default diff --git a/server.ts b/server.ts index e7b13aef522..ce8d9c6aa3e 100644 --- a/server.ts +++ b/server.ts @@ -162,12 +162,16 @@ export function app() { server.set('view engine', 'ejs'); /** - * Serve the robots.txt ejs template, filling in the origin variable + * Serve the robots.txt ejs template, filling in the origin variable and the + * per-instance optional-block toggles (see config `robots`). */ server.get('/robots.txt', (req, res) => { res.setHeader('content-type', 'text/plain'); res.render('assets/robots.txt.ejs', { - 'origin': req.protocol + '://' + req.headers.host + 'origin': req.protocol + '://' + req.headers.host, + 'disallowHandle': environment.robots.disallowHandle, + 'disallowBrowse': environment.robots.disallowBrowse, + 'disallowBitstreams': environment.robots.disallowBitstreams }); }); diff --git a/src/config/app-config.interface.ts b/src/config/app-config.interface.ts index 96a6f91f8ec..67357d94033 100644 --- a/src/config/app-config.interface.ts +++ b/src/config/app-config.interface.ts @@ -27,6 +27,7 @@ import { SearchConfig } from './search-page-config.interface'; import { AccessibilitySettingsConfig } from '../app/accessibility/accessibility-settings.config'; import { MatomoConfig } from './matomo-config'; import { StatisticsConfig } from './statistics-config'; +import { RobotsConfig } from './robots-config.interface'; interface AppConfig extends Config { ui: UIServerConfig; @@ -59,6 +60,7 @@ interface AppConfig extends Config { signpostingEnabled: boolean; matomo: MatomoConfig; statistics?: StatisticsConfig; + robots: RobotsConfig; } /** diff --git a/src/config/default-app-config.ts b/src/config/default-app-config.ts index 38cb942e457..3fce69d7e6d 100644 --- a/src/config/default-app-config.ts +++ b/src/config/default-app-config.ts @@ -26,6 +26,7 @@ import { LiveRegionConfig } from '../app/shared/live-region/live-region.config'; import { SearchConfig } from './search-page-config.interface'; import { AccessibilitySettingsConfig } from '../app/accessibility/accessibility-settings.config'; import { MatomoConfig } from './matomo-config'; +import { RobotsConfig } from './robots-config.interface'; export class DefaultAppConfig implements AppConfig { production = false; @@ -415,6 +416,15 @@ export class DefaultAppConfig implements AppConfig { mathjax: false, }; + // Optional per-instance blocks of the served robots.txt. The facet-trap rules + // that protect SSR are always emitted; these switch the extra blocks that + // de-index content some instances intentionally expose. + robots: RobotsConfig = { + disallowHandle: false, + disallowBrowse: false, + disallowBitstreams: false, + }; + // Which vocabularies should be used for which search filters // and whether to show the filter in the search sidebar // Take a look at the filter-vocabulary-config.ts file for documentation on how the options are obtained diff --git a/src/config/robots-config.interface.ts b/src/config/robots-config.interface.ts new file mode 100644 index 00000000000..dad67dc79e5 --- /dev/null +++ b/src/config/robots-config.interface.ts @@ -0,0 +1,27 @@ +import { Config } from './config.interface'; + +/** + * Config for the optional, per-instance blocks of the served `robots.txt` + * (see `src/robots.txt.ejs`). The facet-trap rules that protect SSR are always + * emitted; only these extra blocks are switchable, because each de-indexes + * content some instances intentionally expose. + */ +export interface RobotsConfig extends Config { + + /** + * Emit `Disallow: /handle`. Off by default: a blanket handle block de-indexes + * persistent identifiers on instances that rely on handle redirects. + */ + disallowHandle: boolean; + + /** + * Emit `Disallow: /browse`. + */ + disallowBrowse: boolean; + + /** + * Emit `Disallow: /bitstream/` and `Disallow: /bitstreams/` to keep crawlers + * out of bitstream content (reachable by handle path or by UUID). + */ + disallowBitstreams: boolean; +} diff --git a/src/environments/environment.production.ts b/src/environments/environment.production.ts index e5066ab48fc..79ddbd47859 100644 --- a/src/environments/environment.production.ts +++ b/src/environments/environment.production.ts @@ -20,6 +20,15 @@ export const environment: Partial = { pattern: '^/collections/[a-f0-9-]{36}/browse(/.*)?$', flag: 'i', }, + // Scoped search + its facet links: the crawler trap that saturated SSR in production. + { + pattern: '^/communities/[a-f0-9-]{36}/search(/.*)?$', + flag: 'i', + }, + { + pattern: '^/collections/[a-f0-9-]{36}/search(/.*)?$', + flag: 'i', + }, { pattern: '^/browse/' }, { pattern: '^/search' }, { pattern: '^/community-list$' }, diff --git a/src/environments/environment.test.ts b/src/environments/environment.test.ts index a0cab16ef83..6abbfa7d469 100644 --- a/src/environments/environment.test.ts +++ b/src/environments/environment.test.ts @@ -23,6 +23,15 @@ export const environment: BuildConfig = { pattern: '^/collections/[a-f0-9-]{36}/browse(/.*)?$', flag: 'i', }, + // Scoped search + its facet links: the crawler trap that saturated SSR in production. + { + pattern: '^/communities/[a-f0-9-]{36}/search(/.*)?$', + flag: 'i', + }, + { + pattern: '^/collections/[a-f0-9-]{36}/search(/.*)?$', + flag: 'i', + }, { pattern: '^/browse/' }, { pattern: '^/search' }, { pattern: '^/community-list$' }, @@ -327,6 +336,11 @@ export const environment: BuildConfig = { enabled: false, mathjax: false, }, + robots: { + disallowHandle: false, + disallowBrowse: false, + disallowBitstreams: false, + }, comcolSelectionSort: { sortField:'dc.title', sortDirection:'ASC', diff --git a/src/environments/environment.ts b/src/environments/environment.ts index 8be4ee7dbfe..3ef855a57c7 100644 --- a/src/environments/environment.ts +++ b/src/environments/environment.ts @@ -25,6 +25,15 @@ export const environment: Partial = { pattern: '^/collections/[a-f0-9-]{36}/browse(/.*)?$', flag: 'i', }, + // Scoped search + its facet links: the crawler trap that saturated SSR in production. + { + pattern: '^/communities/[a-f0-9-]{36}/search(/.*)?$', + flag: 'i', + }, + { + pattern: '^/collections/[a-f0-9-]{36}/search(/.*)?$', + flag: 'i', + }, { pattern: '^/browse/' }, { pattern: '^/search' }, { pattern: '^/community-list$' }, diff --git a/src/robots.txt.ejs b/src/robots.txt.ejs index f6dc7fa5cb5..767ec8ea223 100644 --- a/src/robots.txt.ejs +++ b/src/robots.txt.ejs @@ -18,6 +18,27 @@ Disallow: /profile Disallow: /workflowitems # Crawlers should be able to access entity pages, but not the facet search links present on entity pages Disallow: /entities/*?f +# "Disallow: /search" above only matches paths that START with /search, so scoped +# search inside a community or collection stayed crawlable. Enumerating its facet +# links saturated SSR in production (HTTP 504); block it and every facet URL. +Disallow: /collections/*/search +Disallow: /communities/*/search +# Any URL carrying a Discovery facet filter (f.author, f.subject, ...). Two rules +# because the facet can be the first query parameter (?f.) or a later one (&f.), +# and robots.txt cannot express "either". +Disallow: /*?f. +Disallow: /*&f. +<% if (disallowBrowse) { -%> +Disallow: /browse +<% } -%> +<% if (disallowHandle) { -%> +Disallow: /handle +<% } -%> +<% if (disallowBitstreams) { -%> +# Bitstream content is reachable by handle path and by UUID; block both. +Disallow: /bitstream/ +Disallow: /bitstreams/ +<% } -%> # Optionally uncomment the following line ONLY if sitemaps are working # and you have verified that your site is being indexed correctly. From e82c952ac62891f079d16f5b5969423f894e8372 Mon Sep 17 00:00:00 2001 From: Juraj Roka <95219754+jr-rk@users.noreply.github.com> Date: Wed, 12 Aug 2026 09:08:22 +0200 Subject: [PATCH 2/2] fix(ssr): drop dead scoped-search patterns, add robots.txt test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copilot review (and an independent verification pass) found that the two scoped-search SSR excludes and their matching robots.txt lines never match anything on this 7.6.5 branch: collection/community routing has no `:id/search` child route here (that shape is DSpace 9.x, where the mendelu source commit came from), and scoped search on 7.6.5 is `/search?scope=&f.*` — already covered by the pre-existing `Disallow: /search` and `^/search` SSR exclusion. - Remove the dead `^/(communities|collections)/[uuid]/search` patterns from config.yml, config.example.yml, environment.ts, environment.production.ts, environment.test.ts, and the matching dead robots.txt Disallow lines. - Keep the effective, new parts: the generic facet trap (Disallow: /*?f. and /*&f.) and the three opt-in toggles — those are unaffected by the routing question. - Extract the toggle-line assembly into buildOptionalRobotsDisallows() (src/config/robots.util.ts), rendered as a single EJS local from server.ts, and add a unit test (robots.util.spec.ts) covering each toggle and the no-blank-line-in-group invariant Copilot flagged as untested. Not applied: rewriting isExcludedFromSsr to inspect facet query keys. The documented incident (scoped-search facet enumeration) is already covered by the existing /search SSR exclusion; changing core SSR routing logic for every request is out of scope for this fix. Co-Authored-By: Claude Fable 5 --- config/config.example.yml | 8 ----- config/config.yml | 8 ----- server.ts | 7 ++-- src/config/robots.util.spec.ts | 40 ++++++++++++++++++++++ src/config/robots.util.ts | 27 +++++++++++++++ src/environments/environment.production.ts | 9 ----- src/environments/environment.test.ts | 9 ----- src/environments/environment.ts | 9 ----- src/robots.txt.ejs | 25 +++----------- 9 files changed, 75 insertions(+), 67 deletions(-) create mode 100644 src/config/robots.util.spec.ts create mode 100644 src/config/robots.util.ts diff --git a/config/config.example.yml b/config/config.example.yml index 16368b4c105..508aba90298 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -31,14 +31,6 @@ universal: flag: "i" - pattern: "^/collections/[a-f0-9-]{36}/browse(/.*)?$" flag: "i" - # Scoped search inside a community/collection is the most expensive route, and its facet - # links are effectively unbounded; crawlers walking them saturated SSR in production - # (HTTP 504). Serve these as a plain CSR shell. src/robots.txt.ejs asks well-behaved - # crawlers to stay out of the same paths; this handles the ones that ignore robots.txt. - - pattern: "^/communities/[a-f0-9-]{36}/search(/.*)?$" - flag: "i" - - pattern: "^/collections/[a-f0-9-]{36}/search(/.*)?$" - flag: "i" - pattern: "^/browse/" - pattern: "^/search$" - pattern: "^/community-list$" diff --git a/config/config.yml b/config/config.yml index bcf6a3dd01c..2c59ae2e0d7 100644 --- a/config/config.yml +++ b/config/config.yml @@ -19,14 +19,6 @@ universal: flag: "i" - pattern: "^/collections/[a-f0-9-]{36}/browse(/.*)?$" flag: "i" - # Scoped search inside a community/collection is the most expensive route, and its facet - # links are effectively unbounded; crawlers walking them saturated SSR in production - # (HTTP 504). Serve these as a plain CSR shell. src/robots.txt.ejs asks well-behaved - # crawlers to stay out of the same paths; this handles the ones that ignore robots.txt. - - pattern: "^/communities/[a-f0-9-]{36}/search(/.*)?$" - flag: "i" - - pattern: "^/collections/[a-f0-9-]{36}/search(/.*)?$" - flag: "i" - pattern: "^/browse/" - pattern: "^/search$" - pattern: "^/community-list$" diff --git a/server.ts b/server.ts index ce8d9c6aa3e..e5f3ebcab8d 100644 --- a/server.ts +++ b/server.ts @@ -56,6 +56,7 @@ import { extendEnvironmentWithAppConfig } from './src/config/config.util'; import { logStartupMessage } from './startup-message'; import { TOKENITEM } from './src/app/core/auth/models/auth-token-info.model'; import { SsrExcludePatterns } from './src/config/universal-config.interface'; +import { buildOptionalRobotsDisallows } from './src/config/robots.util'; /* @@ -163,15 +164,13 @@ export function app() { /** * Serve the robots.txt ejs template, filling in the origin variable and the - * per-instance optional-block toggles (see config `robots`). + * per-instance optional Disallow blocks (see config `robots`). */ server.get('/robots.txt', (req, res) => { res.setHeader('content-type', 'text/plain'); res.render('assets/robots.txt.ejs', { 'origin': req.protocol + '://' + req.headers.host, - 'disallowHandle': environment.robots.disallowHandle, - 'disallowBrowse': environment.robots.disallowBrowse, - 'disallowBitstreams': environment.robots.disallowBitstreams + 'optionalDisallows': buildOptionalRobotsDisallows(environment.robots) }); }); diff --git a/src/config/robots.util.spec.ts b/src/config/robots.util.spec.ts new file mode 100644 index 00000000000..ee0e884801d --- /dev/null +++ b/src/config/robots.util.spec.ts @@ -0,0 +1,40 @@ +import { buildOptionalRobotsDisallows } from './robots.util'; +import { RobotsConfig } from './robots-config.interface'; + +describe('buildOptionalRobotsDisallows', () => { + const allOff: RobotsConfig = { + disallowHandle: false, + disallowBrowse: false, + disallowBitstreams: false, + }; + + it('emits nothing when every toggle is off', () => { + expect(buildOptionalRobotsDisallows(allOff)).toBe(''); + }); + + it('emits only /browse when disallowBrowse is on', () => { + expect(buildOptionalRobotsDisallows({ ...allOff, disallowBrowse: true })) + .toBe('\nDisallow: /browse'); + }); + + it('emits only /handle when disallowHandle is on', () => { + expect(buildOptionalRobotsDisallows({ ...allOff, disallowHandle: true })) + .toBe('\nDisallow: /handle'); + }); + + it('emits both bitstream rules when disallowBitstreams is on', () => { + const out = buildOptionalRobotsDisallows({ ...allOff, disallowBitstreams: true }); + expect(out).toContain('Disallow: /bitstream/'); + expect(out).toContain('Disallow: /bitstreams/'); + }); + + it('leads with a newline and never a blank line, so the block stays inside the group', () => { + const out = buildOptionalRobotsDisallows({ + disallowHandle: true, + disallowBrowse: true, + disallowBitstreams: true, + }); + expect(out.startsWith('\n')).toBeTrue(); + expect(out).not.toContain('\n\n'); + }); +}); diff --git a/src/config/robots.util.ts b/src/config/robots.util.ts new file mode 100644 index 00000000000..acea906f58b --- /dev/null +++ b/src/config/robots.util.ts @@ -0,0 +1,27 @@ +import { RobotsConfig } from './robots-config.interface'; + +/** + * Build the optional, per-instance `Disallow` lines for the served robots.txt + * (see `src/robots.txt.ejs`) from config. The always-on facet-trap rules live in + * the template itself; only these switchable blocks are assembled here so the + * toggle logic has a unit-testable seam. + * + * Returns a string that already starts with a newline when non-empty, so it can + * be appended directly after the last static rule without leaving a blank line + * inside the `User-agent: *` group record (blank lines end a group). + */ +export function buildOptionalRobotsDisallows(robots: RobotsConfig): string { + const lines: string[] = []; + if (robots?.disallowBrowse) { + lines.push('Disallow: /browse'); + } + if (robots?.disallowHandle) { + lines.push('Disallow: /handle'); + } + if (robots?.disallowBitstreams) { + // Bitstream content is reachable by handle path and by UUID; block both. + lines.push('Disallow: /bitstream/'); + lines.push('Disallow: /bitstreams/'); + } + return lines.length ? '\n' + lines.join('\n') : ''; +} diff --git a/src/environments/environment.production.ts b/src/environments/environment.production.ts index 79ddbd47859..e5066ab48fc 100644 --- a/src/environments/environment.production.ts +++ b/src/environments/environment.production.ts @@ -20,15 +20,6 @@ export const environment: Partial = { pattern: '^/collections/[a-f0-9-]{36}/browse(/.*)?$', flag: 'i', }, - // Scoped search + its facet links: the crawler trap that saturated SSR in production. - { - pattern: '^/communities/[a-f0-9-]{36}/search(/.*)?$', - flag: 'i', - }, - { - pattern: '^/collections/[a-f0-9-]{36}/search(/.*)?$', - flag: 'i', - }, { pattern: '^/browse/' }, { pattern: '^/search' }, { pattern: '^/community-list$' }, diff --git a/src/environments/environment.test.ts b/src/environments/environment.test.ts index 6abbfa7d469..31dad7e336f 100644 --- a/src/environments/environment.test.ts +++ b/src/environments/environment.test.ts @@ -23,15 +23,6 @@ export const environment: BuildConfig = { pattern: '^/collections/[a-f0-9-]{36}/browse(/.*)?$', flag: 'i', }, - // Scoped search + its facet links: the crawler trap that saturated SSR in production. - { - pattern: '^/communities/[a-f0-9-]{36}/search(/.*)?$', - flag: 'i', - }, - { - pattern: '^/collections/[a-f0-9-]{36}/search(/.*)?$', - flag: 'i', - }, { pattern: '^/browse/' }, { pattern: '^/search' }, { pattern: '^/community-list$' }, diff --git a/src/environments/environment.ts b/src/environments/environment.ts index 3ef855a57c7..8be4ee7dbfe 100644 --- a/src/environments/environment.ts +++ b/src/environments/environment.ts @@ -25,15 +25,6 @@ export const environment: Partial = { pattern: '^/collections/[a-f0-9-]{36}/browse(/.*)?$', flag: 'i', }, - // Scoped search + its facet links: the crawler trap that saturated SSR in production. - { - pattern: '^/communities/[a-f0-9-]{36}/search(/.*)?$', - flag: 'i', - }, - { - pattern: '^/collections/[a-f0-9-]{36}/search(/.*)?$', - flag: 'i', - }, { pattern: '^/browse/' }, { pattern: '^/search' }, { pattern: '^/community-list$' }, diff --git a/src/robots.txt.ejs b/src/robots.txt.ejs index 767ec8ea223..ccca458ec0b 100644 --- a/src/robots.txt.ejs +++ b/src/robots.txt.ejs @@ -18,27 +18,12 @@ Disallow: /profile Disallow: /workflowitems # Crawlers should be able to access entity pages, but not the facet search links present on entity pages Disallow: /entities/*?f -# "Disallow: /search" above only matches paths that START with /search, so scoped -# search inside a community or collection stayed crawlable. Enumerating its facet -# links saturated SSR in production (HTTP 504); block it and every facet URL. -Disallow: /collections/*/search -Disallow: /communities/*/search -# Any URL carrying a Discovery facet filter (f.author, f.subject, ...). Two rules -# because the facet can be the first query parameter (?f.) or a later one (&f.), -# and robots.txt cannot express "either". +# Discovery facet links (f.author, f.subject, ...) multiply into an effectively +# unbounded set of crawlable URLs; enumerating them saturated SSR in production +# (HTTP 504). Two rules because the facet can be the first query parameter (?f.) +# or a later one (&f.), and robots.txt cannot express "either". Disallow: /*?f. -Disallow: /*&f. -<% if (disallowBrowse) { -%> -Disallow: /browse -<% } -%> -<% if (disallowHandle) { -%> -Disallow: /handle -<% } -%> -<% if (disallowBitstreams) { -%> -# Bitstream content is reachable by handle path and by UUID; block both. -Disallow: /bitstream/ -Disallow: /bitstreams/ -<% } -%> +Disallow: /*&f.<%- optionalDisallows %> # Optionally uncomment the following line ONLY if sitemaps are working # and you have verified that your site is being indexed correctly.