diff --git a/config/config.example.yml b/config/config.example.yml index 8b56711c7d2..508aba90298 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -60,6 +60,18 @@ universal: # Disable this setting to avoid URL replacement during SSR. In this the state is not transferred to avoid security issues. replaceRestUrl: true +# Optional per-instance blocks of the served /robots.txt (src/robots.txt.ejs). +# The Discovery facet-trap rules are always emitted; these switch the extra blocks. +# All default false because each de-indexes content some instances intentionally +# expose (a blanket /handle block breaks instances relying on handle redirects). +robots: + # Disallow: /handle + disallowHandle: false + # Disallow: /browse + disallowBrowse: false + # Disallow: /bitstream/ and /bitstreams/ + disallowBitstreams: false + # The REST API server settings # NOTE: these settings define which (publicly available) REST API to use. They are usually # 'synced' with the 'dspace.server.url' setting in your backend's local.cfg. diff --git a/config/config.yml b/config/config.yml index 9257bd2b09d..2c59ae2e0d7 100644 --- a/config/config.yml +++ b/config/config.yml @@ -38,6 +38,15 @@ universal: # If set to false the component will not be included in the HTML returned from the server side rendering. enableBrowseComponent: false +# Optional per-instance blocks of the served /robots.txt (src/robots.txt.ejs). +# The Discovery facet-trap rules are always emitted; these switch the extra blocks. +# All default false because each de-indexes content some instances intentionally +# expose (a blanket /handle block breaks instances relying on handle redirects). +robots: + disallowHandle: false + disallowBrowse: false + disallowBitstreams: false + # Caching settings cache: # NOTE: how long should objects be cached for by default diff --git a/server.ts b/server.ts index e7b13aef522..e5f3ebcab8d 100644 --- a/server.ts +++ b/server.ts @@ -56,6 +56,7 @@ import { extendEnvironmentWithAppConfig } from './src/config/config.util'; import { logStartupMessage } from './startup-message'; import { TOKENITEM } from './src/app/core/auth/models/auth-token-info.model'; import { SsrExcludePatterns } from './src/config/universal-config.interface'; +import { buildOptionalRobotsDisallows } from './src/config/robots.util'; /* @@ -162,12 +163,14 @@ export function app() { server.set('view engine', 'ejs'); /** - * Serve the robots.txt ejs template, filling in the origin variable + * Serve the robots.txt ejs template, filling in the origin variable and the + * per-instance optional Disallow blocks (see config `robots`). */ server.get('/robots.txt', (req, res) => { res.setHeader('content-type', 'text/plain'); res.render('assets/robots.txt.ejs', { - 'origin': req.protocol + '://' + req.headers.host + 'origin': req.protocol + '://' + req.headers.host, + 'optionalDisallows': buildOptionalRobotsDisallows(environment.robots) }); }); diff --git a/src/config/app-config.interface.ts b/src/config/app-config.interface.ts index 96a6f91f8ec..67357d94033 100644 --- a/src/config/app-config.interface.ts +++ b/src/config/app-config.interface.ts @@ -27,6 +27,7 @@ import { SearchConfig } from './search-page-config.interface'; import { AccessibilitySettingsConfig } from '../app/accessibility/accessibility-settings.config'; import { MatomoConfig } from './matomo-config'; import { StatisticsConfig } from './statistics-config'; +import { RobotsConfig } from './robots-config.interface'; interface AppConfig extends Config { ui: UIServerConfig; @@ -59,6 +60,7 @@ interface AppConfig extends Config { signpostingEnabled: boolean; matomo: MatomoConfig; statistics?: StatisticsConfig; + robots: RobotsConfig; } /** diff --git a/src/config/default-app-config.ts b/src/config/default-app-config.ts index 38cb942e457..3fce69d7e6d 100644 --- a/src/config/default-app-config.ts +++ b/src/config/default-app-config.ts @@ -26,6 +26,7 @@ import { LiveRegionConfig } from '../app/shared/live-region/live-region.config'; import { SearchConfig } from './search-page-config.interface'; import { AccessibilitySettingsConfig } from '../app/accessibility/accessibility-settings.config'; import { MatomoConfig } from './matomo-config'; +import { RobotsConfig } from './robots-config.interface'; export class DefaultAppConfig implements AppConfig { production = false; @@ -415,6 +416,15 @@ export class DefaultAppConfig implements AppConfig { mathjax: false, }; + // Optional per-instance blocks of the served robots.txt. The facet-trap rules + // that protect SSR are always emitted; these switch the extra blocks that + // de-index content some instances intentionally expose. + robots: RobotsConfig = { + disallowHandle: false, + disallowBrowse: false, + disallowBitstreams: false, + }; + // Which vocabularies should be used for which search filters // and whether to show the filter in the search sidebar // Take a look at the filter-vocabulary-config.ts file for documentation on how the options are obtained diff --git a/src/config/robots-config.interface.ts b/src/config/robots-config.interface.ts new file mode 100644 index 00000000000..dad67dc79e5 --- /dev/null +++ b/src/config/robots-config.interface.ts @@ -0,0 +1,27 @@ +import { Config } from './config.interface'; + +/** + * Config for the optional, per-instance blocks of the served `robots.txt` + * (see `src/robots.txt.ejs`). The facet-trap rules that protect SSR are always + * emitted; only these extra blocks are switchable, because each de-indexes + * content some instances intentionally expose. + */ +export interface RobotsConfig extends Config { + + /** + * Emit `Disallow: /handle`. Off by default: a blanket handle block de-indexes + * persistent identifiers on instances that rely on handle redirects. + */ + disallowHandle: boolean; + + /** + * Emit `Disallow: /browse`. + */ + disallowBrowse: boolean; + + /** + * Emit `Disallow: /bitstream/` and `Disallow: /bitstreams/` to keep crawlers + * out of bitstream content (reachable by handle path or by UUID). + */ + disallowBitstreams: boolean; +} diff --git a/src/config/robots.util.spec.ts b/src/config/robots.util.spec.ts new file mode 100644 index 00000000000..ee0e884801d --- /dev/null +++ b/src/config/robots.util.spec.ts @@ -0,0 +1,40 @@ +import { buildOptionalRobotsDisallows } from './robots.util'; +import { RobotsConfig } from './robots-config.interface'; + +describe('buildOptionalRobotsDisallows', () => { + const allOff: RobotsConfig = { + disallowHandle: false, + disallowBrowse: false, + disallowBitstreams: false, + }; + + it('emits nothing when every toggle is off', () => { + expect(buildOptionalRobotsDisallows(allOff)).toBe(''); + }); + + it('emits only /browse when disallowBrowse is on', () => { + expect(buildOptionalRobotsDisallows({ ...allOff, disallowBrowse: true })) + .toBe('\nDisallow: /browse'); + }); + + it('emits only /handle when disallowHandle is on', () => { + expect(buildOptionalRobotsDisallows({ ...allOff, disallowHandle: true })) + .toBe('\nDisallow: /handle'); + }); + + it('emits both bitstream rules when disallowBitstreams is on', () => { + const out = buildOptionalRobotsDisallows({ ...allOff, disallowBitstreams: true }); + expect(out).toContain('Disallow: /bitstream/'); + expect(out).toContain('Disallow: /bitstreams/'); + }); + + it('leads with a newline and never a blank line, so the block stays inside the group', () => { + const out = buildOptionalRobotsDisallows({ + disallowHandle: true, + disallowBrowse: true, + disallowBitstreams: true, + }); + expect(out.startsWith('\n')).toBeTrue(); + expect(out).not.toContain('\n\n'); + }); +}); diff --git a/src/config/robots.util.ts b/src/config/robots.util.ts new file mode 100644 index 00000000000..acea906f58b --- /dev/null +++ b/src/config/robots.util.ts @@ -0,0 +1,27 @@ +import { RobotsConfig } from './robots-config.interface'; + +/** + * Build the optional, per-instance `Disallow` lines for the served robots.txt + * (see `src/robots.txt.ejs`) from config. The always-on facet-trap rules live in + * the template itself; only these switchable blocks are assembled here so the + * toggle logic has a unit-testable seam. + * + * Returns a string that already starts with a newline when non-empty, so it can + * be appended directly after the last static rule without leaving a blank line + * inside the `User-agent: *` group record (blank lines end a group). + */ +export function buildOptionalRobotsDisallows(robots: RobotsConfig): string { + const lines: string[] = []; + if (robots?.disallowBrowse) { + lines.push('Disallow: /browse'); + } + if (robots?.disallowHandle) { + lines.push('Disallow: /handle'); + } + if (robots?.disallowBitstreams) { + // Bitstream content is reachable by handle path and by UUID; block both. + lines.push('Disallow: /bitstream/'); + lines.push('Disallow: /bitstreams/'); + } + return lines.length ? '\n' + lines.join('\n') : ''; +} diff --git a/src/environments/environment.test.ts b/src/environments/environment.test.ts index a0cab16ef83..31dad7e336f 100644 --- a/src/environments/environment.test.ts +++ b/src/environments/environment.test.ts @@ -327,6 +327,11 @@ export const environment: BuildConfig = { enabled: false, mathjax: false, }, + robots: { + disallowHandle: false, + disallowBrowse: false, + disallowBitstreams: false, + }, comcolSelectionSort: { sortField:'dc.title', sortDirection:'ASC', diff --git a/src/robots.txt.ejs b/src/robots.txt.ejs index f6dc7fa5cb5..ccca458ec0b 100644 --- a/src/robots.txt.ejs +++ b/src/robots.txt.ejs @@ -18,6 +18,12 @@ Disallow: /profile Disallow: /workflowitems # Crawlers should be able to access entity pages, but not the facet search links present on entity pages Disallow: /entities/*?f +# Discovery facet links (f.author, f.subject, ...) multiply into an effectively +# unbounded set of crawlable URLs; enumerating them saturated SSR in production +# (HTTP 504). Two rules because the facet can be the first query parameter (?f.) +# or a later one (&f.), and robots.txt cannot express "either". +Disallow: /*?f. +Disallow: /*&f.<%- optionalDisallows %> # Optionally uncomment the following line ONLY if sitemaps are working # and you have verified that your site is being indexed correctly.