[Port to dtq-dev] Issue dspace-customers#902: Add configurable crawler policy (robots.txt + SSR excludes)

config/config.example.yml

@@ -60,6 +60,18 @@ universal:
   # Disable this setting to avoid URL replacement during SSR. In this the state is not transferred to avoid security issues.
   replaceRestUrl: true
 
+# Optional per-instance blocks of the served /robots.txt (src/robots.txt.ejs).
+# The Discovery facet-trap rules are always emitted; these switch the extra blocks.
+# All default false because each de-indexes content some instances intentionally
+# expose (a blanket /handle block breaks instances relying on handle redirects).
+robots:
+  # Disallow: /handle
+  disallowHandle: false
+  # Disallow: /browse
+  disallowBrowse: false
+  # Disallow: /bitstream/ and /bitstreams/
+  disallowBitstreams: false
+
 # The REST API server settings
 # NOTE: these settings define which (publicly available) REST API to use. They are usually
 # 'synced' with the 'dspace.server.url' setting in your backend's local.cfg.

config/config.yml

@@ -38,6 +38,15 @@ universal:
   # If set to false the component will not be included in the HTML returned from the server side rendering.
   enableBrowseComponent: false
 
+# Optional per-instance blocks of the served /robots.txt (src/robots.txt.ejs).
+# The Discovery facet-trap rules are always emitted; these switch the extra blocks.
+# All default false because each de-indexes content some instances intentionally
+# expose (a blanket /handle block breaks instances relying on handle redirects).
+robots:
+  disallowHandle: false
+  disallowBrowse: false
+  disallowBitstreams: false
+
 # Caching settings
 cache:
   # NOTE: how long should objects be cached for by default

server.ts

@@ -56,6 +56,7 @@ import { extendEnvironmentWithAppConfig } from './src/config/config.util';
 import { logStartupMessage } from './startup-message';
 import { TOKENITEM } from './src/app/core/auth/models/auth-token-info.model';
 import { SsrExcludePatterns } from './src/config/universal-config.interface';
+import { buildOptionalRobotsDisallows } from './src/config/robots.util';
 
 
 /*
@@ -162,12 +163,14 @@ export function app() {
   server.set('view engine', 'ejs');
 
   /**
-   * Serve the robots.txt ejs template, filling in the origin variable
+   * Serve the robots.txt ejs template, filling in the origin variable and the
+   * per-instance optional Disallow blocks (see config `robots`).
    */
   server.get('/robots.txt', (req, res) => {
     res.setHeader('content-type', 'text/plain');
     res.render('assets/robots.txt.ejs', {
-      'origin': req.protocol + '://' + req.headers.host
+      'origin': req.protocol + '://' + req.headers.host,
+      'optionalDisallows': buildOptionalRobotsDisallows(environment.robots)
     });
   });
 

src/config/app-config.interface.ts

@@ -27,6 +27,7 @@ import { SearchConfig } from './search-page-config.interface';
 import { AccessibilitySettingsConfig } from '../app/accessibility/accessibility-settings.config';
 import { MatomoConfig } from './matomo-config';
 import { StatisticsConfig } from './statistics-config';
+import { RobotsConfig } from './robots-config.interface';
 
 interface AppConfig extends Config {
   ui: UIServerConfig;
@@ -59,6 +60,7 @@ interface AppConfig extends Config {
   signpostingEnabled: boolean;
   matomo: MatomoConfig;
   statistics?: StatisticsConfig;
+  robots: RobotsConfig;
 }
 
 /**

src/config/default-app-config.ts

@@ -26,6 +26,7 @@ import { LiveRegionConfig } from '../app/shared/live-region/live-region.config';
 import { SearchConfig } from './search-page-config.interface';
 import { AccessibilitySettingsConfig } from '../app/accessibility/accessibility-settings.config';
 import { MatomoConfig } from './matomo-config';
+import { RobotsConfig } from './robots-config.interface';
 
 export class DefaultAppConfig implements AppConfig {
   production = false;
@@ -415,6 +416,15 @@ export class DefaultAppConfig implements AppConfig {
     mathjax: false,
   };
 
+  // Optional per-instance blocks of the served robots.txt. The facet-trap rules
+  // that protect SSR are always emitted; these switch the extra blocks that
+  // de-index content some instances intentionally expose.
+  robots: RobotsConfig = {
+    disallowHandle: false,
+    disallowBrowse: false,
+    disallowBitstreams: false,
+  };
+
   // Which vocabularies should be used for which search filters
   // and whether to show the filter in the search sidebar
   // Take a look at the filter-vocabulary-config.ts file for documentation on how the options are obtained

src/config/robots-config.interface.ts

@@ -0,0 +1,27 @@
+import { Config } from './config.interface';
+
+/**
+ * Config for the optional, per-instance blocks of the served `robots.txt`
+ * (see `src/robots.txt.ejs`). The facet-trap rules that protect SSR are always
+ * emitted; only these extra blocks are switchable, because each de-indexes
+ * content some instances intentionally expose.
+ */
+export interface RobotsConfig extends Config {
+
+  /**
+   * Emit `Disallow: /handle`. Off by default: a blanket handle block de-indexes
+   * persistent identifiers on instances that rely on handle redirects.
+   */
+  disallowHandle: boolean;
+
+  /**
+   * Emit `Disallow: /browse`.
+   */
+  disallowBrowse: boolean;
+
+  /**
+   * Emit `Disallow: /bitstream/` and `Disallow: /bitstreams/` to keep crawlers
+   * out of bitstream content (reachable by handle path or by UUID).
+   */
+  disallowBitstreams: boolean;
+}

src/config/robots.util.spec.ts

@@ -0,0 +1,40 @@
+import { buildOptionalRobotsDisallows } from './robots.util';
+import { RobotsConfig } from './robots-config.interface';
+
+describe('buildOptionalRobotsDisallows', () => {
+  const allOff: RobotsConfig = {
+    disallowHandle: false,
+    disallowBrowse: false,
+    disallowBitstreams: false,
+  };
+
+  it('emits nothing when every toggle is off', () => {
+    expect(buildOptionalRobotsDisallows(allOff)).toBe('');
+  });
+
+  it('emits only /browse when disallowBrowse is on', () => {
+    expect(buildOptionalRobotsDisallows({ ...allOff, disallowBrowse: true }))
+      .toBe('\nDisallow: /browse');
+  });
+
+  it('emits only /handle when disallowHandle is on', () => {
+    expect(buildOptionalRobotsDisallows({ ...allOff, disallowHandle: true }))
+      .toBe('\nDisallow: /handle');
+  });
+
+  it('emits both bitstream rules when disallowBitstreams is on', () => {
+    const out = buildOptionalRobotsDisallows({ ...allOff, disallowBitstreams: true });
+    expect(out).toContain('Disallow: /bitstream/');
+    expect(out).toContain('Disallow: /bitstreams/');
+  });
+
+  it('leads with a newline and never a blank line, so the block stays inside the group', () => {
+    const out = buildOptionalRobotsDisallows({
+      disallowHandle: true,
+      disallowBrowse: true,
+      disallowBitstreams: true,
+    });
+    expect(out.startsWith('\n')).toBeTrue();
+    expect(out).not.toContain('\n\n');
+  });
+});

src/config/robots.util.ts

@@ -0,0 +1,27 @@
+import { RobotsConfig } from './robots-config.interface';
+
+/**
+ * Build the optional, per-instance `Disallow` lines for the served robots.txt
+ * (see `src/robots.txt.ejs`) from config. The always-on facet-trap rules live in
+ * the template itself; only these switchable blocks are assembled here so the
+ * toggle logic has a unit-testable seam.
+ *
+ * Returns a string that already starts with a newline when non-empty, so it can
+ * be appended directly after the last static rule without leaving a blank line
+ * inside the `User-agent: *` group record (blank lines end a group).
+ */
+export function buildOptionalRobotsDisallows(robots: RobotsConfig): string {
+  const lines: string[] = [];
+  if (robots?.disallowBrowse) {
+    lines.push('Disallow: /browse');
+  }
+  if (robots?.disallowHandle) {
+    lines.push('Disallow: /handle');
+  }
+  if (robots?.disallowBitstreams) {
+    // Bitstream content is reachable by handle path and by UUID; block both.
+    lines.push('Disallow: /bitstream/');
+    lines.push('Disallow: /bitstreams/');
+  }
+  return lines.length ? '\n' + lines.join('\n') : '';
+}

src/environments/environment.test.ts

@@ -327,6 +327,11 @@ export const environment: BuildConfig = {
     enabled: false,
     mathjax: false,
   },
+  robots: {
+    disallowHandle: false,
+    disallowBrowse: false,
+    disallowBitstreams: false,
+  },
   comcolSelectionSort: {
     sortField:'dc.title',
     sortDirection:'ASC',

src/robots.txt.ejs

@@ -18,6 +18,12 @@ Disallow: /profile
 Disallow: /workflowitems
 # Crawlers should be able to access entity pages, but not the facet search links present on entity pages
 Disallow: /entities/*?f
+# Discovery facet links (f.author, f.subject, ...) multiply into an effectively
+# unbounded set of crawlable URLs; enumerating them saturated SSR in production
+# (HTTP 504). Two rules because the facet can be the first query parameter (?f.)
+# or a later one (&f.), and robots.txt cannot express "either".
+Disallow: /*?f.
+Disallow: /*&f.<%- optionalDisallows %>
 
 # Optionally uncomment the following line ONLY if sitemaps are working
 # and you have verified that your site is being indexed correctly.