ZCU-PUB/Add a configurable noindex meta tag for item pages

config/config.example.yml

@@ -286,6 +286,8 @@ item:
     undoTimeout: 10000 # 10 seconds
   # Show the item access status label in items lists
   showAccessStatuses: false
+  # Item uuids or handles to exclude from search engine indexes. Empty = off.
+  noIndex: []
   bitstream:
     # Number of entries in the bitstream list in the item view page.
     # Rounded to the nearest size in the list of selectable sizes on the

config/config.yml

@@ -192,3 +192,6 @@ item:
   bitstream:
     # Per-bitstream embargo-date badge in the Item View
     showAccessStatuses: true
+  # Item uuids or handles to exclude from search engine indexes. Empty = off.
+  # Keep empty: this config is served to the browser, so entries here are public.
+  noIndex: []

src/app/core/metadata/metadata.service.spec.ts

@@ -12,8 +12,10 @@ import {
   ItemMock,
   MockBitstream1,
   MockBitstream3,
-  MockBitstream2
+  MockBitstream2,
+  NonDiscoverableItemMock
 } from '../../shared/mocks/item.mock';
+import { DSpaceObject } from '../shared/dspace-object.model';
 import { createSuccessfulRemoteDataObject, createSuccessfulRemoteDataObject$ } from '../../shared/remote-data.utils';
 import { PaginatedList } from '../data/paginated-list.model';
 import { Bitstream } from '../shared/bitstream.model';
@@ -96,6 +98,7 @@ describe('MetadataService', () => {
 
     appConfig = {
       item: {
+        noIndex: [],
         bitstream: {
           pageSize: 5
         }
@@ -406,6 +409,90 @@ describe('MetadataService', () => {
     });
   });
 
+  describe('robots meta tag', () => {
+    const noIndexTag = { name: 'robots', content: 'noindex, noarchive' };
+
+    const routeTo = (dso: any) => {
+      (metadataService as any).processRouteChange({
+        data: { value: { dso: createSuccessfulRemoteDataObject(dso) } }
+      });
+      tick();
+    };
+
+    it('should not add a robots tag for a normal discoverable item', fakeAsync(() => {
+      routeTo(ItemMock);
+      expect(meta.addTag).not.toHaveBeenCalledWith(jasmine.objectContaining({ name: 'robots' }));
+    }));
+
+    it('should add a robots noindex tag for a non-discoverable item', fakeAsync(() => {
+      routeTo(NonDiscoverableItemMock);
+      expect(meta.addTag).toHaveBeenCalledWith(noIndexTag);
+    }));
+
+    it('should add a robots noindex tag when the item uuid is configured', fakeAsync(() => {
+      appConfig.item.noIndex = ['0ec7ff22-f211-40ab-a69e-c819b0b1f357'];
+      routeTo(ItemMock);
+      expect(meta.addTag).toHaveBeenCalledWith(noIndexTag);
+    }));
+
+    it('should add a robots noindex tag when the item handle is configured', fakeAsync(() => {
+      appConfig.item.noIndex = ['10673/6'];
+      routeTo(ItemMock);
+      expect(meta.addTag).toHaveBeenCalledWith(noIndexTag);
+    }));
+
+    it('should normalize handle URLs, casing and whitespace in item.noIndex', fakeAsync(() => {
+      appConfig.item.noIndex = ['  HTTP://hdl.handle.net/10673/6  '];
+      routeTo(ItemMock);
+      expect(meta.addTag).toHaveBeenCalledWith(noIndexTag);
+    }));
+
+    it('should not add a robots tag for an item that is not configured', fakeAsync(() => {
+      appConfig.item.noIndex = ['11025/9501', 'f4c45569-cdfc-4b3d-98df-46bfeba016b9'];
+      routeTo(ItemMock);
+      expect(meta.addTag).not.toHaveBeenCalledWith(jasmine.objectContaining({ name: 'robots' }));
+    }));
+
+    it('should not add a robots tag for a non-Item DSpaceObject', fakeAsync(() => {
+      appConfig.item.noIndex = ['10673/6'];
+      routeTo(Object.assign(new DSpaceObject(), { uuid: '10673/6', handle: '10673/6', metadata: {} }));
+      expect(meta.addTag).not.toHaveBeenCalledWith(jasmine.objectContaining({ name: 'robots' }));
+    }));
+
+    it('should register the robots tag in the meta tag store so it is cleared on the next route change', fakeAsync(() => {
+      appConfig.item.noIndex = ['10673/6'];
+      routeTo(ItemMock);
+      expect(store.dispatch).toHaveBeenCalledWith(new AddMetaTagAction('robots'));
+    }));
+
+    it('should suppress citation_pdf_url for a noindex item', fakeAsync(() => {
+      appConfig.item.noIndex = ['10673/6'];
+      routeTo(ItemMock);
+      expect(meta.addTag).not.toHaveBeenCalledWith(jasmine.objectContaining({ name: 'citation_pdf_url' }));
+      expect(meta.addTag).toHaveBeenCalledWith(jasmine.objectContaining({ name: 'citation_title' }));
+    }));
+
+    it('should keep citation_pdf_url for a normal item', fakeAsync(() => {
+      routeTo(ItemMock);
+      expect(meta.addTag).toHaveBeenCalledWith(jasmine.objectContaining({ name: 'citation_pdf_url' }));
+    }));
+
+    it('should not break the other meta tags when item.noIndex is a scalar instead of a list', fakeAsync(() => {
+      // Must degrade to "off", never throw - that would strip the meta tags off every page.
+      appConfig.item.noIndex = '10673/6' as any;
+      expect(() => routeTo(ItemMock)).not.toThrow();
+      expect(meta.addTag).toHaveBeenCalledWith(jasmine.objectContaining({ name: 'citation_title' }));
+      expect(meta.addTag).not.toHaveBeenCalledWith(jasmine.objectContaining({ name: 'robots' }));
+    }));
+
+    it('should ignore non-string entries in item.noIndex without throwing', fakeAsync(() => {
+      appConfig.item.noIndex = [9501 as any, null, '10673/6'];
+      expect(() => routeTo(ItemMock)).not.toThrow();
+      expect(meta.addTag).toHaveBeenCalledWith(noIndexTag);
+      expect(meta.addTag).toHaveBeenCalledWith(jasmine.objectContaining({ name: 'citation_title' }));
+    }));
+  });
+
   describe(`when there's no bitstream with an allowed format on the first page`, () => {
     let bitstreams;
 

src/app/core/metadata/metadata.service.ts

@@ -63,6 +63,9 @@ const tagsInUseSelector =
     (state: MetaTagState) => state.tagsInUse,
   );
 
+// No `nofollow`: crawlers should still follow the bitstream links to pick up their own noindex.
+export const NO_INDEX_META_CONTENT = 'noindex, noarchive';
+
 @Injectable()
 export class MetadataService {
 
@@ -147,6 +150,8 @@ export class MetadataService {
 
   private setDSOMetaTags(): void {
 
+    this.setNoIndexTag();
+
     this.setTitleTag();
     this.setDescriptionTag();
 
@@ -194,6 +199,46 @@ export class MetadataService {
 
   }
 
+  /**
+   * Add <meta name="robots"> for Items that must not be indexed by search engines.
+   *
+   * Uses addMetaTag() so the tag is registered in the meta tag store and cleared on the next route
+   * change; this.meta.addTag() would leak it onto every page visited afterwards.
+   */
+  protected setNoIndexTag(): void {
+    if (this.isNoIndex()) {
+      this.addMetaTag('robots', NO_INDEX_META_CONTENT);
+    }
+  }
+
+  private isNoIndex(): boolean {
+    if (!(this.currentObject.value instanceof Item)) {
+      return false;
+    }
+    const item = this.currentObject.value as Item;
+    if (item.isDiscoverable === false) {
+      return true;
+    }
+    // Array.isArray, not hasNoValue: a misconfigured scalar also has a length, and throwing here
+    // would strip the meta tags off every page.
+    const configured = this.appConfig?.item?.noIndex;
+    if (!Array.isArray(configured) || configured.length === 0) {
+      return false;
+    }
+    const itemIds = [item.uuid, item.handle]
+      .filter((id) => isNotEmpty(id))
+      .map((id) => this.normalizeNoIndexId(id));
+    return configured.some((id: any) => typeof id === 'string' && isNotEmpty(id)
+      && itemIds.includes(this.normalizeNoIndexId(id)));
+  }
+
+  // Accepts a bare handle, a hdl.handle.net URL or a uuid, in any casing.
+  private normalizeNoIndexId(id: string): string {
+    return id.trim().toLowerCase()
+      .replace(/^https?:\/\/hdl\.handle\.net\//, '')
+      .replace(/^\/+/, '');
+  }
+
   /**
    * Add <meta name="title" ... >  to the <head>
    */
@@ -349,6 +394,10 @@ export class MetadataService {
    * Add <meta name="citation_pdf_url" ... >  to the <head>
    */
   private setCitationPdfUrlTag(): void {
+    // Google Scholar keys off this tag and does not reliably honour the landing page robots tag.
+    if (this.isNoIndex()) {
+      return;
+    }
     if (this.currentObject.value instanceof Item) {
       const item = this.currentObject.value as Item;
 

src/app/shared/mocks/item.mock.ts

@@ -294,3 +294,12 @@ export const ItemMock: Item = Object.assign(new Item(), {
   )
 });
 /* eslint-enable @typescript-eslint/no-shadow */
+
+// `metadata` is copied because spec helpers mutate it in place and would hit ItemMock too.
+export const NonDiscoverableItemMock: Item = Object.assign(new Item(), ItemMock, {
+  handle: '10673/7',
+  id: '0ec7ff22-f211-40ab-a69e-c819b0b1f358',
+  uuid: '0ec7ff22-f211-40ab-a69e-c819b0b1f358',
+  isDiscoverable: false,
+  metadata: Object.assign({}, ItemMock.metadata),
+});

src/config/default-app-config.ts

@@ -281,6 +281,8 @@ export class DefaultAppConfig implements AppConfig {
     },
     // Show the item access status label in items lists
     showAccessStatuses: false,
+    // Item uuids or handles to exclude from search engine indexes
+    noIndex: [],
     bitstream: {
       // Number of entries in the bitstream list in the item view page.
       // Rounded to the nearest size in the list of selectable sizes on the

src/config/item-config.interface.ts

@@ -7,6 +7,9 @@ export interface ItemConfig extends Config {
   // This is used to show the access status label of items in results lists
   showAccessStatuses: boolean;
 
+  // Item uuids or handles to exclude from search engine indexes. Empty = off.
+  noIndex: string[];
+
   bitstream: {
     // Number of entries in the bitstream list in the item view page.
     // Rounded to the nearest size in the list of selectable sizes on the

src/environments/environment.test.ts

@@ -250,6 +250,8 @@ export const environment: BuildConfig = {
     },
     // Show the item access status label in items lists
     showAccessStatuses: false,
+    // Items excluded from search engine indexes (uuids or handles)
+    noIndex: [],
     bitstream: {
       // Number of entries in the bitstream list in the item view page.
       // Rounded to the nearest size in the list of selectable sizes on the