diff --git a/config/config.example.yml b/config/config.example.yml index 9aae1fc79b4..b43d15196db 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -286,6 +286,8 @@ item: undoTimeout: 10000 # 10 seconds # Show the item access status label in items lists showAccessStatuses: false + # Item uuids or handles to exclude from search engine indexes. Empty = off. + noIndex: [] bitstream: # Number of entries in the bitstream list in the item view page. # Rounded to the nearest size in the list of selectable sizes on the diff --git a/config/config.yml b/config/config.yml index cdf957944f8..f1c986ed661 100644 --- a/config/config.yml +++ b/config/config.yml @@ -192,3 +192,6 @@ item: bitstream: # Per-bitstream embargo-date badge in the Item View showAccessStatuses: true + # Item uuids or handles to exclude from search engine indexes. Empty = off. + # Keep empty: this config is served to the browser, so entries here are public. + noIndex: [] diff --git a/src/app/core/metadata/metadata.service.spec.ts b/src/app/core/metadata/metadata.service.spec.ts index fac3937643e..bc07445554a 100644 --- a/src/app/core/metadata/metadata.service.spec.ts +++ b/src/app/core/metadata/metadata.service.spec.ts @@ -12,8 +12,10 @@ import { ItemMock, MockBitstream1, MockBitstream3, - MockBitstream2 + MockBitstream2, + NonDiscoverableItemMock } from '../../shared/mocks/item.mock'; +import { DSpaceObject } from '../shared/dspace-object.model'; import { createSuccessfulRemoteDataObject, createSuccessfulRemoteDataObject$ } from '../../shared/remote-data.utils'; import { PaginatedList } from '../data/paginated-list.model'; import { Bitstream } from '../shared/bitstream.model'; @@ -96,6 +98,7 @@ describe('MetadataService', () => { appConfig = { item: { + noIndex: [], bitstream: { pageSize: 5 } @@ -406,6 +409,90 @@ describe('MetadataService', () => { }); }); + describe('robots meta tag', () => { + const noIndexTag = { name: 'robots', content: 'noindex, noarchive' }; + + const routeTo = (dso: any) => { + (metadataService as any).processRouteChange({ + data: { value: { dso: createSuccessfulRemoteDataObject(dso) } } + }); + tick(); + }; + + it('should not add a robots tag for a normal discoverable item', fakeAsync(() => { + routeTo(ItemMock); + expect(meta.addTag).not.toHaveBeenCalledWith(jasmine.objectContaining({ name: 'robots' })); + })); + + it('should add a robots noindex tag for a non-discoverable item', fakeAsync(() => { + routeTo(NonDiscoverableItemMock); + expect(meta.addTag).toHaveBeenCalledWith(noIndexTag); + })); + + it('should add a robots noindex tag when the item uuid is configured', fakeAsync(() => { + appConfig.item.noIndex = ['0ec7ff22-f211-40ab-a69e-c819b0b1f357']; + routeTo(ItemMock); + expect(meta.addTag).toHaveBeenCalledWith(noIndexTag); + })); + + it('should add a robots noindex tag when the item handle is configured', fakeAsync(() => { + appConfig.item.noIndex = ['10673/6']; + routeTo(ItemMock); + expect(meta.addTag).toHaveBeenCalledWith(noIndexTag); + })); + + it('should normalize handle URLs, casing and whitespace in item.noIndex', fakeAsync(() => { + appConfig.item.noIndex = [' HTTP://hdl.handle.net/10673/6 ']; + routeTo(ItemMock); + expect(meta.addTag).toHaveBeenCalledWith(noIndexTag); + })); + + it('should not add a robots tag for an item that is not configured', fakeAsync(() => { + appConfig.item.noIndex = ['11025/9501', 'f4c45569-cdfc-4b3d-98df-46bfeba016b9']; + routeTo(ItemMock); + expect(meta.addTag).not.toHaveBeenCalledWith(jasmine.objectContaining({ name: 'robots' })); + })); + + it('should not add a robots tag for a non-Item DSpaceObject', fakeAsync(() => { + appConfig.item.noIndex = ['10673/6']; + routeTo(Object.assign(new DSpaceObject(), { uuid: '10673/6', handle: '10673/6', metadata: {} })); + expect(meta.addTag).not.toHaveBeenCalledWith(jasmine.objectContaining({ name: 'robots' })); + })); + + it('should register the robots tag in the meta tag store so it is cleared on the next route change', fakeAsync(() => { + appConfig.item.noIndex = ['10673/6']; + routeTo(ItemMock); + expect(store.dispatch).toHaveBeenCalledWith(new AddMetaTagAction('robots')); + })); + + it('should suppress citation_pdf_url for a noindex item', fakeAsync(() => { + appConfig.item.noIndex = ['10673/6']; + routeTo(ItemMock); + expect(meta.addTag).not.toHaveBeenCalledWith(jasmine.objectContaining({ name: 'citation_pdf_url' })); + expect(meta.addTag).toHaveBeenCalledWith(jasmine.objectContaining({ name: 'citation_title' })); + })); + + it('should keep citation_pdf_url for a normal item', fakeAsync(() => { + routeTo(ItemMock); + expect(meta.addTag).toHaveBeenCalledWith(jasmine.objectContaining({ name: 'citation_pdf_url' })); + })); + + it('should not break the other meta tags when item.noIndex is a scalar instead of a list', fakeAsync(() => { + // Must degrade to "off", never throw - that would strip the meta tags off every page. + appConfig.item.noIndex = '10673/6' as any; + expect(() => routeTo(ItemMock)).not.toThrow(); + expect(meta.addTag).toHaveBeenCalledWith(jasmine.objectContaining({ name: 'citation_title' })); + expect(meta.addTag).not.toHaveBeenCalledWith(jasmine.objectContaining({ name: 'robots' })); + })); + + it('should ignore non-string entries in item.noIndex without throwing', fakeAsync(() => { + appConfig.item.noIndex = [9501 as any, null, '10673/6']; + expect(() => routeTo(ItemMock)).not.toThrow(); + expect(meta.addTag).toHaveBeenCalledWith(noIndexTag); + expect(meta.addTag).toHaveBeenCalledWith(jasmine.objectContaining({ name: 'citation_title' })); + })); + }); + describe(`when there's no bitstream with an allowed format on the first page`, () => { let bitstreams; diff --git a/src/app/core/metadata/metadata.service.ts b/src/app/core/metadata/metadata.service.ts index e774dd342c7..8da1f501e6e 100644 --- a/src/app/core/metadata/metadata.service.ts +++ b/src/app/core/metadata/metadata.service.ts @@ -63,6 +63,9 @@ const tagsInUseSelector = (state: MetaTagState) => state.tagsInUse, ); +// No `nofollow`: crawlers should still follow the bitstream links to pick up their own noindex. +export const NO_INDEX_META_CONTENT = 'noindex, noarchive'; + @Injectable() export class MetadataService { @@ -147,6 +150,8 @@ export class MetadataService { private setDSOMetaTags(): void { + this.setNoIndexTag(); + this.setTitleTag(); this.setDescriptionTag(); @@ -194,6 +199,46 @@ export class MetadataService { } + /** + * Add for Items that must not be indexed by search engines. + * + * Uses addMetaTag() so the tag is registered in the meta tag store and cleared on the next route + * change; this.meta.addTag() would leak it onto every page visited afterwards. + */ + protected setNoIndexTag(): void { + if (this.isNoIndex()) { + this.addMetaTag('robots', NO_INDEX_META_CONTENT); + } + } + + private isNoIndex(): boolean { + if (!(this.currentObject.value instanceof Item)) { + return false; + } + const item = this.currentObject.value as Item; + if (item.isDiscoverable === false) { + return true; + } + // Array.isArray, not hasNoValue: a misconfigured scalar also has a length, and throwing here + // would strip the meta tags off every page. + const configured = this.appConfig?.item?.noIndex; + if (!Array.isArray(configured) || configured.length === 0) { + return false; + } + const itemIds = [item.uuid, item.handle] + .filter((id) => isNotEmpty(id)) + .map((id) => this.normalizeNoIndexId(id)); + return configured.some((id: any) => typeof id === 'string' && isNotEmpty(id) + && itemIds.includes(this.normalizeNoIndexId(id))); + } + + // Accepts a bare handle, a hdl.handle.net URL or a uuid, in any casing. + private normalizeNoIndexId(id: string): string { + return id.trim().toLowerCase() + .replace(/^https?:\/\/hdl\.handle\.net\//, '') + .replace(/^\/+/, ''); + } + /** * Add to the
*/ @@ -349,6 +394,10 @@ export class MetadataService { * Add to the */ private setCitationPdfUrlTag(): void { + // Google Scholar keys off this tag and does not reliably honour the landing page robots tag. + if (this.isNoIndex()) { + return; + } if (this.currentObject.value instanceof Item) { const item = this.currentObject.value as Item; diff --git a/src/app/shared/mocks/item.mock.ts b/src/app/shared/mocks/item.mock.ts index 77685cca9ac..1e78ebc98f6 100644 --- a/src/app/shared/mocks/item.mock.ts +++ b/src/app/shared/mocks/item.mock.ts @@ -294,3 +294,12 @@ export const ItemMock: Item = Object.assign(new Item(), { ) }); /* eslint-enable @typescript-eslint/no-shadow */ + +// `metadata` is copied because spec helpers mutate it in place and would hit ItemMock too. +export const NonDiscoverableItemMock: Item = Object.assign(new Item(), ItemMock, { + handle: '10673/7', + id: '0ec7ff22-f211-40ab-a69e-c819b0b1f358', + uuid: '0ec7ff22-f211-40ab-a69e-c819b0b1f358', + isDiscoverable: false, + metadata: Object.assign({}, ItemMock.metadata), +}); diff --git a/src/config/default-app-config.ts b/src/config/default-app-config.ts index fad2b56ab33..892a8f1089f 100644 --- a/src/config/default-app-config.ts +++ b/src/config/default-app-config.ts @@ -281,6 +281,8 @@ export class DefaultAppConfig implements AppConfig { }, // Show the item access status label in items lists showAccessStatuses: false, + // Item uuids or handles to exclude from search engine indexes + noIndex: [], bitstream: { // Number of entries in the bitstream list in the item view page. // Rounded to the nearest size in the list of selectable sizes on the diff --git a/src/config/item-config.interface.ts b/src/config/item-config.interface.ts index f3141f77238..3b392795fa8 100644 --- a/src/config/item-config.interface.ts +++ b/src/config/item-config.interface.ts @@ -7,6 +7,9 @@ export interface ItemConfig extends Config { // This is used to show the access status label of items in results lists showAccessStatuses: boolean; + // Item uuids or handles to exclude from search engine indexes. Empty = off. + noIndex: string[]; + bitstream: { // Number of entries in the bitstream list in the item view page. // Rounded to the nearest size in the list of selectable sizes on the diff --git a/src/environments/environment.test.ts b/src/environments/environment.test.ts index 46164ae5da4..2be8007f1ff 100644 --- a/src/environments/environment.test.ts +++ b/src/environments/environment.test.ts @@ -250,6 +250,8 @@ export const environment: BuildConfig = { }, // Show the item access status label in items lists showAccessStatuses: false, + // Items excluded from search engine indexes (uuids or handles) + noIndex: [], bitstream: { // Number of entries in the bitstream list in the item view page. // Rounded to the nearest size in the list of selectable sizes on the