feat(docs, blog): add support for tags.yml, predefined list of tags (#10137)

Co-authored-by: Sébastien Lorber <slorber@users.noreply.github.com>
Co-authored-by: OzakIOne <OzakIOne@users.noreply.github.com>
Co-authored-by: sebastien <lorber.sebastien@gmail.com>
Co-authored-by: slorber <slorber@users.noreply.github.com>

packages/docusaurus-utils/package.json

@@ -36,6 +36,7 @@
     "shelljs": "^0.8.5",
     "tslib": "^2.6.0",
     "url-loader": "^4.1.1",
+    "utility-types": "^3.10.0",
     "webpack": "^5.88.1"
   },
   "engines": {

packages/docusaurus-utils/src/__tests__/tags.test.ts

@@ -6,117 +6,284 @@
  */
 
 import {
-  normalizeFrontMatterTags,
+  reportInlineTags,
   groupTaggedItems,
-  type Tag,
   getTagVisibility,
-} from '../tags';
+} from '@docusaurus/utils';
+import {normalizeTag} from '../tags';
+import type {Tag, TagMetadata, FrontMatterTag, TagsFile} from '../tags';
 
-describe('normalizeFrontMatterTags', () => {
-  it('normalizes simple string tag', () => {
-    const tagsPath = '/all/tags';
-    const input = 'tag';
-    const expectedOutput = {
-      label: 'tag',
-      permalink: `${tagsPath}/tag`,
+describe('normalizeTag', () => {
+  const tagsBaseRoutePath = '/all/tags';
+
+  describe('inline', () => {
+    it('normalizes simple string tag', () => {
+      const input: FrontMatterTag = 'tag';
+      const expectedOutput: TagMetadata = {
+        inline: true,
+        label: 'tag',
+        permalink: `${tagsBaseRoutePath}/tag`,
+        description: undefined,
+      };
+      expect(
+        normalizeTag({tagsBaseRoutePath, tagsFile: null, tag: input}),
+      ).toEqual(expectedOutput);
+    });
+
+    it('normalizes complex string tag', () => {
+      const input: FrontMatterTag = 'some more Complex_tag';
+      const expectedOutput: TagMetadata = {
+        inline: true,
+        label: 'some more Complex_tag',
+        permalink: `${tagsBaseRoutePath}/some-more-complex-tag`,
+        description: undefined,
+      };
+      expect(
+        normalizeTag({tagsBaseRoutePath, tagsFile: null, tag: input}),
+      ).toEqual(expectedOutput);
+    });
+
+    it('normalizes simple object tag', () => {
+      const input: FrontMatterTag = {
+        label: 'tag',
+        permalink: 'tagPermalink',
+      };
+      const expectedOutput: TagMetadata = {
+        inline: true,
+        label: 'tag',
+        permalink: `${tagsBaseRoutePath}/tagPermalink`,
+        description: undefined,
+      };
+      expect(
+        normalizeTag({tagsBaseRoutePath, tagsFile: null, tag: input}),
+      ).toEqual(expectedOutput);
+    });
+
+    it('normalizes complex string tag with object tag', () => {
+      const input: FrontMatterTag = {
+        label: 'tag complex Label',
+        permalink: '/MoreComplex/Permalink',
+      };
+      const expectedOutput: TagMetadata = {
+        inline: true,
+        label: 'tag complex Label',
+        permalink: `${tagsBaseRoutePath}/MoreComplex/Permalink`,
+        description: undefined,
+      };
+      expect(
+        normalizeTag({tagsBaseRoutePath, tagsFile: null, tag: input}),
+      ).toEqual(expectedOutput);
+    });
+  });
+
+  describe('with tags file', () => {
+    const tagsFile: TagsFile = {
+      tag1: {
+        label: 'Tag 1 label',
+        permalink: 'tag-1-permalink',
+        description: 'Tag 1 description',
+      },
+      tag2: {
+        label: 'Tag 2 label',
+        permalink: '/tag-2-permalink',
+        description: undefined,
+      },
     };
-    expect(normalizeFrontMatterTags(tagsPath, [input])).toEqual([
-      expectedOutput,
-    ]);
+
+    it('normalizes tag1 ref', () => {
+      const input: FrontMatterTag = 'tag1';
+      const expectedOutput: TagMetadata = {
+        inline: false,
+        label: tagsFile.tag1.label,
+        description: tagsFile.tag1.description,
+        permalink: `${tagsBaseRoutePath}/tag-1-permalink`,
+      };
+      expect(normalizeTag({tagsBaseRoutePath, tagsFile, tag: input})).toEqual(
+        expectedOutput,
+      );
+    });
+
+    it('normalizes tag2 ref', () => {
+      const input: FrontMatterTag = 'tag2';
+      const expectedOutput: TagMetadata = {
+        inline: false,
+        label: tagsFile.tag2.label,
+        description: tagsFile.tag2.description,
+        permalink: `${tagsBaseRoutePath}/tag-2-permalink`,
+      };
+      expect(normalizeTag({tagsBaseRoutePath, tagsFile, tag: input})).toEqual(
+        expectedOutput,
+      );
+    });
+
+    it('normalizes inline tag not declared in tags file', () => {
+      const input: FrontMatterTag = 'inlineTag';
+      const expectedOutput: TagMetadata = {
+        inline: true,
+        label: 'inlineTag',
+        description: undefined,
+        permalink: `${tagsBaseRoutePath}/inline-tag`,
+      };
+      expect(normalizeTag({tagsBaseRoutePath, tagsFile, tag: input})).toEqual(
+        expectedOutput,
+      );
+    });
+  });
+});
+
+describe('reportInlineTags', () => {
+  const tagsFile: TagsFile = {
+    hello: {
+      label: 'Hello',
+      permalink: '/hello',
+      description: undefined,
+    },
+    test: {
+      label: 'Test',
+      permalink: '/test',
+      description: undefined,
+    },
+    open: {
+      label: 'Open Source',
+      permalink: '/open',
+      description: undefined,
+    },
+  };
+
+  it('throw when inline tags found', () => {
+    const testFn = () =>
+      reportInlineTags({
+        tags: [
+          {
+            label: 'hello',
+            permalink: 'hello',
+            inline: true,
+            description: undefined,
+          },
+          {
+            label: 'world',
+            permalink: 'world',
+            inline: true,
+            description: undefined,
+          },
+        ],
+        source: 'wrong.md',
+        options: {onInlineTags: 'throw', tags: 'tags.yml'},
+      });
+
+    expect(testFn).toThrowErrorMatchingInlineSnapshot(
+      `"Tags [hello, world] used in wrong.md are not defined in tags.yml"`,
+    );
   });
 
-  it('normalizes complex string tag', () => {
-    const tagsPath = '/all/tags';
-    const input = 'some more Complex_tag';
-    const expectedOutput = {
-      label: 'some more Complex_tag',
-      permalink: `${tagsPath}/some-more-complex-tag`,
-    };
-    expect(normalizeFrontMatterTags(tagsPath, [input])).toEqual([
-      expectedOutput,
-    ]);
+  it('warn when docs has invalid tags', () => {
+    const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {});
+
+    reportInlineTags({
+      tags: [
+        {
+          label: 'hello',
+          permalink: 'hello',
+          inline: false,
+          description: undefined,
+        },
+        {
+          label: 'world',
+          permalink: 'world',
+          inline: true,
+          description: undefined,
+        },
+      ],
+      source: 'wrong.md',
+      options: {onInlineTags: 'warn', tags: 'tags.yml'},
+    });
+    expect(warnSpy).toHaveBeenCalledTimes(1);
+    expect(warnSpy.mock.calls).toMatchInlineSnapshot(`
+      [
+        [
+          "[WARNING] Tags [world] used in wrong.md are not defined in tags.yml",
+        ],
+      ]
+    `);
+
+    warnSpy.mockRestore();
   });
 
-  it('normalizes simple object tag', () => {
-    const tagsPath = '/all/tags';
-    const input = {label: 'tag', permalink: 'tagPermalink'};
-    const expectedOutput = {
-      label: 'tag',
-      permalink: `${tagsPath}/tagPermalink`,
-    };
-    expect(normalizeFrontMatterTags(tagsPath, [input])).toEqual([
-      expectedOutput,
-    ]);
+  it('ignore when docs has invalid tags', () => {
+    const errorSpy = jest.spyOn(console, 'error').mockImplementation(() => {});
+    const warnSpy = jest.spyOn(console, 'warn').mockImplementation(() => {});
+    const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {});
+
+    reportInlineTags({
+      tags: [
+        {
+          label: 'hello',
+          permalink: 'hello',
+          inline: false,
+          description: undefined,
+        },
+        {
+          label: 'world',
+          permalink: 'world',
+          inline: true,
+          description: undefined,
+        },
+      ],
+      source: 'wrong.md',
+      options: {onInlineTags: 'ignore', tags: 'tags.yml'},
+    });
+    expect(errorSpy).not.toHaveBeenCalled();
+    expect(warnSpy).not.toHaveBeenCalled();
+    expect(logSpy).not.toHaveBeenCalled();
+
+    errorSpy.mockRestore();
+    warnSpy.mockRestore();
+    logSpy.mockRestore();
   });
 
-  it('normalizes complex string tag with object tag', () => {
-    const tagsPath = '/all/tags';
-    const input = {
-      label: 'tag complex Label',
-      permalink: '/MoreComplex/Permalink',
-    };
-    const expectedOutput = {
-      label: 'tag complex Label',
-      permalink: `${tagsPath}/MoreComplex/Permalink`,
-    };
-    expect(normalizeFrontMatterTags(tagsPath, [input])).toEqual([
-      expectedOutput,
-    ]);
+  it('throw for unknown string and object tag', () => {
+    const frontmatter = ['open', 'world'];
+    const tags = frontmatter.map((tag) =>
+      normalizeTag({
+        tagsBaseRoutePath: '/tags',
+        tagsFile,
+        tag,
+      }),
+    );
+
+    const testFn = () =>
+      reportInlineTags({
+        tags,
+        source: 'default.md',
+        options: {
+          onInlineTags: 'throw',
+          tags: 'tags.yml',
+        },
+      });
+    expect(testFn).toThrowErrorMatchingInlineSnapshot(
+      `"Tags [world] used in default.md are not defined in tags.yml"`,
+    );
   });
 
-  type Input = Parameters<typeof normalizeFrontMatterTags>[1];
-  type Output = ReturnType<typeof normalizeFrontMatterTags>;
-
-  it('normalizes string list', () => {
-    const tagsPath = '/all/tags';
-    const input: Input = ['tag 1', 'tag-1', 'tag 3', 'tag1', 'tag-2'];
-    // Keep user input order but remove tags that lead to same permalink
-    const expectedOutput: Output = [
-      {
-        label: 'tag 1',
-        permalink: `${tagsPath}/tag-1`,
-      },
-      {
-        label: 'tag 3',
-        permalink: `${tagsPath}/tag-3`,
-      },
-      {
-        label: 'tag-2',
-        permalink: `${tagsPath}/tag-2`,
-      },
-    ];
-    expect(normalizeFrontMatterTags(tagsPath, input)).toEqual(expectedOutput);
-  });
-
-  it('succeeds for empty list', () => {
-    expect(normalizeFrontMatterTags('/foo')).toEqual([]);
-  });
-
-  it('normalizes complex mixed list', () => {
-    const tagsPath = '/all/tags';
-    const input: Input = [
-      'tag 1',
-      {label: 'tag-1', permalink: '/tag-1'},
-      'tag 3',
-      'tag1',
-      {label: 'tag 4', permalink: '/tag4Permalink'},
-    ];
-    // Keep user input order but remove tags that lead to same permalink
-    const expectedOutput: Output = [
-      {
-        label: 'tag 1',
-        permalink: `${tagsPath}/tag-1`,
-      },
-      {
-        label: 'tag 3',
-        permalink: `${tagsPath}/tag-3`,
-      },
-      {
-        label: 'tag 4',
-        permalink: `${tagsPath}/tag4Permalink`,
-      },
-    ];
-    expect(normalizeFrontMatterTags(tagsPath, input)).toEqual(expectedOutput);
+  it('does not throw when docs has valid tags', () => {
+    const frontmatter = ['open'];
+    const tags = frontmatter.map((tag) =>
+      normalizeTag({
+        tagsBaseRoutePath: '/tags',
+        tagsFile,
+        tag,
+      }),
+    );
+    const testFn = () =>
+      reportInlineTags({
+        tags,
+        source: 'wrong.md',
+        options: {
+          onInlineTags: 'throw',
+          tags: 'tags.yml',
+        },
+      });
+    expect(testFn).not.toThrow();
   });
 });
 
@@ -135,14 +302,23 @@ describe('groupTaggedItems', () => {
   type Output = ReturnType<typeof groupItems>;
 
   it('groups items by tag permalink', () => {
-    const tagGuide = {label: 'Guide', permalink: '/guide'};
-    const tagTutorial = {label: 'Tutorial', permalink: '/tutorial'};
-    const tagAPI = {label: 'API', permalink: '/api'};
+    const tagGuide = {
+      label: 'Guide',
+      permalink: '/guide',
+      description: undefined,
+    };
+    const tagTutorial = {
+      label: 'Tutorial',
+      permalink: '/tutorial',
+      description: undefined,
+    };
+    const tagAPI = {label: 'API', permalink: '/api', description: undefined};
 
     // This one will be grouped under same permalink and label is ignored
     const tagTutorialOtherLabel = {
       label: 'TutorialOtherLabel',
       permalink: '/tutorial',
+      description: undefined,
     };
 
     const item1: SomeTaggedItem = {

packages/docusaurus-utils/src/index.ts

@@ -55,10 +55,13 @@ export {
 export type {URLPath} from './urlUtils';
 export {
   type Tag,
+  type TagsFile,
+  type TagsFileInput,
+  type TagMetadata,
   type TagsListItem,
   type TagModule,
   type FrontMatterTag,
-  normalizeFrontMatterTags,
+  type TagsPluginOptions,
   groupTaggedItems,
   getTagVisibility,
 } from './tags';
@@ -120,3 +123,5 @@ export {
   type LastUpdateData,
   type FrontMatterLastUpdate,
 } from './lastUpdateUtils';
+
+export {normalizeTags, reportInlineTags} from './tags';

packages/docusaurus-utils/src/tags.ts

@@ -6,13 +6,34 @@
  */
 
 import _ from 'lodash';
+import logger from '@docusaurus/logger';
 import {normalizeUrl} from './urlUtils';
+import type {Optional} from 'utility-types';
 
-/** What the user configures. */
 export type Tag = {
+  /** The display label of a tag */
   label: string;
   /** Permalink to this tag's page, without the `/tags/` base path. */
   permalink: string;
+  /** An optional description of the tag */
+  description: string | undefined;
+};
+
+export type TagsFileInput = Record<string, Partial<Tag> | null>;
+
+export type TagsFile = Record<string, Tag>;
+
+// Tags plugins options shared between docs/blog
+export type TagsPluginOptions = {
+  // TODO allow option tags later? | TagsFile;
+  /** Path to the tags file. */
+  tags: string | false | null | undefined;
+  /** The behavior of Docusaurus when it found inline tags. */
+  onInlineTags: 'ignore' | 'log' | 'warn' | 'throw';
+};
+
+export type TagMetadata = Tag & {
+  inline: boolean;
 };
 
 /** What the tags list page should know about each tag. */
@@ -29,62 +50,126 @@ export type TagModule = TagsListItem & {
   unlisted: boolean;
 };
 
-export type FrontMatterTag = string | Tag;
+export type FrontMatterTag = string | Optional<Tag, 'description'>;
 
-function normalizeFrontMatterTag(
-  tagsPath: string,
+// We always apply tagsBaseRoutePath on purpose. For versioned docs, v1/doc.md
+// and v2/doc.md tags with custom permalinks don't lead to the same created
+// page. tagsBaseRoutePath is different for each doc version
+function normalizeTagPermalink({
+  tagsBaseRoutePath,
+  permalink,
+}: {
+  tagsBaseRoutePath: string;
+  permalink: string;
+}): string {
+  return normalizeUrl([tagsBaseRoutePath, permalink]);
+}
+
+function normalizeInlineTag(
+  tagsBaseRoutePath: string,
   frontMatterTag: FrontMatterTag,
-): Tag {
-  function toTagObject(tagString: string): Tag {
+): TagMetadata {
+  function toTagObject(tagString: string): TagMetadata {
     return {
+      inline: true,
       label: tagString,
       permalink: _.kebabCase(tagString),
+      description: undefined,
     };
   }
 
-  // TODO maybe make ensure the permalink is valid url path?
-  function normalizeTagPermalink(permalink: string): string {
-    // Note: we always apply tagsPath on purpose. For versioned docs, v1/doc.md
-    // and v2/doc.md tags with custom permalinks don't lead to the same created
-    // page. tagsPath is different for each doc version
-    return normalizeUrl([tagsPath, permalink]);
-  }
-
   const tag: Tag =
     typeof frontMatterTag === 'string'
       ? toTagObject(frontMatterTag)
-      : frontMatterTag;
+      : {...frontMatterTag, description: frontMatterTag.description};
 
   return {
+    inline: true,
     label: tag.label,
-    permalink: normalizeTagPermalink(tag.permalink),
+    permalink: normalizeTagPermalink({
+      permalink: tag.permalink,
+      tagsBaseRoutePath,
+    }),
+    description: tag.description,
   };
 }
 
-/**
- * Takes tag objects as they are defined in front matter, and normalizes each
- * into a standard tag object. The permalink is created by appending the
- * sluggified label to `tagsPath`. Front matter tags already containing
- * permalinks would still have `tagsPath` prepended.
- *
- * The result will always be unique by permalinks. The behavior with colliding
- * permalinks is undetermined.
- */
-export function normalizeFrontMatterTags(
-  /** Base path to append the tag permalinks to. */
-  tagsPath: string,
-  /** Can be `undefined`, so that we can directly pipe in `frontMatter.tags`. */
-  frontMatterTags: FrontMatterTag[] | undefined = [],
-): Tag[] {
-  const tags = frontMatterTags.map((tag) =>
-    normalizeFrontMatterTag(tagsPath, tag),
-  );
+export function normalizeTag({
+  tag,
+  tagsFile,
+  tagsBaseRoutePath,
+}: {
+  tag: FrontMatterTag;
+  tagsBaseRoutePath: string;
+  tagsFile: TagsFile | null;
+}): TagMetadata {
+  if (typeof tag === 'string') {
+    const tagDescription = tagsFile?.[tag];
+    if (tagDescription) {
+      // pre-defined tag from tags.yml
+      return {
+        inline: false,
+        label: tagDescription.label,
+        permalink: normalizeTagPermalink({
+          permalink: tagDescription.permalink,
+          tagsBaseRoutePath,
+        }),
+        description: tagDescription.description,
+      };
+    }
+  }
+  // legacy inline tag object, always inline, unknown because isn't a string
+  return normalizeInlineTag(tagsBaseRoutePath, tag);
+}
 
-  return _.uniqBy(tags, (tag) => tag.permalink);
+export function normalizeTags({
+  options,
+  source,
+  frontMatterTags,
+  tagsBaseRoutePath,
+  tagsFile,
+}: {
+  options: TagsPluginOptions;
+  source: string;
+  frontMatterTags: FrontMatterTag[] | undefined;
+  tagsBaseRoutePath: string;
+  tagsFile: TagsFile | null;
+}): TagMetadata[] {
+  const tags = (frontMatterTags ?? []).map((tag) =>
+    normalizeTag({tag, tagsBaseRoutePath, tagsFile}),
+  );
+  if (tagsFile !== null) {
+    reportInlineTags({tags, source, options});
+  }
+  return tags;
+}
+
+export function reportInlineTags({
+  tags,
+  source,
+  options,
+}: {
+  tags: TagMetadata[];
+  source: string;
+  options: TagsPluginOptions;
+}): void {
+  if (options.onInlineTags === 'ignore') {
+    return;
+  }
+  const inlineTags = tags.filter((tag) => tag.inline);
+  if (inlineTags.length > 0) {
+    const uniqueUnknownTags = [...new Set(inlineTags.map((tag) => tag.label))];
+    const tagListString = uniqueUnknownTags.join(', ');
+    logger.report(options.onInlineTags)(
+      `Tags [${tagListString}] used in ${source} are not defined in ${
+        options.tags ?? 'tags.yml'
+      }`,
+    );
+  }
 }
 
 type TaggedItemGroup<Item> = {
-  tag: Tag;
+  tag: TagMetadata;
   items: Item[];
 };
 
@@ -102,7 +187,7 @@ export function groupTaggedItems<Item>(
    * A callback telling me how to get the tags list of the current item. Usually
    * simply getting it from some metadata of the current item.
    */
-  getItemTags: (item: Item) => readonly Tag[],
+  getItemTags: (item: Item) => readonly TagMetadata[],
 ): {[permalink: string]: TaggedItemGroup<Item>} {
   const result: {[permalink: string]: TaggedItemGroup<Item>} = {};