feat(v2): add ability to set custom heading id (#4222)

* feat(v2): add ability to set custom heading id

* Add cli command

* Fix slugger

* write-heading-ids doc + add in commands/templates

* refactor + add tests for writeHeadingIds

* polish writeHeadingIds

* polish writeHeadingIds

* remove i18n goals todo section as the  remaining items are quite abstract/useless

* fix edge case with 2 md links in heading

* extract parseMarkdownHeadingId helper function

* refactor using the shared parseMarkdownHeadingId utility fn

* change logic of edge case

* Handle edge case

* Document explicit ids feature

Co-authored-by: slorber <lorber.sebastien@gmail.com>

packages/docusaurus-init/templates/bootstrap/package.json

@@ -8,9 +8,10 @@
     "build": "docusaurus build",
     "swizzle": "docusaurus swizzle",
     "deploy": "docusaurus deploy",
-    "serve": "docusaurus serve",
     "clear": "docusaurus clear",
-    "write-translations": "write-translations"
+    "serve": "docusaurus serve",
+    "write-translations": "docusaurus write-translations",
+    "write-heading-ids": "docusaurus write-heading-ids"
   },
   "dependencies": {
     "@docusaurus/core": "2.0.0-alpha.70",

packages/docusaurus-init/templates/classic/package.json

@@ -8,9 +8,10 @@
     "build": "docusaurus build",
     "swizzle": "docusaurus swizzle",
     "deploy": "docusaurus deploy",
-    "serve": "docusaurus serve",
     "clear": "docusaurus clear",
-    "write-translations": "write-translations"
+    "serve": "docusaurus serve",
+    "write-translations": "docusaurus write-translations",
+    "write-heading-ids": "docusaurus write-heading-ids"
   },
   "dependencies": {
     "@docusaurus/core": "2.0.0-alpha.70",

packages/docusaurus-init/templates/facebook/package.json

@@ -8,9 +8,10 @@
     "build": "docusaurus build",
     "swizzle": "docusaurus swizzle",
     "deploy": "docusaurus deploy",
-    "serve": "docusaurus serve",
     "clear": "docusaurus clear",
-    "write-translations": "write-translations",
+    "serve": "docusaurus serve",
+    "write-translations": "docusaurus write-translations",
+    "write-heading-ids": "docusaurus write-heading-ids",
     "ci": "yarn lint && yarn prettier:diff",
     "lint": "eslint --cache \"**/*.js\" && stylelint \"**/*.css\"",
     "prettier": "prettier --config .prettierrc --write \"**/*.{js,jsx,ts,tsx,md,mdx}\"",

packages/docusaurus-mdx-loader/src/index.js

@@ -11,7 +11,7 @@ const mdx = require('@mdx-js/mdx');
 const emoji = require('remark-emoji');
 const matter = require('gray-matter');
 const stringifyObject = require('stringify-object');
-const slug = require('./remark/slug');
+const headings = require('./remark/headings');
 const toc = require('./remark/toc');
 const unwrapMdxCodeBlocks = require('./remark/unwrapMdxCodeBlocks');
 const transformImage = require('./remark/transformImage');
@@ -19,7 +19,7 @@ const transformLinks = require('./remark/transformLinks');
 
 const DEFAULT_OPTIONS = {
   rehypePlugins: [],
-  remarkPlugins: [unwrapMdxCodeBlocks, emoji, slug, toc],
+  remarkPlugins: [unwrapMdxCodeBlocks, emoji, headings, toc],
 };
 
 module.exports = async function docusaurusMdxLoader(fileString) {

packages/docusaurus-mdx-loader/src/remark/headings/__tests__/index.test.js

@@ -5,13 +5,15 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-/* Based on remark-slug (https://github.com/remarkjs/remark-slug) */
+/* Based on remark-slug (https://github.com/remarkjs/remark-slug) and gatsby-remark-autolink-headers (https://github.com/gatsbyjs/gatsby/blob/master/packages/gatsby-remark-autolink-headers) */
 
 /* eslint-disable no-param-reassign */
 
 import remark from 'remark';
 import u from 'unist-builder';
 import removePosition from 'unist-util-remove-position';
+import toString from 'mdast-util-to-string';
+import visit from 'unist-util-visit';
 import slug from '../index';
 
 function process(doc, plugins = []) {
@@ -27,7 +29,7 @@ function heading(label, id) {
   );
 }
 
-describe('slug plugin', () => {
+describe('headings plugin', () => {
   test('should patch `id`s and `data.hProperties.id', () => {
     const result = process('# Normal\n\n## Table of Contents\n\n# Baz\n');
     const expected = u('root', [
@@ -157,7 +159,7 @@ describe('slug plugin', () => {
     expect(result).toEqual(expected);
   });
 
-  test('should create GitHub slugs', () => {
+  test('should create GitHub-style headings ids', () => {
     const result = process(
       [
         '## I ♥ unicode',
@@ -225,7 +227,7 @@ describe('slug plugin', () => {
     expect(result).toEqual(expected);
   });
 
-  test('should generate slug from only text contents of headings if they contains HTML tags', () => {
+  test('should generate id from only text contents of headings if they contains HTML tags', () => {
     const result = process('# <span class="normal-header">Normal</span>\n');
     const expected = u('root', [
       u(
@@ -244,4 +246,70 @@ describe('slug plugin', () => {
 
     expect(result).toEqual(expected);
   });
+
+  test('should create custom headings ids', () => {
+    const result = process(`
+# Heading One {#custom_h1}
+
+## Heading Two {#custom-heading-two}
+
+# With *Bold* {#custom-withbold}
+
+# With *Bold* hello{#custom-withbold-hello}
+
+# With *Bold* hello2 {#custom-withbold-hello2}
+
+# Snake-cased ID {#this_is_custom_id}
+
+# No custom ID
+
+# {#id-only}
+
+# {#text-after} custom ID
+  `);
+
+    const headers = [];
+    visit(result, 'heading', (node) => {
+      headers.push({text: toString(node), id: node.data.id});
+    });
+
+    expect(headers).toEqual([
+      {
+        id: 'custom_h1',
+        text: 'Heading One',
+      },
+      {
+        id: 'custom-heading-two',
+        text: 'Heading Two',
+      },
+      {
+        id: 'custom-withbold',
+        text: 'With Bold',
+      },
+      {
+        id: 'custom-withbold-hello',
+        text: 'With Bold hello',
+      },
+      {
+        id: 'custom-withbold-hello2',
+        text: 'With Bold hello2',
+      },
+      {
+        id: 'this_is_custom_id',
+        text: 'Snake-cased ID',
+      },
+      {
+        id: 'no-custom-id',
+        text: 'No custom ID',
+      },
+      {
+        id: 'id-only',
+        text: '',
+      },
+      {
+        id: 'text-after-custom-id',
+        text: '{#text-after} custom ID',
+      },
+    ]);
+  });
 });

packages/docusaurus-mdx-loader/src/remark/headings/index.js

@@ -0,0 +1,74 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/* Based on remark-slug (https://github.com/remarkjs/remark-slug) and gatsby-remark-autolink-headers (https://github.com/gatsbyjs/gatsby/blob/master/packages/gatsby-remark-autolink-headers) */
+
+const {parseMarkdownHeadingId} = require('@docusaurus/utils');
+const visit = require('unist-util-visit');
+const toString = require('mdast-util-to-string');
+const slugs = require('github-slugger')();
+
+function headings() {
+  const transformer = (ast) => {
+    slugs.reset();
+
+    function visitor(headingNode) {
+      const data = headingNode.data || (headingNode.data = {}); // eslint-disable-line
+      const properties = data.hProperties || (data.hProperties = {});
+      let {id} = properties;
+
+      if (id) {
+        id = slugs.slug(id, true);
+      } else {
+        const headingTextNodes = headingNode.children.filter(
+          ({type}) => !['html', 'jsx'].includes(type),
+        );
+        const heading = toString(
+          headingTextNodes.length > 0
+            ? {children: headingTextNodes}
+            : headingNode,
+        );
+
+        // Support explicit heading IDs
+        const parsedHeading = parseMarkdownHeadingId(heading);
+
+        id = parsedHeading.id || slugs.slug(heading);
+
+        if (parsedHeading.id) {
+          // When there's an id, it is always in the last child node
+          // Sometimes heading is in multiple "parts" (** syntax creates a child node):
+          // ## part1 *part2* part3 {#id}
+          const lastNode =
+            headingNode.children[headingNode.children.length - 1];
+
+          if (headingNode.children.length > 1) {
+            const lastNodeText = parseMarkdownHeadingId(lastNode.value).text;
+            // When last part contains test+id, remove the id
+            if (lastNodeText) {
+              lastNode.value = lastNodeText;
+            }
+            // When last part contains only the id: completely remove that node
+            else {
+              headingNode.children.pop();
+            }
+          } else {
+            lastNode.value = parsedHeading.text;
+          }
+        }
+      }
+
+      data.id = id;
+      properties.id = id;
+    }
+
+    visit(ast, 'heading', visitor);
+  };
+
+  return transformer;
+}
+
+module.exports = headings;

packages/docusaurus-mdx-loader/src/remark/slug/index.js

@@ -1,46 +0,0 @@
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-/* Based on remark-slug (https://github.com/remarkjs/remark-slug) */
-
-const visit = require('unist-util-visit');
-const toString = require('mdast-util-to-string');
-const slugs = require('github-slugger')();
-
-function slug() {
-  const transformer = (ast) => {
-    slugs.reset();
-
-    function visitor(headingNode) {
-      const data = headingNode.data || (headingNode.data = {}); // eslint-disable-line
-      const properties = data.hProperties || (data.hProperties = {});
-      let {id} = properties;
-
-      if (id) {
-        id = slugs.slug(id, true);
-      } else {
-        const headingTextNodes = headingNode.children.filter(
-          ({type}) => !['html', 'jsx'].includes(type),
-        );
-        const normalizedHeadingNode =
-          headingTextNodes.length > 0
-            ? {children: headingTextNodes}
-            : headingNode;
-        id = slugs.slug(toString(normalizedHeadingNode));
-      }
-
-      data.id = id;
-      properties.id = id;
-    }
-
-    visit(ast, 'heading', visitor);
-  };
-
-  return transformer;
-}
-
-module.exports = slug;

packages/docusaurus-mdx-loader/src/remark/toc/__tests__/index.test.js

@@ -10,13 +10,13 @@ import remark from 'remark';
 import mdx from 'remark-mdx';
 import vfile from 'to-vfile';
 import plugin from '../index';
-import slug from '../../slug/index';
+import headings from '../../headings/index';
 
 const processFixture = async (name, options) => {
   const path = join(__dirname, 'fixtures', `${name}.md`);
   const file = await vfile.read(path);
   const result = await remark()
-    .use(slug)
+    .use(headings)
     .use(mdx)
     .use(plugin, options)
     .process(file);

packages/docusaurus-mdx-loader/src/remark/transformImage/__tests__/index.test.js

@@ -10,13 +10,13 @@ import remark from 'remark';
 import mdx from 'remark-mdx';
 import vfile from 'to-vfile';
 import plugin from '../index';
-import slug from '../../slug/index';
+import headings from '../../headings/index';
 
 const processFixture = async (name, options) => {
   const path = join(__dirname, 'fixtures', `${name}.md`);
   const file = await vfile.read(path);
   const result = await remark()
-    .use(slug)
+    .use(headings)
     .use(mdx)
     .use(plugin, {...options, filePath: path})
     .process(file);

packages/docusaurus-utils/package.json

@@ -19,6 +19,7 @@
   "license": "MIT",
   "dependencies": {
     "@docusaurus/types": "2.0.0-alpha.70",
+    "@types/github-slugger": "^1.3.0",
     "chalk": "^4.1.0",
     "escape-string-regexp": "^4.0.0",
     "fs-extra": "^9.1.0",

packages/docusaurus-utils/src/__tests__/index.test.ts

@@ -35,6 +35,7 @@ import {
   getFolderContainingFile,
   updateTranslationFileMessages,
   readDefaultCodeTranslationMessages,
+  parseMarkdownHeadingId,
 } from '../index';
 import {sum} from 'lodash';
 
@@ -806,3 +807,51 @@ describe('readDefaultCodeTranslationMessages', () => {
     ).resolves.toEqual(await readAsJSON('en.json'));
   });
 });
+
+describe('parseMarkdownHeadingId', () => {
+  test('can parse simple heading without id', () => {
+    expect(parseMarkdownHeadingId('## Some heading')).toEqual({
+      text: '## Some heading',
+      id: undefined,
+    });
+  });
+
+  test('can parse simple heading with id', () => {
+    expect(parseMarkdownHeadingId('## Some heading {#custom-_id}')).toEqual({
+      text: '## Some heading',
+      id: 'custom-_id',
+    });
+  });
+
+  test('can parse heading not ending with the id', () => {
+    expect(parseMarkdownHeadingId('## {#custom-_id} Some heading')).toEqual({
+      text: '## {#custom-_id} Some heading',
+      id: undefined,
+    });
+  });
+
+  test('can parse heading with multiple id', () => {
+    expect(parseMarkdownHeadingId('## Some heading {#id1} {#id2}')).toEqual({
+      text: '## Some heading {#id1}',
+      id: 'id2',
+    });
+  });
+
+  test('can parse heading with link and id', () => {
+    expect(
+      parseMarkdownHeadingId(
+        '## Some heading [facebook](https://facebook.com) {#id}',
+      ),
+    ).toEqual({
+      text: '## Some heading [facebook](https://facebook.com)',
+      id: 'id',
+    });
+  });
+
+  test('can parse heading with only id', () => {
+    expect(parseMarkdownHeadingId('## {#id}')).toEqual({
+      text: '##',
+      id: 'id',
+    });
+  });
+});

packages/docusaurus-utils/src/index.ts

@@ -642,3 +642,23 @@ export function getDateTimeFormat(locale: string) {
     : // eslint-disable-next-line @typescript-eslint/no-var-requires
       require('intl').DateTimeFormat;
 }
+
+// Input: ## Some heading {#some-heading}
+// Output: {text: "## Some heading", id: "some-heading"}
+export function parseMarkdownHeadingId(
+  heading: string,
+): {
+  text: string;
+  id?: string;
+} {
+  const customHeadingIdRegex = /^(.*?)\s*\{#([\w-]+)\}$/;
+  const matches = customHeadingIdRegex.exec(heading);
+  if (matches) {
+    return {
+      text: matches[1],
+      id: matches[2],
+    };
+  } else {
+    return {text: heading, id: undefined};
+  }
+}

packages/docusaurus/bin/docusaurus.js

@@ -23,6 +23,7 @@ const {
   serve,
   clear,
   writeTranslations,
+  writeHeadingIds,
 } = require('../lib');
 const {
   name,
@@ -284,6 +285,13 @@ cli
     },
   );
 
+cli
+  .command('write-heading-ids [contentDir]')
+  .description('Generate heading ids in Markdown content')
+  .action((siteDir = '.') => {
+    wrapCommand(writeHeadingIds)(siteDir);
+  });
+
 cli.arguments('<command>').action((cmd) => {
   cli.outputHelp();
   console.log(`  ${chalk.red(`\n  Unknown command ${chalk.yellow(cmd)}.`)}`);
@@ -299,6 +307,7 @@ function isInternalCommand(command) {
     'serve',
     'clear',
     'write-translations',
+    'write-heading-ids',
   ].includes(command);
 }
 

packages/docusaurus/package.json

@@ -73,6 +73,7 @@
     "express": "^4.17.1",
     "file-loader": "^6.2.0",
     "fs-extra": "^9.1.0",
+    "github-slugger": "^1.3.0",
     "globby": "^11.0.2",
     "html-minifier-terser": "^5.1.1",
     "html-tags": "^3.1.0",

packages/docusaurus/src/commands/__tests__/writeHeadingIds.test.ts

@@ -0,0 +1,118 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import {
+  transformMarkdownHeadingLine,
+  transformMarkdownContent,
+} from '../writeHeadingIds';
+import GithubSlugger from 'github-slugger';
+
+describe('transformMarkdownHeadingLine', () => {
+  test('throws when not a heading', () => {
+    expect(() =>
+      transformMarkdownHeadingLine('ABC', new GithubSlugger()),
+    ).toThrowErrorMatchingInlineSnapshot(
+      `"Line is not a markdown heading: ABC"`,
+    );
+  });
+
+  test('works for simple level-2 heading', () => {
+    expect(transformMarkdownHeadingLine('## ABC', new GithubSlugger())).toEqual(
+      '## ABC {#abc}',
+    );
+  });
+
+  test('works for simple level-3 heading', () => {
+    expect(transformMarkdownHeadingLine('###ABC', new GithubSlugger())).toEqual(
+      '###ABC {#abc}',
+    );
+  });
+
+  test('works for simple level-4 heading', () => {
+    expect(
+      transformMarkdownHeadingLine('#### ABC', new GithubSlugger()),
+    ).toEqual('#### ABC {#abc}');
+  });
+
+  test('works for simple level-2 heading', () => {
+    expect(transformMarkdownHeadingLine('## ABC', new GithubSlugger())).toEqual(
+      '## ABC {#abc}',
+    );
+  });
+
+  test('unwraps markdown links', () => {
+    const input = `## hello [facebook](https://facebook.com) [crowdin](https://crowdin.com/translate/docusaurus-v2/126/en-fr?filter=basic&value=0)`;
+    expect(transformMarkdownHeadingLine(input, new GithubSlugger())).toEqual(
+      `${input} {#hello-facebook-crowdin}`,
+    );
+  });
+
+  test('can slugify complex headings', () => {
+    const input = '## abc [Hello] How are you %Sébastien_-_$)( ## -56756';
+    expect(transformMarkdownHeadingLine(input, new GithubSlugger())).toEqual(
+      `${input} {#abc-hello-how-are-you-sébastien_-_---56756}`,
+    );
+  });
+
+  test('does not duplicate duplicate id', () => {
+    expect(
+      transformMarkdownHeadingLine(
+        '# hello world {#hello-world}',
+        new GithubSlugger(),
+      ),
+    ).toEqual('# hello world {#hello-world}');
+  });
+});
+
+describe('transformMarkdownContent', () => {
+  test('transform the headings', () => {
+    const input = `
+
+# Hello world
+
+## abc
+
+\`\`\`
+# Heading in code block
+\`\`\`
+
+## Hello world
+
+    \`\`\`
+    # Heading in escaped code block
+    \`\`\`
+
+### abc {#abc}
+
+    `;
+
+    // TODO the first heading should probably rather be slugified to abc-1
+    // otherwise we end up with 2 x "abc" anchors
+    // not sure how to implement that atm
+    const expected = `
+
+# Hello world {#hello-world}
+
+## abc {#abc}
+
+\`\`\`
+# Heading in code block
+\`\`\`
+
+## Hello world {#hello-world-1}
+
+    \`\`\`
+    # Heading in escaped code block
+    \`\`\`
+
+### abc {#abc}
+
+    `;
+
+    expect(transformMarkdownContent(input)).toEqual(expected);
+  });
+});

packages/docusaurus/src/commands/writeHeadingIds.ts

@@ -0,0 +1,132 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import globby from 'globby';
+import fs from 'fs-extra';
+import GithubSlugger from 'github-slugger';
+import chalk from 'chalk';
+import {loadContext, loadPluginConfigs} from '../server';
+import initPlugins from '../server/plugins/init';
+
+import {flatten} from 'lodash';
+import {parseMarkdownHeadingId} from '@docusaurus/utils';
+
+export function unwrapMarkdownLinks(line) {
+  return line.replace(/\[([^\]]+)\]\([^)]+\)/g, (match, p1) => p1);
+}
+
+function addHeadingId(line, slugger) {
+  let headingLevel = 0;
+  while (line.charAt(headingLevel) === '#') {
+    headingLevel += 1;
+  }
+
+  const headingText = line.slice(headingLevel).trimEnd();
+  const headingHashes = line.slice(0, headingLevel);
+  const slug = slugger.slug(unwrapMarkdownLinks(headingText));
+
+  return `${headingHashes}${headingText} {#${slug}}`;
+}
+
+export function transformMarkdownHeadingLine(
+  line: string,
+  slugger: GithubSlugger,
+) {
+  if (!line.startsWith('#')) {
+    throw new Error(`Line is not a markdown heading: ${line}`);
+  }
+
+  const parsedHeading = parseMarkdownHeadingId(line);
+
+  // Do not process if id is already therer
+  if (parsedHeading.id) {
+    return line;
+  }
+  return addHeadingId(line, slugger);
+}
+
+export function transformMarkdownLine(
+  line: string,
+  slugger: GithubSlugger,
+): string {
+  if (line.startsWith('#')) {
+    return transformMarkdownHeadingLine(line, slugger);
+  } else {
+    return line;
+  }
+}
+
+function transformMarkdownLines(lines: string[]): string[] {
+  let inCode = false;
+  const slugger = new GithubSlugger();
+
+  return lines.map((line) => {
+    if (line.startsWith('```')) {
+      inCode = !inCode;
+      return line;
+    } else {
+      if (inCode) {
+        return line;
+      }
+      return transformMarkdownLine(line, slugger);
+    }
+  });
+}
+
+export function transformMarkdownContent(content: string): string {
+  return transformMarkdownLines(content.split('\n')).join('\n');
+}
+
+async function transformMarkdownFile(
+  filepath: string,
+): Promise<string | undefined> {
+  const content = await fs.readFile(filepath, 'utf8');
+  const updatedContent = transformMarkdownLines(content.split('\n')).join('\n');
+  if (content !== updatedContent) {
+    await fs.writeFile(filepath, updatedContent);
+    return filepath;
+  }
+  return undefined;
+}
+
+// We only handle the "paths to watch" because these are the paths where the markdown files are
+// Also we don't want to transform the site md docs that do not belong to a content plugin
+// For example ./README.md should not be transformed
+async function getPathsToWatch(siteDir: string): Promise<string[]> {
+  const context = await loadContext(siteDir);
+  const pluginConfigs = loadPluginConfigs(context);
+  const plugins = await initPlugins({
+    pluginConfigs,
+    context,
+  });
+  return flatten(plugins.map((plugin) => plugin?.getPathsToWatch?.() ?? []));
+}
+
+export default async function writeHeadingIds(siteDir: string): Promise<void> {
+  const markdownFiles = await globby(await getPathsToWatch(siteDir), {
+    expandDirectories: ['**/*.{md,mdx}'],
+  });
+
+  const result = await Promise.all(markdownFiles.map(transformMarkdownFile));
+
+  const pathsModified = result.filter(Boolean) as string[];
+
+  if (pathsModified.length) {
+    console.log(
+      chalk.green(`Heading ids added to markdown files (${
+        pathsModified.length
+      }/${markdownFiles.length} files):
+- ${pathsModified.join('\n- ')}`),
+    );
+  } else {
+    console.log(
+      chalk.yellow(
+        `${markdownFiles.length} markdown files already have explicit heading ids`,
+      ),
+    );
+  }
+}

packages/docusaurus/src/index.ts

@@ -13,3 +13,4 @@ export {default as externalCommand} from './commands/external';
 export {default as serve} from './commands/serve';
 export {default as clear} from './commands/clear';
 export {default as writeTranslations} from './commands/writeTranslations';
+export {default as writeHeadingIds} from './commands/writeHeadingIds';