feat(v2): add ability to set custom heading id (#4222)

* feat(v2): add ability to set custom heading id

* Add cli command

* Fix slugger

* write-heading-ids doc + add in commands/templates

* refactor + add tests for writeHeadingIds

* polish writeHeadingIds

* polish writeHeadingIds

* remove i18n goals todo section as the  remaining items are quite abstract/useless

* fix edge case with 2 md links in heading

* extract parseMarkdownHeadingId helper function

* refactor using the shared parseMarkdownHeadingId utility fn

* change logic of edge case

* Handle edge case

* Document explicit ids feature

Co-authored-by: slorber <lorber.sebastien@gmail.com>

packages/docusaurus/bin/docusaurus.js

@@ -23,6 +23,7 @@ const {
   serve,
   clear,
   writeTranslations,
+  writeHeadingIds,
 } = require('../lib');
 const {
   name,
@@ -284,6 +285,13 @@ cli
     },
   );
 
+cli
+  .command('write-heading-ids [contentDir]')
+  .description('Generate heading ids in Markdown content')
+  .action((siteDir = '.') => {
+    wrapCommand(writeHeadingIds)(siteDir);
+  });
+
 cli.arguments('<command>').action((cmd) => {
   cli.outputHelp();
   console.log(`  ${chalk.red(`\n  Unknown command ${chalk.yellow(cmd)}.`)}`);
@@ -299,6 +307,7 @@ function isInternalCommand(command) {
     'serve',
     'clear',
     'write-translations',
+    'write-heading-ids',
   ].includes(command);
 }
 

packages/docusaurus/package.json

@@ -73,6 +73,7 @@
     "express": "^4.17.1",
     "file-loader": "^6.2.0",
     "fs-extra": "^9.1.0",
+    "github-slugger": "^1.3.0",
     "globby": "^11.0.2",
     "html-minifier-terser": "^5.1.1",
     "html-tags": "^3.1.0",

packages/docusaurus/src/commands/__tests__/writeHeadingIds.test.ts

@@ -0,0 +1,118 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import {
+  transformMarkdownHeadingLine,
+  transformMarkdownContent,
+} from '../writeHeadingIds';
+import GithubSlugger from 'github-slugger';
+
+describe('transformMarkdownHeadingLine', () => {
+  test('throws when not a heading', () => {
+    expect(() =>
+      transformMarkdownHeadingLine('ABC', new GithubSlugger()),
+    ).toThrowErrorMatchingInlineSnapshot(
+      `"Line is not a markdown heading: ABC"`,
+    );
+  });
+
+  test('works for simple level-2 heading', () => {
+    expect(transformMarkdownHeadingLine('## ABC', new GithubSlugger())).toEqual(
+      '## ABC {#abc}',
+    );
+  });
+
+  test('works for simple level-3 heading', () => {
+    expect(transformMarkdownHeadingLine('###ABC', new GithubSlugger())).toEqual(
+      '###ABC {#abc}',
+    );
+  });
+
+  test('works for simple level-4 heading', () => {
+    expect(
+      transformMarkdownHeadingLine('#### ABC', new GithubSlugger()),
+    ).toEqual('#### ABC {#abc}');
+  });
+
+  test('works for simple level-2 heading', () => {
+    expect(transformMarkdownHeadingLine('## ABC', new GithubSlugger())).toEqual(
+      '## ABC {#abc}',
+    );
+  });
+
+  test('unwraps markdown links', () => {
+    const input = `## hello [facebook](https://facebook.com) [crowdin](https://crowdin.com/translate/docusaurus-v2/126/en-fr?filter=basic&value=0)`;
+    expect(transformMarkdownHeadingLine(input, new GithubSlugger())).toEqual(
+      `${input} {#hello-facebook-crowdin}`,
+    );
+  });
+
+  test('can slugify complex headings', () => {
+    const input = '## abc [Hello] How are you %Sébastien_-_$)( ## -56756';
+    expect(transformMarkdownHeadingLine(input, new GithubSlugger())).toEqual(
+      `${input} {#abc-hello-how-are-you-sébastien_-_---56756}`,
+    );
+  });
+
+  test('does not duplicate duplicate id', () => {
+    expect(
+      transformMarkdownHeadingLine(
+        '# hello world {#hello-world}',
+        new GithubSlugger(),
+      ),
+    ).toEqual('# hello world {#hello-world}');
+  });
+});
+
+describe('transformMarkdownContent', () => {
+  test('transform the headings', () => {
+    const input = `
+
+# Hello world
+
+## abc
+
+\`\`\`
+# Heading in code block
+\`\`\`
+
+## Hello world
+
+    \`\`\`
+    # Heading in escaped code block
+    \`\`\`
+
+### abc {#abc}
+
+    `;
+
+    // TODO the first heading should probably rather be slugified to abc-1
+    // otherwise we end up with 2 x "abc" anchors
+    // not sure how to implement that atm
+    const expected = `
+
+# Hello world {#hello-world}
+
+## abc {#abc}
+
+\`\`\`
+# Heading in code block
+\`\`\`
+
+## Hello world {#hello-world-1}
+
+    \`\`\`
+    # Heading in escaped code block
+    \`\`\`
+
+### abc {#abc}
+
+    `;
+
+    expect(transformMarkdownContent(input)).toEqual(expected);
+  });
+});

packages/docusaurus/src/commands/writeHeadingIds.ts

@@ -0,0 +1,132 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import globby from 'globby';
+import fs from 'fs-extra';
+import GithubSlugger from 'github-slugger';
+import chalk from 'chalk';
+import {loadContext, loadPluginConfigs} from '../server';
+import initPlugins from '../server/plugins/init';
+
+import {flatten} from 'lodash';
+import {parseMarkdownHeadingId} from '@docusaurus/utils';
+
+export function unwrapMarkdownLinks(line) {
+  return line.replace(/\[([^\]]+)\]\([^)]+\)/g, (match, p1) => p1);
+}
+
+function addHeadingId(line, slugger) {
+  let headingLevel = 0;
+  while (line.charAt(headingLevel) === '#') {
+    headingLevel += 1;
+  }
+
+  const headingText = line.slice(headingLevel).trimEnd();
+  const headingHashes = line.slice(0, headingLevel);
+  const slug = slugger.slug(unwrapMarkdownLinks(headingText));
+
+  return `${headingHashes}${headingText} {#${slug}}`;
+}
+
+export function transformMarkdownHeadingLine(
+  line: string,
+  slugger: GithubSlugger,
+) {
+  if (!line.startsWith('#')) {
+    throw new Error(`Line is not a markdown heading: ${line}`);
+  }
+
+  const parsedHeading = parseMarkdownHeadingId(line);
+
+  // Do not process if id is already therer
+  if (parsedHeading.id) {
+    return line;
+  }
+  return addHeadingId(line, slugger);
+}
+
+export function transformMarkdownLine(
+  line: string,
+  slugger: GithubSlugger,
+): string {
+  if (line.startsWith('#')) {
+    return transformMarkdownHeadingLine(line, slugger);
+  } else {
+    return line;
+  }
+}
+
+function transformMarkdownLines(lines: string[]): string[] {
+  let inCode = false;
+  const slugger = new GithubSlugger();
+
+  return lines.map((line) => {
+    if (line.startsWith('```')) {
+      inCode = !inCode;
+      return line;
+    } else {
+      if (inCode) {
+        return line;
+      }
+      return transformMarkdownLine(line, slugger);
+    }
+  });
+}
+
+export function transformMarkdownContent(content: string): string {
+  return transformMarkdownLines(content.split('\n')).join('\n');
+}
+
+async function transformMarkdownFile(
+  filepath: string,
+): Promise<string | undefined> {
+  const content = await fs.readFile(filepath, 'utf8');
+  const updatedContent = transformMarkdownLines(content.split('\n')).join('\n');
+  if (content !== updatedContent) {
+    await fs.writeFile(filepath, updatedContent);
+    return filepath;
+  }
+  return undefined;
+}
+
+// We only handle the "paths to watch" because these are the paths where the markdown files are
+// Also we don't want to transform the site md docs that do not belong to a content plugin
+// For example ./README.md should not be transformed
+async function getPathsToWatch(siteDir: string): Promise<string[]> {
+  const context = await loadContext(siteDir);
+  const pluginConfigs = loadPluginConfigs(context);
+  const plugins = await initPlugins({
+    pluginConfigs,
+    context,
+  });
+  return flatten(plugins.map((plugin) => plugin?.getPathsToWatch?.() ?? []));
+}
+
+export default async function writeHeadingIds(siteDir: string): Promise<void> {
+  const markdownFiles = await globby(await getPathsToWatch(siteDir), {
+    expandDirectories: ['**/*.{md,mdx}'],
+  });
+
+  const result = await Promise.all(markdownFiles.map(transformMarkdownFile));
+
+  const pathsModified = result.filter(Boolean) as string[];
+
+  if (pathsModified.length) {
+    console.log(
+      chalk.green(`Heading ids added to markdown files (${
+        pathsModified.length
+      }/${markdownFiles.length} files):
+- ${pathsModified.join('\n- ')}`),
+    );
+  } else {
+    console.log(
+      chalk.yellow(
+        `${markdownFiles.length} markdown files already have explicit heading ids`,
+      ),
+    );
+  }
+}

packages/docusaurus/src/index.ts

@@ -13,3 +13,4 @@ export {default as externalCommand} from './commands/external';
 export {default as serve} from './commands/serve';
 export {default as clear} from './commands/clear';
 export {default as writeTranslations} from './commands/writeTranslations';
+export {default as writeHeadingIds} from './commands/writeHeadingIds';