feat(v2): read first heading as title and use it in front-matter (#4485)

* feat(v2): read first heading as title and pass it to front-matter

* fix(v2): always trim content after extracting front-matter

* fix(v2): remove heading from rss and keep duplicate heading

* fix(v2): rollback some unnecessary comment changes

* test(v2): add unit tests to blog

* test(v2): add unit tests to docs

* test(v2): correct issue on windows

* test(v2): add additional test cases

packages/docusaurus-mdx-loader/package.json

@@ -8,7 +8,8 @@
     "access": "public"
   },
   "scripts": {
-    "build": "tsc"
+    "build": "tsc",
+    "watch": "tsc --watch"
   },
   "repository": {
     "type": "git",
@@ -27,7 +28,6 @@
     "file-loader": "^6.2.0",
     "fs-extra": "^9.1.0",
     "github-slugger": "^1.3.0",
-    "gray-matter": "^4.0.2",
     "loader-utils": "^2.0.0",
     "mdast-util-to-string": "^2.0.0",
     "remark-emoji": "^2.1.0",

packages/docusaurus-mdx-loader/src/index.js

@@ -9,7 +9,7 @@ const {getOptions} = require('loader-utils');
 const {readFile} = require('fs-extra');
 const mdx = require('@mdx-js/mdx');
 const emoji = require('remark-emoji');
-const matter = require('gray-matter');
+const {readFrontMatter} = require('@docusaurus/utils');
 const stringifyObject = require('stringify-object');
 const headings = require('./remark/headings');
 const toc = require('./remark/toc');
@@ -24,9 +24,15 @@ const DEFAULT_OPTIONS = {
 
 module.exports = async function docusaurusMdxLoader(fileString) {
   const callback = this.async();
-
-  const {data, content} = matter(fileString);
   const reqOptions = getOptions(this) || {};
+
+  const {frontMatter, content, hasFrontMatter} = readFrontMatter(
+    fileString,
+    this.resourcePath,
+    {},
+    reqOptions.removeTitleHeading,
+  );
+
   const options = {
     ...reqOptions,
     remarkPlugins: [
@@ -58,7 +64,7 @@ module.exports = async function docusaurusMdxLoader(fileString) {
     return callback(err);
   }
 
-  let exportStr = `export const frontMatter = ${stringifyObject(data)};`;
+  let exportStr = `export const frontMatter = ${stringifyObject(frontMatter)};`;
 
   // Read metadata for this MDX and export it.
   if (options.metadataPath && typeof options.metadataPath === 'function') {
@@ -77,10 +83,7 @@ module.exports = async function docusaurusMdxLoader(fileString) {
     options.forbidFrontMatter &&
     typeof options.forbidFrontMatter === 'function'
   ) {
-    if (
+    if (options.forbidFrontMatter(this.resourcePath) && hasFrontMatter) {
-      options.forbidFrontMatter(this.resourcePath) &&
-      Object.keys(data).length > 0
-    ) {
       return callback(new Error(`Front matter is forbidden in this file`));
     }
   }

packages/docusaurus-plugin-content-blog/src/__tests__/__fixtures__/website/blog/heading-as-title.md

@@ -0,0 +1,5 @@
+---
+date: 2019-01-02
+---
+
+# some heading

packages/docusaurus-plugin-content-blog/src/__tests__/index.test.ts

@@ -164,12 +164,33 @@ describe('loadBlog', () => {
       tags: [],
       truncated: false,
     });
+
+    expect({
+      ...blogPosts.find((v) => v.metadata.title === 'some heading')!.metadata,
+      prevItem: undefined,
+    }).toEqual({
+      editUrl: `${BaseEditUrl}/blog/heading-as-title.md`,
+      permalink: '/blog/heading-as-title',
+      readingTime: 0,
+      source: path.posix.join('@site', PluginPath, 'heading-as-title.md'),
+      title: 'some heading',
+      description: '',
+      date: new Date('2019-01-02'),
+      formattedDate: 'January 2, 2019',
+      prevItem: undefined,
+      tags: [],
+      nextItem: {
+        permalink: '/blog/date-matter',
+        title: 'date-matter',
+      },
+      truncated: false,
+    });
   });
 
   test('simple website blog dates localized', async () => {
     const siteDir = path.join(__dirname, '__fixtures__', 'website');
     const blogPostsFrench = await getBlogPosts(siteDir, {}, getI18n('fr'));
-    expect(blogPostsFrench).toHaveLength(5);
+    expect(blogPostsFrench).toHaveLength(6);
     expect(blogPostsFrench[0].metadata.formattedDate).toMatchInlineSnapshot(
       `"16 août 2020"`,
     );
@@ -180,9 +201,12 @@ describe('loadBlog', () => {
       `"27 février 2020"`,
     );
     expect(blogPostsFrench[3].metadata.formattedDate).toMatchInlineSnapshot(
-      `"1 janvier 2019"`,
+      `"2 janvier 2019"`,
     );
     expect(blogPostsFrench[4].metadata.formattedDate).toMatchInlineSnapshot(
+      `"1 janvier 2019"`,
+    );
+    expect(blogPostsFrench[5].metadata.formattedDate).toMatchInlineSnapshot(
       `"14 décembre 2018"`,
     );
   });
@@ -212,7 +236,7 @@ describe('loadBlog', () => {
       expect(blogPost.metadata.editUrl).toEqual(hardcodedEditUrl);
     });
 
-    expect(editUrlFunction).toHaveBeenCalledTimes(5);
+    expect(editUrlFunction).toHaveBeenCalledTimes(6);
     expect(editUrlFunction).toHaveBeenCalledWith({
       blogDirPath: 'blog',
       blogPath: 'date-matter.md',
@@ -243,6 +267,12 @@ describe('loadBlog', () => {
       permalink: '/blog/2018/12/14/Happy-First-Birthday-Slash',
       locale: 'en',
     });
+    expect(editUrlFunction).toHaveBeenCalledWith({
+      blogDirPath: 'blog',
+      blogPath: 'heading-as-title.md',
+      locale: 'en',
+      permalink: '/blog/heading-as-title',
+    });
   });
 
   test('draft blog post not exists in production build', async () => {

packages/docusaurus-plugin-content-docs/src/__tests__/__fixtures__/simple-site/docs/headingAsTitle.md

@@ -0,0 +1 @@
+# My heading as title

packages/docusaurus-plugin-content-docs/src/__tests__/__snapshots__/index.test.ts.snap

@@ -67,6 +67,11 @@ Object {
               "path": "/docs/foo/bazSlug.html",
               "sidebar": "docs",
             },
+            Object {
+              "id": "headingAsTitle",
+              "path": "/docs/headingAsTitle",
+              "sidebar": undefined,
+            },
             Object {
               "id": "hello",
               "path": "/docs/",
@@ -172,6 +177,17 @@ Object {
     \\"title\\": \\"Hello, World !\\",
     \\"permalink\\": \\"/docs/\\"
   }
+}",
+  "site-docs-heading-as-title-md-c6d.json": "{
+  \\"unversionedId\\": \\"headingAsTitle\\",
+  \\"id\\": \\"headingAsTitle\\",
+  \\"isDocsHomePage\\": false,
+  \\"title\\": \\"My heading as title\\",
+  \\"description\\": \\"\\",
+  \\"source\\": \\"@site/docs/headingAsTitle.md\\",
+  \\"slug\\": \\"/headingAsTitle\\",
+  \\"permalink\\": \\"/docs/headingAsTitle\\",
+  \\"version\\": \\"current\\"
 }",
   "site-docs-hello-md-9df.json": "{
   \\"unversionedId\\": \\"hello\\",
@@ -383,6 +399,11 @@ Object {
               "path": "/docs/foo/bazSlug.html",
               "sidebar": "docs",
             },
+            Object {
+              "id": "headingAsTitle",
+              "path": "/docs/headingAsTitle",
+              "sidebar": undefined,
+            },
             Object {
               "id": "hello",
               "path": "/docs/",
@@ -494,6 +515,14 @@ Array [
         },
         "path": "/docs/foo/bazSlug.html",
       },
+      Object {
+        "component": "@theme/DocItem",
+        "exact": true,
+        "modules": Object {
+          "content": "@site/docs/headingAsTitle.md",
+        },
+        "path": "/docs/headingAsTitle",
+      },
       Object {
         "component": "@theme/DocItem",
         "exact": true,
@@ -579,6 +608,7 @@ These sidebar document ids do not exist:
 Available document ids=
 - foo/bar
 - foo/baz
+- headingAsTitle
 - hello
 - ipsum
 - lorem

packages/docusaurus-plugin-content-docs/src/__tests__/docs.test.ts

@@ -160,6 +160,7 @@ describe('simple site', () => {
         'rootRelativeSlug.md',
         'rootResolvedSlug.md',
         'rootTryToEscapeSlug.md',
+        'headingAsTitle.md',
         'foo/bar.md',
         'foo/baz.md',
         'slugs/absoluteSlug.md',

packages/docusaurus-plugin-content-docs/src/__tests__/index.test.ts

@@ -211,6 +211,7 @@ describe('simple website', () => {
     expect(isMatch('docs/hello.js', matchPattern)).toEqual(false);
     expect(isMatch('docs/super.mdl', matchPattern)).toEqual(false);
     expect(isMatch('docs/mdx', matchPattern)).toEqual(false);
+    expect(isMatch('docs/headingAsTitle.md', matchPattern)).toEqual(true);
     expect(isMatch('sidebars.json', matchPattern)).toEqual(true);
     expect(isMatch('versioned_docs/hello.md', matchPattern)).toEqual(false);
     expect(isMatch('hello.md', matchPattern)).toEqual(false);

packages/docusaurus-plugin-content-docs/src/docs.ts

@@ -119,7 +119,7 @@ export function processDocMetadata({
   // ex: myDoc -> .
   const docsFileDirName = path.dirname(source);
 
-  const {frontMatter = {}, excerpt} = parseMarkdownString(content);
+  const {frontMatter = {}, excerpt} = parseMarkdownString(content, source);
   const {
     sidebar_label: sidebarLabel,
     custom_edit_url: customEditURL,

packages/docusaurus-plugin-content-pages/src/index.ts

@@ -223,6 +223,7 @@ export default function pluginContentPages(
                     rehypePlugins,
                     beforeDefaultRehypePlugins,
                     beforeDefaultRemarkPlugins,
+                    removeTitleHeading: false,
                     staticDir: path.join(siteDir, STATIC_DIR_NAME),
                     // Note that metadataPath must be the same/in-sync as
                     // the path from createData for each MDX.

packages/docusaurus-utils/package.json

@@ -30,5 +30,9 @@
   },
   "engines": {
     "node": ">=12.13.0"
+  },
+  "devDependencies": {
+    "@types/dedent": "^0.7.0",
+    "dedent": "^0.7.0"
   }
 }

packages/docusaurus-utils/src/__tests__/__snapshots__/parseMarkdown.test.ts.snap

@@ -0,0 +1,137 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`load utils: parseMarkdown parseMarkdownString should delete only first heading 1`] = `
+Object {
+  "content": "
+test test test test test test
+test test test # test bar
+# test
+### test",
+  "excerpt": "",
+  "frontMatter": Object {
+    "title": "test",
+  },
+  "hasFrontMatter": false,
+}
+`;
+
+exports[`load utils: parseMarkdown parseMarkdownString should ignore heading if its not a first text 1`] = `
+Object {
+  "content": "foo
+# test",
+  "excerpt": "foo",
+  "frontMatter": Object {},
+  "hasFrontMatter": false,
+}
+`;
+
+exports[`load utils: parseMarkdown parseMarkdownString should parse first heading as title 1`] = `
+Object {
+  "content": "",
+  "excerpt": "",
+  "frontMatter": Object {
+    "title": "test",
+  },
+  "hasFrontMatter": false,
+}
+`;
+
+exports[`load utils: parseMarkdown parseMarkdownString should preserve front-matter title and warn about duplication 1`] = `
+Object {
+  "content": "# test",
+  "excerpt": "test",
+  "frontMatter": Object {
+    "title": "title",
+  },
+  "hasFrontMatter": true,
+}
+`;
+
+exports[`load utils: parseMarkdown parseMarkdownString should read front matter 1`] = `
+Object {
+  "content": "",
+  "excerpt": undefined,
+  "frontMatter": Object {
+    "title": "test",
+  },
+  "hasFrontMatter": true,
+}
+`;
+
+exports[`load utils: parseMarkdown readFrontMatter should delete only first heading 1`] = `
+Object {
+  "content": "test test test # test bar
+# test
+### test",
+  "excerpt": "",
+  "frontMatter": Object {
+    "title": "test",
+  },
+  "hasFrontMatter": false,
+}
+`;
+
+exports[`load utils: parseMarkdown readFrontMatter should ignore heading if its not a first text 1`] = `
+Object {
+  "content": "foo
+# test",
+  "excerpt": "",
+  "frontMatter": Object {},
+  "hasFrontMatter": false,
+}
+`;
+
+exports[`load utils: parseMarkdown readFrontMatter should parse first heading as title 1`] = `
+Object {
+  "content": "",
+  "excerpt": "",
+  "frontMatter": Object {
+    "title": "test",
+  },
+  "hasFrontMatter": false,
+}
+`;
+
+exports[`load utils: parseMarkdown readFrontMatter should parse first heading as title and keep it in content 1`] = `
+Object {
+  "content": "# test",
+  "excerpt": "",
+  "frontMatter": Object {
+    "title": "test",
+  },
+  "hasFrontMatter": false,
+}
+`;
+
+exports[`load utils: parseMarkdown readFrontMatter should parse front-matter and ignore h2 1`] = `
+Object {
+  "content": "## test",
+  "excerpt": "",
+  "frontMatter": Object {
+    "title": "title",
+  },
+  "hasFrontMatter": true,
+}
+`;
+
+exports[`load utils: parseMarkdown readFrontMatter should preserve front-matter title and warn about duplication 1`] = `
+Object {
+  "content": "# test",
+  "excerpt": "",
+  "frontMatter": Object {
+    "title": "title",
+  },
+  "hasFrontMatter": true,
+}
+`;
+
+exports[`load utils: parseMarkdown readFrontMatter should read front matter 1`] = `
+Object {
+  "content": "",
+  "excerpt": "",
+  "frontMatter": Object {
+    "title": "test",
+  },
+  "hasFrontMatter": true,
+}
+`;

packages/docusaurus-utils/src/__tests__/parseMarkdown.test.ts

@@ -0,0 +1,160 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import {parseMarkdownString, readFrontMatter} from '../index';
+import dedent from 'dedent';
+
+describe('load utils: parseMarkdown', () => {
+  describe('readFrontMatter', () => {
+    test('should read front matter', () => {
+      const warn = jest.spyOn(console, 'warn').mockImplementation(() => {});
+      expect(
+        readFrontMatter(dedent`
+        ---
+        title: test
+        ---
+        `),
+      ).toMatchSnapshot();
+      expect(warn).not.toBeCalled();
+    });
+    test('should parse first heading as title', () => {
+      const warn = jest.spyOn(console, 'warn').mockImplementation(() => {});
+      expect(
+        readFrontMatter(dedent`
+        # test
+        `),
+      ).toMatchSnapshot();
+      expect(warn).not.toBeCalled();
+    });
+    test('should preserve front-matter title and warn about duplication', () => {
+      const warn = jest.spyOn(console, 'warn').mockImplementation(() => {});
+      expect(
+        readFrontMatter(dedent`
+        ---
+        title: title
+        ---
+        # test
+        `),
+      ).toMatchSnapshot();
+      expect(warn).toBeCalledWith('Duplicate title detected in `this` file');
+      warn.mockReset();
+    });
+    test('should ignore heading if its not a first text', () => {
+      const warn = jest.spyOn(console, 'warn').mockImplementation(() => {});
+      expect(
+        readFrontMatter(dedent`
+        foo
+        # test
+        `),
+      ).toMatchSnapshot();
+      expect(warn).not.toBeCalled();
+    });
+    test('should parse first heading as title and keep it in content', () => {
+      const warn = jest.spyOn(console, 'warn').mockImplementation(() => {});
+      expect(
+        readFrontMatter(
+          dedent`
+          # test
+          `,
+          undefined,
+          {},
+          false,
+        ),
+      ).toMatchSnapshot();
+      expect(warn).not.toBeCalled();
+    });
+    test('should delete only first heading', () => {
+      const warn = jest.spyOn(console, 'warn').mockImplementation(() => {});
+      expect(
+        readFrontMatter(dedent`
+        # test
+        test test test # test bar
+        # test
+        ### test
+        `),
+      ).toMatchSnapshot();
+      expect(warn).not.toBeCalled();
+    });
+    test('should parse front-matter and ignore h2', () => {
+      const warn = jest.spyOn(console, 'warn').mockImplementation(() => {});
+      expect(
+        readFrontMatter(
+          dedent`
+          ---
+          title: title
+          ---
+          ## test
+          `,
+          undefined,
+          {},
+          false,
+        ),
+      ).toMatchSnapshot();
+      expect(warn).not.toBeCalled();
+    });
+  });
+
+  describe('parseMarkdownString', () => {
+    test('should read front matter', () => {
+      const warn = jest.spyOn(console, 'warn').mockImplementation(() => {});
+      expect(
+        parseMarkdownString(dedent`
+        ---
+        title: test
+        ---
+        `),
+      ).toMatchSnapshot();
+      expect(warn).not.toBeCalled();
+    });
+    test('should parse first heading as title', () => {
+      const warn = jest.spyOn(console, 'warn').mockImplementation(() => {});
+      expect(
+        parseMarkdownString(dedent`
+        # test
+        `),
+      ).toMatchSnapshot();
+      expect(warn).not.toBeCalled();
+    });
+    test('should preserve front-matter title and warn about duplication', () => {
+      const warn = jest.spyOn(console, 'warn').mockImplementation(() => {});
+      expect(
+        parseMarkdownString(dedent`
+        ---
+        title: title
+        ---
+        # test
+        `),
+      ).toMatchSnapshot();
+      expect(warn).toBeCalledWith('Duplicate title detected in `this` file');
+      warn.mockReset();
+    });
+    test('should ignore heading if its not a first text', () => {
+      const warn = jest.spyOn(console, 'warn').mockImplementation(() => {});
+      expect(
+        parseMarkdownString(dedent`
+        foo
+        # test
+        `),
+      ).toMatchSnapshot();
+      expect(warn).not.toBeCalled();
+    });
+    test('should delete only first heading', () => {
+      const warn = jest.spyOn(console, 'warn').mockImplementation(() => {});
+      expect(
+        parseMarkdownString(dedent`
+        # test
+
+        test test test test test test
+        test test test # test bar
+        # test
+        ### test
+        `),
+      ).toMatchSnapshot();
+      expect(warn).not.toBeCalled();
+    });
+  });
+});

packages/docusaurus-utils/src/index.ts

@@ -252,30 +252,50 @@ export function createExcerpt(fileString: string): string | undefined {
 }
 
 type ParsedMarkdown = {
-  frontMatter: {
+  // Returned by gray-matter
-    // Returned by gray-matter
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  frontMatter: Record<string, any>;
-    [key: string]: any;
-  };
   content: string;
   excerpt: string | undefined;
+  hasFrontMatter: boolean;
 };
-export function parseMarkdownString(markdownString: string): ParsedMarkdown {
-  const options: Record<string, unknown> = {
-    excerpt: (file: matter.GrayMatterFile<string>): void => {
-      // Hacky way of stripping out import statements from the excerpt
-      // TODO: Find a better way to do so, possibly by compiling the Markdown content,
-      // stripping out HTML tags and obtaining the first line.
-      file.excerpt = createExcerpt(file.content);
-    },
-  };
 
+export function readFrontMatter(
+  markdownString: string,
+  source?: string,
+  options: Record<string, unknown> = {},
+  removeTitleHeading = true,
+): ParsedMarkdown {
   try {
-    const {data: frontMatter, content, excerpt} = matter(
+    const result = matter(markdownString, options);
-      markdownString,
+    result.data = result.data || {};
-      options,
+    result.content = result.content.trim();
-    );
+
-    return {frontMatter, content, excerpt};
+    const hasFrontMatter = Object.keys(result.data).length > 0;
+
+    const heading = /^# (.*)[\n\r]?/gi.exec(result.content);
+    if (heading) {
+      if (result.data.title) {
+        console.warn(
+          `Duplicate title detected in \`${source || 'this'}\` file`,
+        );
+      } else {
+        result.data.title = heading[1].trim();
+        if (removeTitleHeading) {
+          result.content = result.content.replace(heading[0], '');
+          if (result.excerpt) {
+            result.excerpt = result.excerpt.replace(heading[1], '');
+          }
+        }
+      }
+    }
+
+    return {
+      frontMatter: result.data,
+      content: result.content,
+      excerpt: result.excerpt,
+      hasFrontMatter,
+    };
   } catch (e) {
     throw new Error(`Error while parsing markdown front matter.
 This can happen if you use special characters like : in frontmatter values (try using "" around that value)
@@ -283,12 +303,26 @@ ${e.message}`);
   }
 }
 
+export function parseMarkdownString(
+  markdownString: string,
+  source?: string,
+): ParsedMarkdown {
+  return readFrontMatter(markdownString, source, {
+    excerpt: (file: matter.GrayMatterFile<string>): void => {
+      // Hacky way of stripping out import statements from the excerpt
+      // TODO: Find a better way to do so, possibly by compiling the Markdown content,
+      // stripping out HTML tags and obtaining the first line.
+      file.excerpt = createExcerpt(file.content);
+    },
+  });
+}
+
 export async function parseMarkdownFile(
   source: string,
 ): Promise<ParsedMarkdown> {
   const markdownString = await fs.readFile(source, 'utf-8');
   try {
-    return parseMarkdownString(markdownString);
+    return parseMarkdownString(markdownString, source);
   } catch (e) {
     throw new Error(
       `Error while parsing markdown file ${source}

website/docs/cli.md

@@ -1,8 +1,9 @@
 ---
 id: cli
-title: CLI
 ---
 
+# CLI
+
 Docusaurus provides a set of scripts to help you generate, serve, and deploy your website.
 
 Once your website is bootstrapped, the website source will contain the Docusaurus scripts that you can invoke with your package manager:

yarn.lock

@@ -3304,6 +3304,11 @@
   dependencies:
     "@types/node" "*"
 
+"@types/dedent@^0.7.0":
+  version "0.7.0"
+  resolved "https://registry.yarnpkg.com/@types/dedent/-/dedent-0.7.0.tgz#155f339ca404e6dd90b9ce46a3f78fd69ca9b050"
+  integrity sha512-EGlKlgMhnLt/cM4DbUSafFdrkeJoC9Mvnj0PUCU7tFmTjMjNRT957kXCx0wYm3JuEq4o4ZsS5vG+NlkM2DMd2A==
+
 "@types/detect-port@^1.3.0":
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/@types/detect-port/-/detect-port-1.3.0.tgz#3e9cbd049ec29e84a2ff7852dbc629c81245774c"