perf(v2): smaller bundlesize by embedding metadata to content (#2088)

* wip embed metadata to content

* embed metadata in blog as well

* refactor

* update test

* yarn lock

* avoid overwriting file everytime we run new nodejs process

* nits

packages/docusaurus-plugin-content-docs/src/__tests__/__snapshots__/index.test.ts.snap

@@ -55,7 +55,7 @@ Array [
   Object {
     "component": "@theme/DocPage",
     "modules": Object {
-      "docsMetadata": "@docusaurus-plugin-content-docs/docs-route-ff2.json",
+      "docsMetadata": "~docs/docs-route-ff2.json",
     },
     "path": "/docs/:route",
     "priority": undefined,
@@ -65,7 +65,6 @@ Array [
         "exact": true,
         "modules": Object {
           "content": "@site/docs/foo/bar.md",
-          "metadata": "@docusaurus-plugin-content-docs/docs-foo-bar-cef.json",
         },
         "path": "/docs/foo/bar",
       },
@@ -74,7 +73,6 @@ Array [
         "exact": true,
         "modules": Object {
           "content": "@site/docs/foo/baz.md",
-          "metadata": "@docusaurus-plugin-content-docs/docs-foo-baz-dd9.json",
         },
         "path": "/docs/foo/baz",
       },
@@ -83,7 +81,6 @@ Array [
         "exact": true,
         "modules": Object {
           "content": "@site/docs/hello.md",
-          "metadata": "@docusaurus-plugin-content-docs/docs-hello-da2.json",
         },
         "path": "/docs/hello",
       },
@@ -92,7 +89,6 @@ Array [
         "exact": true,
         "modules": Object {
           "content": "@site/docs/lorem.md",
-          "metadata": "@docusaurus-plugin-content-docs/docs-lorem-17b.json",
         },
         "path": "/docs/lorem",
       },
@@ -106,7 +102,7 @@ Array [
   Object {
     "component": "@theme/DocPage",
     "modules": Object {
-      "docsMetadata": "@docusaurus-plugin-content-docs/docs-1-0-0-route-660.json",
+      "docsMetadata": "~docs/docs-1-0-0-route-660.json",
     },
     "path": "/docs/1.0.0/:route",
     "priority": undefined,
@@ -116,7 +112,6 @@ Array [
         "exact": true,
         "modules": Object {
           "content": "@site/versioned_docs/version-1.0.0/foo/bar.md",
-          "metadata": "@docusaurus-plugin-content-docs/docs-1-0-0-foo-bar-568.json",
         },
         "path": "/docs/1.0.0/foo/bar",
       },
@@ -125,7 +120,6 @@ Array [
         "exact": true,
         "modules": Object {
           "content": "@site/versioned_docs/version-1.0.0/foo/baz.md",
-          "metadata": "@docusaurus-plugin-content-docs/docs-1-0-0-foo-baz-5e1.json",
         },
         "path": "/docs/1.0.0/foo/baz",
       },
@@ -134,7 +128,6 @@ Array [
         "exact": true,
         "modules": Object {
           "content": "@site/versioned_docs/version-1.0.0/hello.md",
-          "metadata": "@docusaurus-plugin-content-docs/docs-1-0-0-hello-1d0.json",
         },
         "path": "/docs/1.0.0/hello",
       },
@@ -143,7 +136,7 @@ Array [
   Object {
     "component": "@theme/DocPage",
     "modules": Object {
-      "docsMetadata": "@docusaurus-plugin-content-docs/docs-next-route-1c8.json",
+      "docsMetadata": "~docs/docs-next-route-1c8.json",
     },
     "path": "/docs/next/:route",
     "priority": undefined,
@@ -153,7 +146,6 @@ Array [
         "exact": true,
         "modules": Object {
           "content": "@site/docs/foo/bar.md",
-          "metadata": "@docusaurus-plugin-content-docs/docs-next-foo-bar-09c.json",
         },
         "path": "/docs/next/foo/bar",
       },
@@ -162,7 +154,6 @@ Array [
         "exact": true,
         "modules": Object {
           "content": "@site/docs/hello.md",
-          "metadata": "@docusaurus-plugin-content-docs/docs-next-hello-64c.json",
         },
         "path": "/docs/next/hello",
       },
@@ -171,7 +162,7 @@ Array [
   Object {
     "component": "@theme/DocPage",
     "modules": Object {
-      "docsMetadata": "@docusaurus-plugin-content-docs/docs-route-ff2.json",
+      "docsMetadata": "~docs/docs-route-ff2.json",
     },
     "path": "/docs/:route",
     "priority": -1,
@@ -181,7 +172,6 @@ Array [
         "exact": true,
         "modules": Object {
           "content": "@site/versioned_docs/version-1.0.1/foo/bar.md",
-          "metadata": "@docusaurus-plugin-content-docs/docs-foo-bar-cef.json",
         },
         "path": "/docs/foo/bar",
       },
@@ -190,7 +180,6 @@ Array [
         "exact": true,
         "modules": Object {
           "content": "@site/versioned_docs/version-1.0.1/hello.md",
-          "metadata": "@docusaurus-plugin-content-docs/docs-hello-da2.json",
         },
         "path": "/docs/hello",
       },

packages/docusaurus-plugin-content-docs/src/index.ts

@@ -277,15 +277,16 @@ export default function pluginContentDocs(
       const {docLayoutComponent, docItemComponent, routeBasePath} = options;
       const {addRoute, createData} = actions;
       const aliasedSource = (source: string) =>
-        `@docusaurus-plugin-content-docs/${path.relative(dataDir, source)}`;
+        `~docs/${path.relative(dataDir, source)}`;
 
       const genRoutes = async (
         metadataItems: Metadata[],
       ): Promise<RouteConfig[]> => {
         const routes = await Promise.all(
           metadataItems.map(async metadataItem => {
-            const metadataPath = await createData(
-              `${docuHash(metadataItem.permalink)}.json`,
+            await createData(
+              // Note that this created data path must be in sync with markdown/index.ts metadataPath
+              `${docuHash(metadataItem.source)}.json`,
               JSON.stringify(metadataItem, null, 2),
             );
             return {
@@ -294,7 +295,6 @@ export default function pluginContentDocs(
               exact: true,
               modules: {
                 content: metadataItem.source,
-                metadata: aliasedSource(metadataPath),
               },
             };
           }),
@@ -388,7 +388,7 @@ export default function pluginContentDocs(
       return {
         resolve: {
           alias: {
-            '@docusaurus-plugin-content-docs': dataDir,
+            '~docs': dataDir,
           },
         },
         module: {
@@ -410,6 +410,7 @@ export default function pluginContentDocs(
                   loader: path.resolve(__dirname, './markdown/index.js'),
                   options: {
                     siteDir,
+                    dataDir,
                     docsDir,
                     sourceToPermalink: sourceToPermalink,
                     versionedDir,

packages/docusaurus-plugin-content-docs/src/markdown/index.ts

@@ -5,25 +5,45 @@
  * LICENSE file in the root directory of this source tree.
  */
 
+import path from 'path';
+import {readFile} from 'fs-extra';
 import {getOptions} from 'loader-utils';
 import {loader} from 'webpack';
 import linkify from './linkify';
+import {docuHash, aliasedSitePath} from '@docusaurus/utils';
 
 export = function(fileString: string) {
   const callback = this.async();
-  const {docsDir, siteDir, versionedDir, sourceToPermalink} = getOptions(this);
-  return (
-    callback &&
-    callback(
-      null,
-      linkify(
-        fileString,
-        this.resourcePath,
-        docsDir,
-        siteDir,
-        sourceToPermalink,
-        versionedDir,
-      ),
-    )
+  const {
+    dataDir,
+    docsDir,
+    siteDir,
+    versionedDir,
+    sourceToPermalink,
+  } = getOptions(this);
+
+  // Replace all markdown linking to correct url
+  const linkifiedStr = linkify(
+    fileString,
+    this.resourcePath,
+    docsDir,
+    siteDir,
+    sourceToPermalink,
+    versionedDir,
   );
+
+  // Read metadata & then embed it to this markdown content
+  // Note that metadataPath must be the same/ in-sync as the path from createData
+  const aliasedSource = aliasedSitePath(this.resourcePath, siteDir);
+  const metadataPath = path.join(dataDir, `${docuHash(aliasedSource)}.json`);
+
+  // Add metadataPath as dependency of this loader result so that we can recompile if metadata is changed
+  this.addDependency(metadataPath);
+
+  readFile(metadataPath, 'utf8', function(err, metadata) {
+    if (err) return callback && callback(err);
+
+    const metadataStr = `export const metadata = ${metadata}`;
+    callback && callback(null, linkifiedStr + '\n' + metadataStr);
+  });
 } as loader.Loader;

packages/docusaurus-plugin-content-docs/src/metadata.ts

@@ -7,7 +7,12 @@
 
 import fs from 'fs-extra';
 import path from 'path';
-import {parse, normalizeUrl, posixPath} from '@docusaurus/utils';
+import {
+  parse,
+  aliasedSitePath,
+  normalizeUrl,
+  posixPath,
+} from '@docusaurus/utils';
 import {LoadContext} from '@docusaurus/types';
 
 import lastUpdate from './lastUpdate';
@@ -84,9 +89,6 @@ export default async function processMetadata({
 
   const relativePath = path.relative(siteDir, filePath);
 
-  // Cannot use path.join() as it resolves '../' and removes the '@site'. Let webpack loader resolve it.
-  const aliasedPath = `@site/${relativePath}`;
-
   const docsEditUrl = editUrl
     ? normalizeUrl([editUrl, posixPath(relativePath)])
     : undefined;
@@ -130,7 +132,7 @@ export default async function processMetadata({
     id,
     title,
     description,
-    source: aliasedPath,
+    source: aliasedSitePath(filePath, siteDir),
     permalink,
     editUrl: custom_edit_url || docsEditUrl,
     version,