chore(v2): ability to test the migration cli easily (#3113)

* ability to test the migration cli easily

* add node scripts to help test migration cli (locally + CI)

* add test for frontmatter quotify

* more tests for shouldQuotifyFrontMatter

* typo

* updated yarn lock

.github/workflows/migration-cli-e2e-test.yml

@@ -25,11 +25,8 @@ jobs:
       - name: Installation
         run: yarn
       - name: Migrate D1 website
-        run: yarn docusaurus-migrate migrate ./website-1.x ./test-migrated
+        run: yarn test:v1Migration:migrate
-      - name: link
+      - name: Build D1 migrated website
-        run: yarn lerna exec -- yarn link
+        run: yarn test:v1Migration:build
-      - name: Build Test website
-        run: yarn build
-        working-directory: test-migrated
         env:
           CI: true

.gitignore

@@ -30,4 +30,5 @@ packages/docusaurus-theme-classic/lib/
 packages/docusaurus-migrate/lib/
 
 website/netlifyDeploy
-_redirects
+
+website-1.x-migrated

package.json

@@ -4,10 +4,10 @@
     "packages/*",
     "website",
     "website-1.x",
+    "website-1.x-migrated",
     "packages/docusaurus-init/templates/*"
   ],
   "scripts": {
-    "testBaseUrl": "yarn build:v2:baseUrl && yarn serve:v2:baseUrl",
     "start": "yarn build:packages && yarn start:v2",
     "start:v1": "yarn workspace docusaurus-1-website start",
     "start:v2": "yarn workspace docusaurus-2-website start",
@@ -38,7 +38,11 @@
     "test:build:v2": "./admin/scripts/test-release.sh",
     "tsc": "yarn build:packages && echo '\n\nDOCUSAURUS: yarn tsc is deprecated and will be removed, use yarn build:packages instead\n\n'",
     "watch": "yarn lerna run --parallel --no-private watch",
-    "clear": "yarn rimraf website/.docusaurus && rimraf -rf website/node_modules/.cache && yarn lerna exec 'yarn rimraf lib' --ignore docusaurus"
+    "clear": "yarn rimraf website/.docusaurus && rimraf -rf website/node_modules/.cache && yarn lerna exec 'yarn rimraf lib' --ignore docusaurus",
+    "test:v1Migration:migrate": "rimraf website-1.x-migrated && docusaurus-migrate migrate ./website-1.x ./website-1.x-migrated && sed -i -- 's/docusaurus-1-website/docusaurus-1-website-migrated/g;' website-1.x-migrated/package.json",
+    "test:v1Migration:start": "yarn workspace docusaurus-1-website-migrated start",
+    "test:v1Migration:build": "yarn workspace docusaurus-1-website-migrated build",
+    "test:baseUrl": "yarn build:v2:baseUrl && yarn serve:v2:baseUrl"
   },
   "devDependencies": {
     "@babel/cli": "^7.9.0",

packages/docusaurus-migrate/src/__tests__/frontMatter.test.ts

@@ -0,0 +1,27 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import {shouldQuotifyFrontMatter} from '../frontMatter';
+
+describe('frontMatter', () => {
+  test('shouldQuotifyFrontMatter', () => {
+    expect(shouldQuotifyFrontMatter(['id', 'value'])).toEqual(false);
+    expect(
+      shouldQuotifyFrontMatter([
+        'title',
+        "Some title front matter with allowed special chars like sàáâãäåçèéêëìíîïðòóôõöùúûüýÿ!;,=+-_?'`&#()[]§%€$",
+      ]),
+    ).toEqual(false);
+
+    expect(shouldQuotifyFrontMatter(['title', 'Special char :'])).toEqual(true);
+
+    expect(shouldQuotifyFrontMatter(['title', 'value!'])).toEqual(false);
+    expect(shouldQuotifyFrontMatter(['title', '!value'])).toEqual(true);
+
+    expect(shouldQuotifyFrontMatter(['tags', '[tag1, tag2]'])).toEqual(false);
+  });
+});

packages/docusaurus-migrate/src/frontMatter.ts

@@ -50,3 +50,27 @@ export default function extractMetadata(content: string): Data {
   }
   return {metadata, rawContent: both.content};
 }
+
+// The new frontmatter parser need some special chars to
+export function shouldQuotifyFrontMatter([key, value]: [
+  string,
+  string,
+]): boolean {
+  if (key === 'tags') {
+    return false;
+  }
+  if (String(value).match(/^("|').+("|')$/)) {
+    return false;
+  }
+  // TODO weird graymatter case
+  // title: !something need quotes
+  // but not title: something!
+  if (!String(value).trim().match(/^\w.*/)) {
+    return true;
+  }
+  // TODO this is not ideal to have to maintain such a list of allowed chars
+  // maybe we should quotify if graymatter throws instead?
+  return !String(value).match(
+    /^([\w .\-sàáâãäåçèéêëìíîïðòóôõöùúûüýÿ!;,=+_?'`&#()[\]§%€$])+$/,
+  );
+}

packages/docusaurus-migrate/src/index.ts

@@ -17,7 +17,7 @@ import {
   ClassicPresetEntries,
   SidebarEntries,
 } from './types';
-import extractMetadata from './frontMatter';
+import extractMetadata, {shouldQuotifyFrontMatter} from './frontMatter';
 import migratePage from './transform';
 import sanitizeMD from './sanitizeMD';
 import path from 'path';
@@ -46,13 +46,9 @@ function sanitizedFileContent(
 ): string {
   const extractedData = extractMetadata(content);
   const extractedMetaData = Object.entries(extractedData.metadata).reduce(
-    (metaData, value) => {
+    (metaData, [key, value]) => {
-      return `${metaData}\n${value[0]}: ${
+      return `${metaData}\n${key}: ${
-        value[0] === 'tags' ||
+        shouldQuotifyFrontMatter([key, value]) ? `"${value}"` : value
-        !!String(value[1]).match(/^(\w| |\.|-)+$/m) ||
-        String(value[1]).match(/^("|').+("|')$/)
-          ? value[1]
-          : `"${value[1]}"`
       }`;
     },
     '',
@@ -377,7 +373,7 @@ function createPages(newDir: string, siteDir: string): void {
 function createDefaultLandingPage(newDir: string) {
   const indexPage = `import Layout from "@theme/Layout";
       import React from "react";
-      
+
       export default () => {
         return <Layout />;
       };

yarn.lock

@@ -17578,7 +17578,7 @@ react-dev-utils@^9.1.0:
     strip-ansi "5.2.0"
     text-table "0.2.0"
 
-react-dom@^16.8.4:
+react-dom@^16.10.2, react-dom@^16.8.4:
   version "16.13.1"
   resolved "https://registry.yarnpkg.com/react-dom/-/react-dom-16.13.1.tgz#c1bd37331a0486c078ee54c4740720993b2e0e7f"
   integrity sha512-81PIMmVLnCNLO/fFOQxdQkvEq/+Hfpv24XNJfpyZhTRfO0QcmQIF/PgCa1zCOj2w1hrn12MFLyaJ/G0+Mxtfag==
@@ -17739,7 +17739,7 @@ react-waypoint@^9.0.2:
     prop-types "^15.0.0"
     react-is "^16.6.3"
 
-react@^16.8.4:
+react@^16.10.2, react@^16.8.4:
   version "16.13.1"
   resolved "https://registry.yarnpkg.com/react/-/react-16.13.1.tgz#2e818822f1a9743122c063d6410d85c1e3afe48e"
   integrity sha512-YMZQQq32xHLX0bz5Mnibv1/LHb3Sqzngu7xstSM+vrkE5Kzr9xE0yMByK5kMoTK30YVJE61WfbxIFFvfeDKT1w==