feat(v2): core v2 i18n support + Docusaurus site Crowdin integration (#3325)

* docs i18n initial poc

* docs i18n initial poc

* docs i18n initial poc

* docs i18n initial poc

* crowdin-v2 attempt

* fix source

* use crowdin env variable

* try to install crowdin on netlify

* try to install crowdin on netlify

* try to use crowdin jar directly

* try to curl the crowdin jar

* add java version cmd

* try to run crowdin jar in netlify

* fix translatedDocsDirPath

* fix loadContext issue due to site baseUrl not being modified in generted config file

* real validateLocalesFile

* add locale option to deploy command

* better LocalizationFile type

* create util getPluginI18nPath

* better core localization context loading code

* More explicit VersionMetadata type for localized docs folders

* Ability to translate blog posts with Crowdin!

* blog: refactor markdown loader + report broken links + try to get linkify working better

* upgrade crowdin config to upload all docs folder files except source code related files

* try to support translated pages

* make markdown pages translation work

* add write-translations cli command template

* fix site not  reloaded with correct options

* refactor a bit the read/write of @generated/i18n.json file

* Add <Translate> + translate() API + use it on the docusaurus homepage

* watch locale translation dir

* early POC of adding babel parsing for translation extraction

* fs.stat => pathExists

* add install:fast script

* TSC: noUnusedLocals false as it's already checked  by eslint

* POC of extracting translations from source code

* minor typo

* fix extracted key to code

* initial docs extracted translations

* stable plugin translations POC

* add crowdin commands

* quickfix for i18n deployment

* POC  of themeConfig translation

* add ability to have localized site without path prefix

* sidebar typo

* refactor translation system to output multiple translation files

* translate properly  the docs plugin

* improve theme classic translation

* rework translation extractor to handle new Chrome I18n JSON format (include id/description)

* writeTranslations: allow to pass locales cli arg

* fix ThemeConfig TS issues

* fix localizePath errors

* temporary add write-translations to netlify deploy preview

* complete example of french translated folder

* update fr folder

* remove all translations from repo

* minor translation  refactors

* fix all docs-related tests

* fix blog feed tests

* fix last blog tests

* refactor i18n context a bit, extract codeTranslations in an extra generated file

* improve @generated/i18n type

* fix some i18n todos

* minor refactor

* fix logo typing issue after merge

* move i18n.json to siteConfig instead

* try to fix windows CI build

* fix config test

* attempt to fix windows non-posix path

* increase v1 minify css jest timeout due to flaky test

* proper support for localizePath on windows

* remove non-functional install:fast

* docs, fix docsDirPathLocalized

* fix Docs i18n / md linkify issues

* ensure theme-classic swizzling will use "nextjs" sources (transpiled less aggressively, to make them human readable)

* fix some snapshots

* improve themeConfig translation code

* refactor a bit getPluginI18nPath

* readTranslationFileContent => ensure files are valid, fail fast

* fix versions tests

* add extractSourceCodeAstTranslations comments/resource links

* ignore eslint: packages/docusaurus-theme-classic/lib-next/

* fix windows CI with cross-env

* crowdin ignore .DS_Store

* improve writeTranslations + add exhaustive tests for translations.ts

* remove typo

* Wire currentLocale to algolia search

* improve i18n locale error

* Add tests for translationsExtractor.ts

* better code translation extraction regarding statically evaluable code

* fix typo

* fix typo

* improve theme-classic transpilation

* refactor  +  add i18n tests

* typo

* test new utils

* add missing snapshots

* fix snapshot

* blog onBrokenMarkdownLink

* add sidebars tests

* theme-classic index should now use ES modules

* tests for theme-classic translations

* useless comment

* add more translation tests

* simplify/cleanup writeTranslations

* try to fix Netlify fr deployment

* blog: test translated md is used during feed generation

* blog: better i18n tests regarding editUrl + md translation application

* more i18n tests for docs plugin

* more i18n tests for docs plugin

* Add tests for pages i18n

* polish docusaurus build i18n logs

packages/docusaurus-plugin-content-pages/src/__tests__/__fixtures__/website/i18n/fr/docusaurus-plugin-content-pages/hello/translatedJs.js

@@ -0,0 +1,14 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import React from 'react';
+
+export default class TranslatedJs extends React.Component {
+  render() {
+    return <div>TranslatedJsPage (fr)</div>;
+  }
+}

packages/docusaurus-plugin-content-pages/src/__tests__/__fixtures__/website/i18n/fr/docusaurus-plugin-content-pages/hello/translatedMd.md

@@ -0,0 +1 @@
+translated markdown page (fr)

packages/docusaurus-plugin-content-pages/src/__tests__/__fixtures__/website/src/pages/hello/translatedJs.js

@@ -0,0 +1,14 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import React from 'react';
+
+export default class TranslatedJs extends React.Component {
+  render() {
+    return <div>TranslatedJsPage</div>;
+  }
+}

packages/docusaurus-plugin-content-pages/src/__tests__/__fixtures__/website/src/pages/hello/translatedMd.md

@@ -0,0 +1 @@
+translated markdown page

packages/docusaurus-plugin-content-pages/src/__tests__/index.test.ts

@@ -14,7 +14,7 @@ import normalizePluginOptions from './pluginOptionSchema.test';
 describe('docusaurus-plugin-content-pages', () => {
   test('simple pages', async () => {
     const siteDir = path.join(__dirname, '__fixtures__', 'website');
-    const context = loadContext(siteDir);
+    const context = await loadContext(siteDir);
     const pluginPath = 'src/pages';
     const plugin = pluginContentPages(
       context,
@@ -22,7 +22,7 @@ describe('docusaurus-plugin-content-pages', () => {
         path: pluginPath,
       }),
     );
-    const pagesMetadatas = await plugin.loadContent();
+    const pagesMetadatas = (await plugin.loadContent?.())!;
 
     expect(pagesMetadatas).toEqual([
       {
@@ -45,6 +45,80 @@ describe('docusaurus-plugin-content-pages', () => {
         permalink: '/hello/mdxPage',
         source: path.join('@site', pluginPath, 'hello', 'mdxPage.mdx'),
       },
+      {
+        type: 'jsx',
+        permalink: '/hello/translatedJs',
+        source: path.join('@site', pluginPath, 'hello', 'translatedJs.js'),
+      },
+      {
+        type: 'mdx',
+        permalink: '/hello/translatedMd',
+        source: path.join('@site', pluginPath, 'hello', 'translatedMd.md'),
+      },
+      {
+        type: 'jsx',
+        permalink: '/hello/world',
+        source: path.join('@site', pluginPath, 'hello', 'world.js'),
+      },
+    ]);
+  });
+
+  test('simple pages with french translations', async () => {
+    const siteDir = path.join(__dirname, '__fixtures__', 'website');
+    const context = await loadContext(siteDir);
+    const pluginPath = 'src/pages';
+    const plugin = pluginContentPages(
+      {
+        ...context,
+        i18n: {
+          ...context.i18n,
+          currentLocale: 'fr',
+        },
+      },
+      normalizePluginOptions({
+        path: pluginPath,
+      }),
+    );
+    const pagesMetadatas = (await plugin.loadContent?.())!;
+
+    const frTranslationsPath = path.join(
+      '@site',
+      'i18n',
+      'fr',
+      'docusaurus-plugin-content-pages',
+    );
+
+    expect(pagesMetadatas).toEqual([
+      {
+        type: 'jsx',
+        permalink: '/',
+        source: path.join('@site', pluginPath, 'index.js'),
+      },
+      {
+        type: 'jsx',
+        permalink: '/typescript',
+        source: path.join('@site', pluginPath, 'typescript.tsx'),
+      },
+      {
+        type: 'mdx',
+        permalink: '/hello/',
+        source: path.join('@site', pluginPath, 'hello', 'index.md'),
+      },
+      {
+        type: 'mdx',
+        permalink: '/hello/mdxPage',
+        source: path.join('@site', pluginPath, 'hello', 'mdxPage.mdx'),
+      },
+      {
+        type: 'jsx',
+        permalink: '/hello/translatedJs',
+        source: path.join(frTranslationsPath, 'hello', 'translatedJs.js'),
+      },
+      {
+        type: 'mdx',
+        permalink: '/hello/translatedMd',
+        source: path.join(frTranslationsPath, 'hello', 'translatedMd.md'),
+      },
       {
         type: 'jsx',
         permalink: '/hello/world',

packages/docusaurus-plugin-content-pages/src/index.ts

@@ -15,6 +15,8 @@ import {
   fileToPath,
   aliasedSitePath,
   docuHash,
+  getPluginI18nPath,
+  getFolderContainingFile,
 } from '@docusaurus/utils';
 import {
   LoadContext,
@@ -32,7 +34,17 @@ import {
   STATIC_DIR_NAME,
 } from '@docusaurus/core/lib/constants';
 
-import {PluginOptions, LoadedContent, Metadata} from './types';
+import {
+  PluginOptions,
+  LoadedContent,
+  Metadata,
+  PagesContentPaths,
+} from './types';
+import {flatten} from 'lodash';
+
+export function getContentPathList(contentPaths: PagesContentPaths) {
+  return [contentPaths.contentPathLocalized, contentPaths.contentPath];
+}
 
 const isMarkdownSource = (source: string) =>
   source.endsWith('.md') || source.endsWith('.mdx');
@@ -46,9 +58,22 @@ export default function pluginContentPages(
       [admonitions, options.admonitions || {}],
     ]);
   }
-  const {siteConfig, siteDir, generatedFilesDir} = context;
+  const {
+    siteConfig,
+    siteDir,
+    generatedFilesDir,
+    i18n: {currentLocale},
+  } = context;
 
-  const contentPath = path.resolve(siteDir, options.path);
+  const contentPaths: PagesContentPaths = {
+    contentPath: path.resolve(siteDir, options.path),
+    contentPathLocalized: getPluginI18nPath({
+      siteDir,
+      locale: currentLocale,
+      pluginName: 'docusaurus-plugin-content-pages',
+      pluginId: options.id,
+    }),
+  };
 
   const pluginDataDirRoot = path.join(
     generatedFilesDir,
@@ -66,8 +91,11 @@ export default function pluginContentPages(
 
     getPathsToWatch() {
       const {include = []} = options;
-      const globPattern = include.map((pattern) => `${contentPath}/${pattern}`);
-      return [...globPattern];
+      return flatten(
+        getContentPathList(contentPaths).map((contentPath) => {
+          return include.map((pattern) => `${contentPath}/${pattern}`);
+        }),
+      );
     },
 
     getClientModules() {
@@ -82,20 +110,25 @@ export default function pluginContentPages(
 
     async loadContent() {
       const {include} = options;
-      const pagesDir = contentPath;
 
-      if (!fs.existsSync(pagesDir)) {
+      if (!fs.existsSync(contentPaths.contentPath)) {
         return null;
       }
 
       const {baseUrl} = siteConfig;
       const pagesFiles = await globby(include, {
-        cwd: pagesDir,
+        cwd: contentPaths.contentPath,
         ignore: options.exclude,
       });
 
-      function toMetadata(relativeSource: string): Metadata {
-        const source = path.join(pagesDir, relativeSource);
+      async function toMetadata(relativeSource: string): Promise<Metadata> {
+        // Lookup in localized folder in priority
+        const contentPath = await getFolderContainingFile(
+          getContentPathList(contentPaths),
+          relativeSource,
+        );
+
+        const source = path.join(contentPath, relativeSource);
         const aliasedSourcePath = aliasedSitePath(source, siteDir);
         const pathName = encodePath(fileToPath(relativeSource));
         const permalink = pathName.replace(/^\//, baseUrl || '');
@@ -114,7 +147,7 @@ export default function pluginContentPages(
         }
       }
 
-      return pagesFiles.map(toMetadata);
+      return Promise.all(pagesFiles.map(toMetadata));
     },
 
     async contentLoaded({content, actions}) {
@@ -177,7 +210,7 @@ export default function pluginContentPages(
           rules: [
             {
               test: /(\.mdx?)$/,
-              include: [contentPath],
+              include: getContentPathList(contentPaths),
               use: [
                 getCacheLoader(isServer),
                 getBabelLoader(isServer),

packages/docusaurus-plugin-content-pages/src/types.ts

@@ -34,3 +34,8 @@ export type MDXPageMetadata = {
 export type Metadata = JSXPageMetadata | MDXPageMetadata;
 
 export type LoadedContent = Metadata[];
+
+export type PagesContentPaths = {
+  contentPath: string;
+  contentPathLocalized: string;
+};