feat(v2): core v2 i18n support + Docusaurus site Crowdin integration (#3325)

* docs i18n initial poc

* docs i18n initial poc

* docs i18n initial poc

* docs i18n initial poc

* crowdin-v2 attempt

* fix source

* use crowdin env variable

* try to install crowdin on netlify

* try to install crowdin on netlify

* try to use crowdin jar directly

* try to curl the crowdin jar

* add java version cmd

* try to run crowdin jar in netlify

* fix translatedDocsDirPath

* fix loadContext issue due to site baseUrl not being modified in generted config file

* real validateLocalesFile

* add locale option to deploy command

* better LocalizationFile type

* create util getPluginI18nPath

* better core localization context loading code

* More explicit VersionMetadata type for localized docs folders

* Ability to translate blog posts with Crowdin!

* blog: refactor markdown loader + report broken links + try to get linkify working better

* upgrade crowdin config to upload all docs folder files except source code related files

* try to support translated pages

* make markdown pages translation work

* add write-translations cli command template

* fix site not  reloaded with correct options

* refactor a bit the read/write of @generated/i18n.json file

* Add <Translate> + translate() API + use it on the docusaurus homepage

* watch locale translation dir

* early POC of adding babel parsing for translation extraction

* fs.stat => pathExists

* add install:fast script

* TSC: noUnusedLocals false as it's already checked  by eslint

* POC of extracting translations from source code

* minor typo

* fix extracted key to code

* initial docs extracted translations

* stable plugin translations POC

* add crowdin commands

* quickfix for i18n deployment

* POC  of themeConfig translation

* add ability to have localized site without path prefix

* sidebar typo

* refactor translation system to output multiple translation files

* translate properly  the docs plugin

* improve theme classic translation

* rework translation extractor to handle new Chrome I18n JSON format (include id/description)

* writeTranslations: allow to pass locales cli arg

* fix ThemeConfig TS issues

* fix localizePath errors

* temporary add write-translations to netlify deploy preview

* complete example of french translated folder

* update fr folder

* remove all translations from repo

* minor translation  refactors

* fix all docs-related tests

* fix blog feed tests

* fix last blog tests

* refactor i18n context a bit, extract codeTranslations in an extra generated file

* improve @generated/i18n type

* fix some i18n todos

* minor refactor

* fix logo typing issue after merge

* move i18n.json to siteConfig instead

* try to fix windows CI build

* fix config test

* attempt to fix windows non-posix path

* increase v1 minify css jest timeout due to flaky test

* proper support for localizePath on windows

* remove non-functional install:fast

* docs, fix docsDirPathLocalized

* fix Docs i18n / md linkify issues

* ensure theme-classic swizzling will use "nextjs" sources (transpiled less aggressively, to make them human readable)

* fix some snapshots

* improve themeConfig translation code

* refactor a bit getPluginI18nPath

* readTranslationFileContent => ensure files are valid, fail fast

* fix versions tests

* add extractSourceCodeAstTranslations comments/resource links

* ignore eslint: packages/docusaurus-theme-classic/lib-next/

* fix windows CI with cross-env

* crowdin ignore .DS_Store

* improve writeTranslations + add exhaustive tests for translations.ts

* remove typo

* Wire currentLocale to algolia search

* improve i18n locale error

* Add tests for translationsExtractor.ts

* better code translation extraction regarding statically evaluable code

* fix typo

* fix typo

* improve theme-classic transpilation

* refactor  +  add i18n tests

* typo

* test new utils

* add missing snapshots

* fix snapshot

* blog onBrokenMarkdownLink

* add sidebars tests

* theme-classic index should now use ES modules

* tests for theme-classic translations

* useless comment

* add more translation tests

* simplify/cleanup writeTranslations

* try to fix Netlify fr deployment

* blog: test translated md is used during feed generation

* blog: better i18n tests regarding editUrl + md translation application

* more i18n tests for docs plugin

* more i18n tests for docs plugin

* Add tests for pages i18n

* polish docusaurus build i18n logs

packages/docusaurus-plugin-content-docs/src/markdown/__tests__/__fixtures__/docs/doc-localized.md

@@ -0,0 +1 @@
+### localized doc

packages/docusaurus-plugin-content-docs/src/markdown/__tests__/__fixtures__/docs/doc2.md

@@ -8,3 +8,5 @@
 
 - [doc1](doc1.md)
 - [doc2](./doc2.md)
+
+- [doc-localized](./doc-localized.md)

packages/docusaurus-plugin-content-docs/src/markdown/__tests__/__snapshots__/linkify.test.ts.snap

@@ -35,6 +35,8 @@ exports[`transform to correct links 1`] = `
 
 - [doc1](/docs/doc1)
 - [doc2](/docs/doc2)
+
+- [doc-localized](/fr/doc-localized)
 "
 `;
 

packages/docusaurus-plugin-content-docs/src/markdown/__tests__/linkify.test.ts

@@ -16,15 +16,21 @@ import {
 } from '../../types';
 import {VERSIONED_DOCS_DIR, CURRENT_VERSION_NAME} from '../../constants';
 
-function createFakeVersion(
-  versionName: string,
-  docsDirPath: string,
-): VersionMetadata {
+function createFakeVersion({
+  versionName,
+  docsDirPath,
+  docsDirPathLocalized,
+}: {
+  versionName: string;
+  docsDirPath: string;
+  docsDirPathLocalized: string;
+}): VersionMetadata {
   return {
     versionName,
     versionLabel: 'Any',
     versionPath: 'any',
     docsDirPath,
+    docsDirPathLocalized,
     sidebarFilePath: 'any',
     routePriority: undefined,
     isLast: false,
@@ -33,14 +39,29 @@ function createFakeVersion(
 
 const siteDir = path.join(__dirname, '__fixtures__');
 
-const versionCurrent = createFakeVersion(
-  CURRENT_VERSION_NAME,
-  path.join(siteDir, 'docs'),
-);
-const version100 = createFakeVersion(
-  CURRENT_VERSION_NAME,
-  path.join(siteDir, VERSIONED_DOCS_DIR, 'version-1.0.0'),
-);
+const versionCurrent = createFakeVersion({
+  versionName: CURRENT_VERSION_NAME,
+  docsDirPath: path.join(siteDir, 'docs'),
+  docsDirPathLocalized: path.join(
+    siteDir,
+    'i18n',
+    'fr',
+    'docusaurus-plugin-content-docs',
+    CURRENT_VERSION_NAME,
+  ),
+});
+
+const version100 = createFakeVersion({
+  versionName: '1.0.0',
+  docsDirPath: path.join(siteDir, VERSIONED_DOCS_DIR, 'version-1.0.0'),
+  docsDirPathLocalized: path.join(
+    siteDir,
+    'i18n',
+    'fr',
+    'docusaurus-plugin-content-docs',
+    'version-1.0.0',
+  ),
+});
 
 const sourceToPermalink: SourceToPermalink = {
   '@site/docs/doc1.md': '/docs/doc1',
@@ -50,6 +71,10 @@ const sourceToPermalink: SourceToPermalink = {
   '@site/versioned_docs/version-1.0.0/doc2.md': '/docs/1.0.0/doc2',
   '@site/versioned_docs/version-1.0.0/subdir/doc1.md':
     '/docs/1.0.0/subdir/doc1',
+
+  '@site/i18n/fr/docusaurus-plugin-content-docs/current/doc-localized.md':
+    '/fr/doc-localized',
+  '@site/docs/doc-localized': '/doc-localized',
 };
 
 function createMarkdownOptions(
@@ -85,9 +110,11 @@ test('transform to correct links', () => {
   expect(transformedContent).toContain('](/docs/doc1');
   expect(transformedContent).toContain('](/docs/doc2');
   expect(transformedContent).toContain('](/docs/subdir/doc3');
+  expect(transformedContent).toContain('](/fr/doc-localized');
   expect(transformedContent).not.toContain('](doc1.md)');
   expect(transformedContent).not.toContain('](./doc2.md)');
   expect(transformedContent).not.toContain('](subdir/doc3.md)');
+  expect(transformedContent).not.toContain('](/doc-localized');
   expect(content).not.toEqual(transformedContent);
 });
 

packages/docusaurus-plugin-content-docs/src/markdown/linkify.ts

@@ -12,10 +12,13 @@ import {
   VersionMetadata,
   BrokenMarkdownLink,
 } from '../types';
+import {getDocsDirPaths} from '../versions';
 
 function getVersion(filePath: string, options: DocsMarkdownOption) {
   const versionFound = options.versionsMetadata.find((version) =>
-    filePath.startsWith(version.docsDirPath),
+    getDocsDirPaths(version).some((docsDirPath) =>
+      filePath.startsWith(docsDirPath),
+    ),
   );
   if (!versionFound) {
     throw new Error(
@@ -32,7 +35,7 @@ function replaceMarkdownLinks(
   options: DocsMarkdownOption,
 ) {
   const {siteDir, sourceToPermalink, onBrokenMarkdownLink} = options;
-  const {docsDirPath} = version;
+  const {docsDirPath, docsDirPathLocalized} = version;
 
   // Replace internal markdown linking (except in fenced blocks).
   let fencedBlock = false;
@@ -53,12 +56,15 @@ function replaceMarkdownLinks(
     while (mdMatch !== null) {
       // Replace it to correct html link.
       const mdLink = mdMatch[1];
-      const targetSource = `${docsDirPath}/${mdLink}`;
+
       const aliasedSource = (source: string) =>
         `@site/${path.relative(siteDir, source)}`;
+
       const permalink =
         sourceToPermalink[aliasedSource(resolve(filePath, mdLink))] ||
-        sourceToPermalink[aliasedSource(targetSource)];
+        sourceToPermalink[aliasedSource(`${docsDirPathLocalized}/${mdLink}`)] ||
+        sourceToPermalink[aliasedSource(`${docsDirPath}/${mdLink}`)];
+
       if (permalink) {
         modifiedLine = modifiedLine.replace(mdLink, permalink);
       } else {