feat(v2): core v2 i18n support + Docusaurus site Crowdin integration (#3325)

* docs i18n initial poc

* docs i18n initial poc

* docs i18n initial poc

* docs i18n initial poc

* crowdin-v2 attempt

* fix source

* use crowdin env variable

* try to install crowdin on netlify

* try to install crowdin on netlify

* try to use crowdin jar directly

* try to curl the crowdin jar

* add java version cmd

* try to run crowdin jar in netlify

* fix translatedDocsDirPath

* fix loadContext issue due to site baseUrl not being modified in generted config file

* real validateLocalesFile

* add locale option to deploy command

* better LocalizationFile type

* create util getPluginI18nPath

* better core localization context loading code

* More explicit VersionMetadata type for localized docs folders

* Ability to translate blog posts with Crowdin!

* blog: refactor markdown loader + report broken links + try to get linkify working better

* upgrade crowdin config to upload all docs folder files except source code related files

* try to support translated pages

* make markdown pages translation work

* add write-translations cli command template

* fix site not  reloaded with correct options

* refactor a bit the read/write of @generated/i18n.json file

* Add <Translate> + translate() API + use it on the docusaurus homepage

* watch locale translation dir

* early POC of adding babel parsing for translation extraction

* fs.stat => pathExists

* add install:fast script

* TSC: noUnusedLocals false as it's already checked  by eslint

* POC of extracting translations from source code

* minor typo

* fix extracted key to code

* initial docs extracted translations

* stable plugin translations POC

* add crowdin commands

* quickfix for i18n deployment

* POC  of themeConfig translation

* add ability to have localized site without path prefix

* sidebar typo

* refactor translation system to output multiple translation files

* translate properly  the docs plugin

* improve theme classic translation

* rework translation extractor to handle new Chrome I18n JSON format (include id/description)

* writeTranslations: allow to pass locales cli arg

* fix ThemeConfig TS issues

* fix localizePath errors

* temporary add write-translations to netlify deploy preview

* complete example of french translated folder

* update fr folder

* remove all translations from repo

* minor translation  refactors

* fix all docs-related tests

* fix blog feed tests

* fix last blog tests

* refactor i18n context a bit, extract codeTranslations in an extra generated file

* improve @generated/i18n type

* fix some i18n todos

* minor refactor

* fix logo typing issue after merge

* move i18n.json to siteConfig instead

* try to fix windows CI build

* fix config test

* attempt to fix windows non-posix path

* increase v1 minify css jest timeout due to flaky test

* proper support for localizePath on windows

* remove non-functional install:fast

* docs, fix docsDirPathLocalized

* fix Docs i18n / md linkify issues

* ensure theme-classic swizzling will use "nextjs" sources (transpiled less aggressively, to make them human readable)

* fix some snapshots

* improve themeConfig translation code

* refactor a bit getPluginI18nPath

* readTranslationFileContent => ensure files are valid, fail fast

* fix versions tests

* add extractSourceCodeAstTranslations comments/resource links

* ignore eslint: packages/docusaurus-theme-classic/lib-next/

* fix windows CI with cross-env

* crowdin ignore .DS_Store

* improve writeTranslations + add exhaustive tests for translations.ts

* remove typo

* Wire currentLocale to algolia search

* improve i18n locale error

* Add tests for translationsExtractor.ts

* better code translation extraction regarding statically evaluable code

* fix typo

* fix typo

* improve theme-classic transpilation

* refactor  +  add i18n tests

* typo

* test new utils

* add missing snapshots

* fix snapshot

* blog onBrokenMarkdownLink

* add sidebars tests

* theme-classic index should now use ES modules

* tests for theme-classic translations

* useless comment

* add more translation tests

* simplify/cleanup writeTranslations

* try to fix Netlify fr deployment

* blog: test translated md is used during feed generation

* blog: better i18n tests regarding editUrl + md translation application

* more i18n tests for docs plugin

* more i18n tests for docs plugin

* Add tests for pages i18n

* polish docusaurus build i18n logs

packages/docusaurus-plugin-content-blog/src/blogUtils.ts

@@ -11,14 +11,23 @@ import chalk from 'chalk';
 import path from 'path';
 import readingTime from 'reading-time';
 import {Feed} from 'feed';
-import {PluginOptions, BlogPost, DateLink} from './types';
+import {
+  PluginOptions,
+  BlogPost,
+  DateLink,
+  BlogContentPaths,
+  BlogBrokenMarkdownLink,
+  BlogMarkdownLoaderOptions,
+} from './types';
 import {
   parseMarkdownFile,
   normalizeUrl,
   aliasedSitePath,
   getEditUrl,
+  getFolderContainingFile,
 } from '@docusaurus/utils';
 import {LoadContext} from '@docusaurus/types';
+import {keyBy} from 'lodash';
 
 export function truncate(fileString: string, truncateMarker: RegExp): string {
   return fileString.split(truncateMarker, 1).shift()!;
@@ -36,6 +45,7 @@ function toUrl({date, link}: DateLink) {
 }
 
 export async function generateBlogFeed(
+  contentPaths: BlogContentPaths,
   context: LoadContext,
   options: PluginOptions,
 ): Promise<Feed | null> {
@@ -44,9 +54,8 @@ export async function generateBlogFeed(
       'Invalid options - `feedOptions` is not expected to be null.',
     );
   }
-  const {siteDir, siteConfig} = context;
-  const contentPath = path.resolve(siteDir, options.path);
-  const blogPosts = await generateBlogPosts(contentPath, context, options);
+  const {siteConfig} = context;
+  const blogPosts = await generateBlogPosts(contentPaths, context, options);
   if (blogPosts == null) {
     return null;
   }
@@ -88,7 +97,7 @@ export async function generateBlogFeed(
 }
 
 export async function generateBlogPosts(
-  blogDir: string,
+  contentPaths: BlogContentPaths,
   {siteConfig, siteDir}: LoadContext,
   options: PluginOptions,
 ): Promise<BlogPost[]> {
@@ -100,24 +109,30 @@ export async function generateBlogPosts(
     editUrl,
   } = options;
 
-  if (!fs.existsSync(blogDir)) {
+  if (!fs.existsSync(contentPaths.contentPath)) {
     return [];
   }
 
   const {baseUrl = ''} = siteConfig;
-  const blogFiles = await globby(include, {
-    cwd: blogDir,
+  const blogSourceFiles = await globby(include, {
+    cwd: contentPaths.contentPath,
   });
 
   const blogPosts: BlogPost[] = [];
 
   await Promise.all(
-    blogFiles.map(async (relativeSource: string) => {
-      const source = path.join(blogDir, relativeSource);
+    blogSourceFiles.map(async (blogSourceFile: string) => {
+      // Lookup in localized folder in priority
+      const contentPath = await getFolderContainingFile(
+        getContentPathList(contentPaths),
+        blogSourceFile,
+      );
+
+      const source = path.join(contentPath, blogSourceFile);
       const aliasedSource = aliasedSitePath(source, siteDir);
-      const refDir = path.parse(blogDir).dir;
-      const relativePath = path.relative(refDir, source);
-      const blogFileName = path.basename(relativeSource);
+
+      const relativePath = path.relative(siteDir, source);
+      const blogFileName = path.basename(blogSourceFile);
 
       const editBlogUrl = getEditUrl(relativePath, editUrl);
 
@@ -184,12 +199,31 @@ export async function generateBlogPosts(
   return blogPosts;
 }
 
-export function linkify(
-  fileContent: string,
-  siteDir: string,
-  blogPath: string,
-  blogPosts: BlogPost[],
-): string {
+export type LinkifyParams = {
+  filePath: string;
+  fileContent: string;
+} & Pick<
+  BlogMarkdownLoaderOptions,
+  'blogPosts' | 'siteDir' | 'contentPaths' | 'onBrokenMarkdownLink'
+>;
+
+export function linkify({
+  filePath,
+  contentPaths,
+  fileContent,
+  siteDir,
+  blogPosts,
+  onBrokenMarkdownLink,
+}: LinkifyParams): string {
+  // TODO temporary, should consider the file being in localized folder!
+  const folderPath = contentPaths.contentPath;
+
+  // TODO perf refactor: do this earlier (once for all md files, not per file)
+  const blogPostsBySource: Record<string, BlogPost> = keyBy(
+    blogPosts,
+    (item) => item.metadata.source,
+  );
+
   let fencedBlock = false;
   const lines = fileContent.split('\n').map((line) => {
     if (line.trim().startsWith('```')) {
@@ -208,18 +242,24 @@ export function linkify(
       const mdLink = mdMatch[1];
       const aliasedPostSource = `@site/${path.relative(
         siteDir,
-        path.resolve(blogPath, mdLink),
+        path.resolve(folderPath, mdLink),
       )}`;
-      let blogPostPermalink = null;
 
-      blogPosts.forEach((blogPost) => {
-        if (blogPost.metadata.source === aliasedPostSource) {
-          blogPostPermalink = blogPost.metadata.permalink;
-        }
-      });
+      const blogPost: BlogPost | undefined =
+        blogPostsBySource[aliasedPostSource];
 
-      if (blogPostPermalink) {
-        modifiedLine = modifiedLine.replace(mdLink, blogPostPermalink);
+      if (blogPost) {
+        modifiedLine = modifiedLine.replace(
+          mdLink,
+          blogPost.metadata.permalink,
+        );
+      } else {
+        const brokenMarkdownLink: BlogBrokenMarkdownLink = {
+          folderPath,
+          filePath,
+          link: mdLink,
+        };
+        onBrokenMarkdownLink(brokenMarkdownLink);
       }
 
       mdMatch = mdRegex.exec(modifiedLine);
@@ -230,3 +270,8 @@ export function linkify(
 
   return lines.join('\n');
 }
+
+// Order matters: we look in priority in localized folder
+export function getContentPathList(contentPaths: BlogContentPaths) {
+  return [contentPaths.contentPathLocalized, contentPaths.contentPath];
+}