feat(v2): core v2 i18n support + Docusaurus site Crowdin integration (#3325)

* docs i18n initial poc

* docs i18n initial poc

* docs i18n initial poc

* docs i18n initial poc

* crowdin-v2 attempt

* fix source

* use crowdin env variable

* try to install crowdin on netlify

* try to install crowdin on netlify

* try to use crowdin jar directly

* try to curl the crowdin jar

* add java version cmd

* try to run crowdin jar in netlify

* fix translatedDocsDirPath

* fix loadContext issue due to site baseUrl not being modified in generted config file

* real validateLocalesFile

* add locale option to deploy command

* better LocalizationFile type

* create util getPluginI18nPath

* better core localization context loading code

* More explicit VersionMetadata type for localized docs folders

* Ability to translate blog posts with Crowdin!

* blog: refactor markdown loader + report broken links + try to get linkify working better

* upgrade crowdin config to upload all docs folder files except source code related files

* try to support translated pages

* make markdown pages translation work

* add write-translations cli command template

* fix site not  reloaded with correct options

* refactor a bit the read/write of @generated/i18n.json file

* Add <Translate> + translate() API + use it on the docusaurus homepage

* watch locale translation dir

* early POC of adding babel parsing for translation extraction

* fs.stat => pathExists

* add install:fast script

* TSC: noUnusedLocals false as it's already checked  by eslint

* POC of extracting translations from source code

* minor typo

* fix extracted key to code

* initial docs extracted translations

* stable plugin translations POC

* add crowdin commands

* quickfix for i18n deployment

* POC  of themeConfig translation

* add ability to have localized site without path prefix

* sidebar typo

* refactor translation system to output multiple translation files

* translate properly  the docs plugin

* improve theme classic translation

* rework translation extractor to handle new Chrome I18n JSON format (include id/description)

* writeTranslations: allow to pass locales cli arg

* fix ThemeConfig TS issues

* fix localizePath errors

* temporary add write-translations to netlify deploy preview

* complete example of french translated folder

* update fr folder

* remove all translations from repo

* minor translation  refactors

* fix all docs-related tests

* fix blog feed tests

* fix last blog tests

* refactor i18n context a bit, extract codeTranslations in an extra generated file

* improve @generated/i18n type

* fix some i18n todos

* minor refactor

* fix logo typing issue after merge

* move i18n.json to siteConfig instead

* try to fix windows CI build

* fix config test

* attempt to fix windows non-posix path

* increase v1 minify css jest timeout due to flaky test

* proper support for localizePath on windows

* remove non-functional install:fast

* docs, fix docsDirPathLocalized

* fix Docs i18n / md linkify issues

* ensure theme-classic swizzling will use "nextjs" sources (transpiled less aggressively, to make them human readable)

* fix some snapshots

* improve themeConfig translation code

* refactor a bit getPluginI18nPath

* readTranslationFileContent => ensure files are valid, fail fast

* fix versions tests

* add extractSourceCodeAstTranslations comments/resource links

* ignore eslint: packages/docusaurus-theme-classic/lib-next/

* fix windows CI with cross-env

* crowdin ignore .DS_Store

* improve writeTranslations + add exhaustive tests for translations.ts

* remove typo

* Wire currentLocale to algolia search

* improve i18n locale error

* Add tests for translationsExtractor.ts

* better code translation extraction regarding statically evaluable code

* fix typo

* fix typo

* improve theme-classic transpilation

* refactor  +  add i18n tests

* typo

* test new utils

* add missing snapshots

* fix snapshot

* blog onBrokenMarkdownLink

* add sidebars tests

* theme-classic index should now use ES modules

* tests for theme-classic translations

* useless comment

* add more translation tests

* simplify/cleanup writeTranslations

* try to fix Netlify fr deployment

* blog: test translated md is used during feed generation

* blog: better i18n tests regarding editUrl + md translation application

* more i18n tests for docs plugin

* more i18n tests for docs plugin

* Add tests for pages i18n

* polish docusaurus build i18n logs

packages/docusaurus-plugin-content-blog/src/index.ts

@@ -8,13 +8,19 @@
 import fs from 'fs-extra';
 import path from 'path';
 import admonitions from 'remark-admonitions';
-import {normalizeUrl, docuHash, aliasedSitePath} from '@docusaurus/utils';
+import {
+  normalizeUrl,
+  docuHash,
+  aliasedSitePath,
+  getPluginI18nPath,
+  reportMessage,
+} from '@docusaurus/utils';
 import {
   STATIC_DIR_NAME,
   DEFAULT_PLUGIN_ID,
 } from '@docusaurus/core/lib/constants';
 import {ValidationError} from 'joi';
-import {take, kebabCase} from 'lodash';
+import {flatten, take, kebabCase} from 'lodash';
 
 import {
   PluginOptions,
@@ -24,6 +30,8 @@ import {
   TagsModule,
   BlogPaginated,
   BlogPost,
+  BlogContentPaths,
+  BlogMarkdownLoaderOptions,
 } from './types';
 import {PluginOptionSchema} from './pluginOptionSchema';
 import {
@@ -36,7 +44,11 @@ import {
   ValidationResult,
 } from '@docusaurus/types';
 import {Configuration, Loader} from 'webpack';
-import {generateBlogFeed, generateBlogPosts} from './blogUtils';
+import {
+  generateBlogFeed,
+  generateBlogPosts,
+  getContentPathList,
+} from './blogUtils';
 
 export default function pluginContentBlog(
   context: LoadContext,
@@ -48,8 +60,22 @@ export default function pluginContentBlog(
     ]);
   }
 
-  const {siteDir, generatedFilesDir} = context;
-  const contentPath = path.resolve(siteDir, options.path);
+  const {
+    siteDir,
+    siteConfig: {onBrokenMarkdownLinks},
+    generatedFilesDir,
+    i18n: {currentLocale},
+  } = context;
+
+  const contentPaths: BlogContentPaths = {
+    contentPath: path.resolve(siteDir, options.path),
+    contentPathLocalized: getPluginI18nPath({
+      siteDir,
+      locale: currentLocale,
+      pluginName: 'docusaurus-plugin-content-blog',
+      pluginId: options.id,
+    }),
+  };
   const pluginId = options.id ?? DEFAULT_PLUGIN_ID;
 
   const pluginDataDirRoot = path.join(
@@ -67,8 +93,11 @@ export default function pluginContentBlog(
 
     getPathsToWatch() {
       const {include = []} = options;
-      const globPattern = include.map((pattern) => `${contentPath}/${pattern}`);
-      return [...globPattern];
+      return flatten(
+        getContentPathList(contentPaths).map((contentPath) => {
+          return include.map((pattern) => `${contentPath}/${pattern}`);
+        }),
+      );
     },
 
     getClientModules() {
@@ -85,7 +114,7 @@ export default function pluginContentBlog(
     async loadContent() {
       const {postsPerPage, routeBasePath} = options;
 
-      blogPosts = await generateBlogPosts(contentPath, context, options);
+      blogPosts = await generateBlogPosts(contentPaths, context, options);
       if (!blogPosts.length) {
         return null;
       }
@@ -379,6 +408,23 @@ export default function pluginContentBlog(
         beforeDefaultRemarkPlugins,
         beforeDefaultRehypePlugins,
       } = options;
+
+      const markdownLoaderOptions: BlogMarkdownLoaderOptions = {
+        siteDir,
+        contentPaths,
+        truncateMarker,
+        blogPosts,
+        onBrokenMarkdownLink: (brokenMarkdownLink) => {
+          if (onBrokenMarkdownLinks === 'ignore') {
+            return;
+          }
+          reportMessage(
+            `Blog markdown link couldn't be resolved: (${brokenMarkdownLink.link}) in ${brokenMarkdownLink.filePath}`,
+            onBrokenMarkdownLinks,
+          );
+        },
+      };
+
       return {
         resolve: {
           alias: {
@@ -389,7 +435,7 @@ export default function pluginContentBlog(
           rules: [
             {
               test: /(\.mdx?)$/,
-              include: [contentPath],
+              include: getContentPathList(contentPaths),
               use: [
                 getCacheLoader(isServer),
                 getBabelLoader(isServer),
@@ -414,12 +460,7 @@ export default function pluginContentBlog(
                 },
                 {
                   loader: path.resolve(__dirname, './markdownLoader.js'),
-                  options: {
-                    siteDir,
-                    contentPath,
-                    truncateMarker,
-                    blogPosts,
-                  },
+                  options: markdownLoaderOptions,
                 },
               ].filter(Boolean) as Loader[],
             },
@@ -433,7 +474,7 @@ export default function pluginContentBlog(
         return;
       }
 
-      const feed = await generateBlogFeed(context, options);
+      const feed = await generateBlogFeed(contentPaths, context, options);
 
       if (!feed) {
         return;