refactor(content-docs): read category metadata files before autogenerating (#6586)

* refactor(content-docs): read category metadata files before autogenerating

* fix tests

* fix Windows...

* warn user when behavior is undetermined

* oops

* fix typo

packages/docusaurus-plugin-content-docs/src/__tests__/__snapshots__/index.test.ts.snap

@@ -1361,6 +1361,17 @@ Object {
 
 exports[`site with custom sidebar items generator sidebarItemsGenerator is called with appropriate data 1`] = `
 Object {
+  "categoriesMetadata": Object {
+    "3-API": Object {
+      "label": "API (label from _category_.json)",
+    },
+    "3-API/02_Extension APIs": Object {
+      "label": "Extension APIs (label from _category_.yml)",
+    },
+    "Guides": Object {
+      "position": 2,
+    },
+  },
   "defaultSidebarItemsGenerator": [Function],
   "docs": Array [
     Object {

packages/docusaurus-plugin-content-docs/src/sidebars/__tests__/generator.test.ts

@@ -5,12 +5,8 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-import {
+import {DefaultSidebarItemsGenerator} from '../generator';
-  DefaultSidebarItemsGenerator,
-  type CategoryMetadataFile,
-} from '../generator';
 import type {Sidebar, SidebarItemsGenerator} from '../types';
-import fs from 'fs-extra';
 import {DefaultNumberPrefixParser} from '../../numberPrefix';
 import {isCategoryIndex} from '../../docs';
 
@@ -34,26 +30,11 @@ describe('DefaultSidebarItemsGenerator', () => {
         sidebarCollapsed: true,
         sidebarCollapsible: true,
       },
+      categoriesMetadata: {},
       ...params,
     });
   }
 
-  function mockCategoryMetadataFiles(
-    categoryMetadataFiles: Record<string, Partial<CategoryMetadataFile>>,
-  ) {
-    jest
-      .spyOn(fs, 'pathExists')
-      .mockImplementation(
-        (metadataFilePath) =>
-          typeof categoryMetadataFiles[metadataFilePath] !== 'undefined',
-      );
-    jest.spyOn(fs, 'readFile').mockImplementation(
-      // @ts-expect-error: annoying TS error due to overrides
-      async (metadataFilePath: string) =>
-        JSON.stringify(categoryMetadataFiles[metadataFilePath]),
-    );
-  }
-
   test('generates empty sidebar slice when no docs and emit a warning', async () => {
     const consoleWarn = jest.spyOn(console, 'warn');
     const sidebarSlice = await testDefaultSidebarItemsGenerator({
@@ -133,19 +114,6 @@ describe('DefaultSidebarItemsGenerator', () => {
   });
 
   test('generates complex nested sidebar', async () => {
-    mockCategoryMetadataFiles({
-      '02-Guides/_category_.json': {collapsed: false} as CategoryMetadataFile,
-      '02-Guides/01-SubGuides/_category_.yml': {
-        label: 'SubGuides (metadata file label)',
-        link: {
-          type: 'generated-index',
-          slug: 'subguides-generated-index-slug',
-          title: 'subguides-title',
-          description: 'subguides-description',
-        },
-      },
-    });
-
     const sidebarSlice = await DefaultSidebarItemsGenerator({
       numberPrefixParser: DefaultNumberPrefixParser,
       isCategoryIndex,
@@ -157,6 +125,18 @@ describe('DefaultSidebarItemsGenerator', () => {
         versionName: 'current',
         contentPath: '',
       },
+      categoriesMetadata: {
+        '02-Guides': {collapsed: false},
+        '02-Guides/01-SubGuides': {
+          label: 'SubGuides (metadata file label)',
+          link: {
+            type: 'generated-index',
+            slug: 'subguides-generated-index-slug',
+            title: 'subguides-title',
+            description: 'subguides-description',
+          },
+        },
+      },
       docs: [
         {
           id: 'intro',
@@ -279,24 +259,6 @@ describe('DefaultSidebarItemsGenerator', () => {
   test('generates subfolder sidebar', async () => {
     // Ensure that category metadata file is correctly read
     // fix edge case found in https://github.com/facebook/docusaurus/issues/4638
-    mockCategoryMetadataFiles({
-      'subfolder/subsubfolder/subsubsubfolder2/_category_.yml': {
-        position: 2,
-        label: 'subsubsubfolder2 (_category_.yml label)',
-        className: 'bar',
-      },
-      'subfolder/subsubfolder/subsubsubfolder3/_category_.json': {
-        position: 1,
-        label: 'subsubsubfolder3 (_category_.json label)',
-        collapsible: false,
-        collapsed: false,
-        link: {
-          type: 'doc',
-          id: 'doc1', // This is a "fully-qualified" ID that can't be found locally
-        },
-      },
-    });
-
     const sidebarSlice = await DefaultSidebarItemsGenerator({
       numberPrefixParser: DefaultNumberPrefixParser,
       isCategoryIndex,
@@ -308,6 +270,23 @@ describe('DefaultSidebarItemsGenerator', () => {
         versionName: 'current',
         contentPath: '',
       },
+      categoriesMetadata: {
+        'subfolder/subsubfolder/subsubsubfolder2': {
+          position: 2,
+          label: 'subsubsubfolder2 (_category_.yml label)',
+          className: 'bar',
+        },
+        'subfolder/subsubfolder/subsubsubfolder3': {
+          position: 1,
+          label: 'subsubsubfolder3 (_category_.json label)',
+          collapsible: false,
+          collapsed: false,
+          link: {
+            type: 'doc',
+            id: 'doc1', // This is a "fully-qualified" ID that can't be found locally
+          },
+        },
+      },
       docs: [
         {
           id: 'doc1',
@@ -408,20 +387,6 @@ describe('DefaultSidebarItemsGenerator', () => {
   });
 
   test('uses explicit link over the index/readme.{md,mdx} naming convention', async () => {
-    mockCategoryMetadataFiles({
-      'Category/_category_.yml': {
-        label: 'Category label',
-        link: {
-          type: 'doc',
-          id: 'doc3', // Using a "local doc id" ("doc1" instead of "parent/doc1") on purpose
-        },
-      },
-      'Category2/_category_.yml': {
-        label: 'Category 2 label',
-        link: null,
-      },
-    });
-
     const sidebarSlice = await DefaultSidebarItemsGenerator({
       numberPrefixParser: DefaultNumberPrefixParser,
       item: {
@@ -432,6 +397,19 @@ describe('DefaultSidebarItemsGenerator', () => {
         versionName: 'current',
         contentPath: '',
       },
+      categoriesMetadata: {
+        Category: {
+          label: 'Category label',
+          link: {
+            type: 'doc',
+            id: 'doc3', // Using a "local doc id" ("doc1" instead of "parent/doc1") on purpose
+          },
+        },
+        Category2: {
+          label: 'Category 2 label',
+          link: null,
+        },
+      },
       docs: [
         {
           id: 'parent/doc1',
@@ -541,6 +519,7 @@ describe('DefaultSidebarItemsGenerator', () => {
         versionName: 'current',
         contentPath: '',
       },
+      categoriesMetadata: {},
       docs: [
         {
           id: 'intro',

packages/docusaurus-plugin-content-docs/src/sidebars/generator.ts

@@ -12,19 +12,11 @@ import type {
   SidebarItemsGenerator,
   SidebarItemsGeneratorDoc,
   SidebarItemCategoryLink,
-  SidebarItemCategoryLinkConfig,
 } from './types';
 import {sortBy, last} from 'lodash';
-import {
+import {addTrailingSlash, posixPath} from '@docusaurus/utils';
-  addTrailingSlash,
-  posixPath,
-  findAsyncSequential,
-} from '@docusaurus/utils';
 import logger from '@docusaurus/logger';
 import path from 'path';
-import fs from 'fs-extra';
-import Yaml from 'js-yaml';
-import {validateCategoryMetadataFile} from './validation';
 import {createDocsByIdIndex, toCategoryIndexMatcherParam} from '../docs';
 
 const BreadcrumbSeparator = '/';
@@ -39,20 +31,6 @@ function getLocalDocId(docId: string): string {
 export const CategoryMetadataFilenameBase = '_category_';
 export const CategoryMetadataFilenamePattern = '_category_.{json,yml,yaml}';
 
-export type CategoryMetadataFile = {
-  label?: string;
-  position?: number;
-  collapsed?: boolean;
-  collapsible?: boolean;
-  className?: string;
-  link?: SidebarItemCategoryLinkConfig | null;
-
-  // TODO should we allow "items" here? how would this work? would an
-  // "autogenerated" type be allowed?
-  // This mkdocs plugin do something like that: https://github.com/lukasgeiter/mkdocs-awesome-pages-plugin/
-  // cf comment: https://github.com/facebook/docusaurus/issues/3464#issuecomment-784765199
-};
-
 type WithPosition<T> = T & {position?: number};
 
 /**
@@ -65,37 +43,6 @@ type Dir = {
   [item: string]: Dir | null;
 };
 
-// TODO I now believe we should read all the category metadata files ahead of
-// time: we may need this metadata to customize docs metadata
-// Example use-case being able to disable number prefix parsing at the folder
-// level, or customize the default base slug for an intermediate directory
-// TODO later if there is `CategoryFolder/with-category-name-doc.md`, we may
-// want to read the metadata as yaml on it
-// see https://github.com/facebook/docusaurus/issues/3464#issuecomment-818670449
-async function readCategoryMetadataFile(
-  categoryDirPath: string,
-): Promise<CategoryMetadataFile | null> {
-  async function tryReadFile(filePath: string): Promise<CategoryMetadataFile> {
-    const contentString = await fs.readFile(filePath, {encoding: 'utf8'});
-    const unsafeContent = Yaml.load(contentString);
-    try {
-      return validateCategoryMetadataFile(unsafeContent);
-    } catch (e) {
-      logger.error`The docs sidebar category metadata file path=${filePath} looks invalid!`;
-      throw e;
-    }
-  }
-  const filePath = await findAsyncSequential(
-    ['.json', '.yml', '.yaml'].map((ext) =>
-      posixPath(
-        path.join(categoryDirPath, `${CategoryMetadataFilenameBase}${ext}`),
-      ),
-    ),
-    fs.pathExists,
-  );
-  return filePath ? tryReadFile(filePath) : null;
-}
-
 // Comment for this feature: https://github.com/facebook/docusaurus/issues/3464#issuecomment-818670449
 export const DefaultSidebarItemsGenerator: SidebarItemsGenerator = async ({
   numberPrefixParser,
@@ -103,7 +50,7 @@ export const DefaultSidebarItemsGenerator: SidebarItemsGenerator = async ({
   docs: allDocs,
   options,
   item: {dirName: autogenDir},
-  version,
+  categoriesMetadata,
 }) => {
   const docsById = createDocsByIdIndex(allDocs);
   const findDoc = (docId: string): SidebarItemsGeneratorDoc | undefined =>
@@ -199,8 +146,8 @@ export const DefaultSidebarItemsGenerator: SidebarItemsGenerator = async ({
       fullPath: string,
       folderName: string,
     ): Promise<WithPosition<SidebarItemCategory>> {
-      const categoryPath = path.join(version.contentPath, autogenDir, fullPath);
+      const categoryMetadata =
-      const categoryMetadata = await readCategoryMetadataFile(categoryPath);
+        categoriesMetadata[posixPath(path.join(autogenDir, fullPath))];
       const className = categoryMetadata?.className;
       const {filename, numberPrefix} = numberPrefixParser(folderName);
       const allItems = await Promise.all(

packages/docusaurus-plugin-content-docs/src/sidebars/index.ts

@@ -9,12 +9,16 @@ import fs from 'fs-extra';
 import importFresh from 'import-fresh';
 import type {SidebarsConfig, Sidebars, NormalizedSidebars} from './types';
 import type {NormalizeSidebarsParams} from '../types';
-import {validateSidebars} from './validation';
+import {validateSidebars, validateCategoryMetadataFile} from './validation';
 import {normalizeSidebars} from './normalization';
 import {processSidebars, type SidebarProcessorParams} from './processor';
 import path from 'path';
-import {createSlugger} from '@docusaurus/utils';
+import {createSlugger, Globby} from '@docusaurus/utils';
+import logger from '@docusaurus/logger';
 import type {PluginOptions} from '@docusaurus/plugin-content-docs';
+import Yaml from 'js-yaml';
+import {groupBy, mapValues} from 'lodash';
+import combinePromises from 'combine-promises';
 
 export const DefaultSidebars: SidebarsConfig = {
   defaultSidebar: [
@@ -38,6 +42,33 @@ export function resolveSidebarPathOption(
     : sidebarPathOption;
 }
 
+async function readCategoriesMetadata(contentPath: string) {
+  const categoryFiles = await Globby('**/_category_.{json,yml,yaml}', {
+    cwd: contentPath,
+  });
+  const categoryToFile = groupBy(categoryFiles, path.dirname);
+  return combinePromises(
+    mapValues(categoryToFile, async (files, folder) => {
+      const [filePath] = files;
+      if (files.length > 1) {
+        logger.warn`There are more than one category metadata files for path=${folder}: ${files.join(
+          ', ',
+        )}. The behavior is undetermined.`;
+      }
+      const content = await fs.readFile(
+        path.join(contentPath, filePath),
+        'utf-8',
+      );
+      try {
+        return validateCategoryMetadataFile(Yaml.load(content));
+      } catch (e) {
+        logger.error`The docs sidebar category metadata file path=${filePath} looks invalid!`;
+        throw e;
+      }
+    }),
+  );
+}
+
 async function loadSidebarsFileUnsafe(
   sidebarFilePath: string | false | undefined,
 ): Promise<SidebarsConfig> {
@@ -80,7 +111,7 @@ export async function loadNormalizedSidebars(
 // Note: sidebarFilePath must be absolute, use resolveSidebarPathOption
 export async function loadSidebars(
   sidebarFilePath: string | false | undefined,
-  options: SidebarProcessorParams,
+  options: Omit<SidebarProcessorParams, 'categoriesMetadata'>,
 ): Promise<Sidebars> {
   const normalizeSidebarsParams: NormalizeSidebarsParams = {
     ...options.sidebarOptions,
@@ -91,5 +122,8 @@ export async function loadSidebars(
     sidebarFilePath,
     normalizeSidebarsParams,
   );
-  return processSidebars(normalizedSidebars, options);
+  const categoriesMetadata = await readCategoriesMetadata(
+    options.version.contentPath,
+  );
+  return processSidebars(normalizedSidebars, {...options, categoriesMetadata});
 }

packages/docusaurus-plugin-content-docs/src/sidebars/processor.ts

@@ -19,6 +19,7 @@ import type {
   NormalizedSidebarItemCategory,
   SidebarItemCategory,
   SidebarItemAutogenerated,
+  CategoryMetadataFile,
 } from './types';
 import {transformSidebarItems} from './utils';
 import {DefaultSidebarItemsGenerator} from './generator';
@@ -39,6 +40,7 @@ export type SidebarProcessorParams = {
   version: VersionMetadata;
   categoryLabelSlugger: Slugger;
   sidebarOptions: SidebarOptions;
+  categoriesMetadata: Record<string, CategoryMetadataFile>;
 };
 
 function toSidebarItemsGeneratorDoc(
@@ -72,6 +74,7 @@ async function processSidebar(
     docs,
     version,
     sidebarOptions,
+    categoriesMetadata,
   } = params;
 
   // Just a minor lazy transformation optimization
@@ -101,6 +104,7 @@ async function processSidebar(
       isCategoryIndex,
       ...getSidebarItemsGeneratorDocsAndVersion(),
       options: sidebarOptions,
+      categoriesMetadata,
     });
     // TODO validate generated items: user can generate bad items
 

packages/docusaurus-plugin-content-docs/src/sidebars/types.ts

@@ -188,6 +188,20 @@ export type PropVersionDocs = {
   [docId: string]: PropVersionDoc;
 };
 
+export type CategoryMetadataFile = {
+  label?: string;
+  position?: number;
+  collapsed?: boolean;
+  collapsible?: boolean;
+  className?: string;
+  link?: SidebarItemCategoryLinkConfig | null;
+
+  // TODO should we allow "items" here? how would this work? would an
+  // "autogenerated" type be allowed?
+  // This mkdocs plugin do something like that: https://github.com/lukasgeiter/mkdocs-awesome-pages-plugin/
+  // cf comment: https://github.com/facebook/docusaurus/issues/3464#issuecomment-784765199
+};
+
 // Reduce API surface for options.sidebarItemsGenerator
 // The user-provided generator fn should receive only a subset of metadata
 // A change to any of these metadata can be considered as a breaking change
@@ -211,6 +225,7 @@ export type SidebarItemsGeneratorArgs = {
   docs: SidebarItemsGeneratorDoc[];
   numberPrefixParser: NumberPrefixParser;
   isCategoryIndex: CategoryIndexMatcher;
+  categoriesMetadata: Record<string, CategoryMetadataFile>;
   options: SidebarOptions;
 };
 export type SidebarItemsGenerator = (

packages/docusaurus-plugin-content-docs/src/sidebars/validation.ts

@@ -19,9 +19,9 @@ import type {
   SidebarsConfig,
   SidebarItemCategoryLinkDoc,
   SidebarItemCategoryLinkGeneratedIndex,
+  CategoryMetadataFile,
 } from './types';
 import {isCategoriesShorthand} from './utils';
-import type {CategoryMetadataFile} from './generator';
 
 // NOTE: we don't add any default values during validation on purpose!
 // Config types are exposed to users for typechecking and we use the same type

website/docs/api/plugins/plugin-content-docs.md

@@ -94,6 +94,7 @@ type SidebarGenerator = (generatorArgs: {
     sidebarPosition?: number | undefined;
   }>; // all the docs of that version (unfiltered)
   numberPrefixParser: PrefixParser; // numberPrefixParser configured for this plugin
+  categoriesMetadata: Record<string, CategoryMetadata>; // key is the path relative to the doc directory, value is the category metadata file's content
   isCategoryIndex: CategoryIndexMatcher; // the default category index matcher, that you can override
   defaultSidebarItemsGenerator: SidebarGenerator; // useful to re-use/enhance default sidebar generation logic from Docusaurus
 }) => Promise<SidebarItem[]>;

website/docs/guides/docs/sidebar/autogenerated.md

@@ -428,6 +428,7 @@ module.exports = {
           item,
           version,
           docs,
+          categoriesMetadata,
           isCategoryIndex,
         }) {
           // Example: return an hardcoded list of static sidebar items