docusaurus/packages/docusaurus-plugin-content-docs/src/sidebars/generator.ts

/**
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */

import type {
  SidebarItem,
  SidebarItemDoc,
  SidebarItemCategory,
  SidebarItemsGenerator,
  SidebarItemsGeneratorDoc,
  SidebarItemCategoryLink,
  SidebarItemCategoryLinkConfig,
} from './types';
import {sortBy, last} from 'lodash';
import {
  addTrailingSlash,
  posixPath,
  findAsyncSequential,
} from '@docusaurus/utils';
import logger from '@docusaurus/logger';
import path from 'path';
import fs from 'fs-extra';
import Yaml from 'js-yaml';
import {validateCategoryMetadataFile} from './validation';
import {createDocsByIdIndex, toCategoryIndexMatcherParam} from '../docs';

const BreadcrumbSeparator = '/';
// To avoid possible name clashes with a folder of the same name as the ID
const docIdPrefix = '$doc$/';

// Just an alias to the make code more explicit
function getLocalDocId(docId: string): string {
  return last(docId.split('/'))!;
}

export const CategoryMetadataFilenameBase = '_category_';
export const CategoryMetadataFilenamePattern = '_category_.{json,yml,yaml}';

export type CategoryMetadataFile = {
  label?: string;
  position?: number;
  collapsed?: boolean;
  collapsible?: boolean;
  className?: string;
  link?: SidebarItemCategoryLinkConfig | null;

  // TODO should we allow "items" here? how would this work? would an
  // "autogenerated" type be allowed?
  // This mkdocs plugin do something like that: https://github.com/lukasgeiter/mkdocs-awesome-pages-plugin/
  // cf comment: https://github.com/facebook/docusaurus/issues/3464#issuecomment-784765199
};

type WithPosition<T> = T & {position?: number};

/**
 * A representation of the fs structure. For each object entry:
 * If it's a folder, the key is the directory name, and value is the directory
 * content; If it's a doc file, the key is the doc id prefixed with '$doc$/',
 * and value is null
 */
type Dir = {
  [item: string]: Dir | null;
};

// TODO I now believe we should read all the category metadata files ahead of
// time: we may need this metadata to customize docs metadata
// Example use-case being able to disable number prefix parsing at the folder
// level, or customize the default base slug for an intermediate directory
// TODO later if there is `CategoryFolder/with-category-name-doc.md`, we may
// want to read the metadata as yaml on it
// see https://github.com/facebook/docusaurus/issues/3464#issuecomment-818670449
async function readCategoryMetadataFile(
  categoryDirPath: string,
): Promise<CategoryMetadataFile | null> {
  async function tryReadFile(filePath: string): Promise<CategoryMetadataFile> {
    const contentString = await fs.readFile(filePath, {encoding: 'utf8'});
    const unsafeContent = Yaml.load(contentString);
    try {
      return validateCategoryMetadataFile(unsafeContent);
    } catch (e) {
      logger.error`The docs sidebar category metadata file path=${filePath} looks invalid!`;
      throw e;
    }
  }
  const filePath = await findAsyncSequential(
    ['.json', '.yml', '.yaml'].map((ext) =>
      posixPath(
        path.join(categoryDirPath, `${CategoryMetadataFilenameBase}${ext}`),
      ),
    ),
    fs.pathExists,
  );
  return filePath ? tryReadFile(filePath) : null;
}

// Comment for this feature: https://github.com/facebook/docusaurus/issues/3464#issuecomment-818670449
export const DefaultSidebarItemsGenerator: SidebarItemsGenerator = async ({
  numberPrefixParser,
  isCategoryIndex,
  docs: allDocs,
  options,
  item: {dirName: autogenDir},
  version,
}) => {
  const docsById = createDocsByIdIndex(allDocs);
  const findDoc = (docId: string): SidebarItemsGeneratorDoc | undefined =>
    docsById[docId];
  const getDoc = (docId: string): SidebarItemsGeneratorDoc => {
    const doc = findDoc(docId);
    if (!doc) {
      throw new Error(
        `Can't find any doc with id=${docId}.\nAvailable doc ids:\n- ${Object.keys(
          docsById,
        ).join('\n- ')}`,
      );
    }
    return doc;
  };

  /**
   * Step 1. Extract the docs that are in the autogen dir.
   */
  function getAutogenDocs(): SidebarItemsGeneratorDoc[] {
    function isInAutogeneratedDir(doc: SidebarItemsGeneratorDoc) {
      return (
        // Doc at the root of the autogenerated sidebar dir
        doc.sourceDirName === autogenDir ||
        // autogen dir is . and doc is in subfolder
        autogenDir === '.' ||
        // autogen dir is not . and doc is in subfolder
        // "api/myDoc" startsWith "api/" (note "api2/myDoc" is not included)
        doc.sourceDirName.startsWith(addTrailingSlash(autogenDir))
      );
    }
    const docs = allDocs.filter(isInAutogeneratedDir);

    if (docs.length === 0) {
      logger.warn`No docs found in path=${autogenDir}: can't auto-generate a sidebar.`;
    }
    return docs;
  }

  /**
   * Step 2. Turn the linear file list into a tree structure.
   */
  function treeify(docs: SidebarItemsGeneratorDoc[]): Dir {
    // Get the category breadcrumb of a doc (relative to the dir of the
    // autogenerated sidebar item)
    // autogenDir=a/b and docDir=a/b/c/d => returns [c, d]
    // autogenDir=a/b and docDir=a/b => returns []
    // TODO: try to use path.relative()
    function getRelativeBreadcrumb(doc: SidebarItemsGeneratorDoc): string[] {
      return autogenDir === doc.sourceDirName
        ? []
        : doc.sourceDirName
            .replace(addTrailingSlash(autogenDir), '')
            .split(BreadcrumbSeparator);
    }
    const treeRoot: Dir = {};
    docs.forEach((doc) => {
      const breadcrumb = getRelativeBreadcrumb(doc);
      let currentDir = treeRoot; // We walk down the file's path to generate the fs structure
      breadcrumb.forEach((dir) => {
        if (typeof currentDir[dir] === 'undefined') {
          currentDir[dir] = {}; // Create new folder.
        }
        currentDir = currentDir[dir]!; // Go into the subdirectory.
      });
      currentDir[`${docIdPrefix}${doc.id}`] = null; // We've walked through the file path. Register the file in this directory.
    });
    return treeRoot;
  }

  /**
   * Step 3. Recursively transform the tree-like structure to sidebar items.
   * (From a record to an array of items, akin to normalizing shorthand)
   */
  function generateSidebar(fsModel: Dir): Promise<WithPosition<SidebarItem>[]> {
    function createDocItem(id: string): WithPosition<SidebarItemDoc> {
      const {
        sidebarPosition: position,
        frontMatter: {sidebar_label: label, sidebar_class_name: className},
      } = getDoc(id);
      return {
        type: 'doc',
        id,
        position,
        // We don't want these fields to magically appear in the generated
        // sidebar
        ...(label !== undefined && {label}),
        ...(className !== undefined && {className}),
      };
    }
    async function createCategoryItem(
      dir: Dir,
      fullPath: string,
      folderName: string,
    ): Promise<WithPosition<SidebarItemCategory>> {
      const categoryPath = path.join(version.contentPath, autogenDir, fullPath);
      const categoryMetadata = await readCategoryMetadataFile(categoryPath);
      const className = categoryMetadata?.className;
      const {filename, numberPrefix} = numberPrefixParser(folderName);
      const allItems = await Promise.all(
        Object.entries(dir).map(([key, content]) =>
          dirToItem(content, key, `${fullPath}/${key}`),
        ),
      );

      // Try to match a doc inside the category folder,
      // using the "local id" (myDoc) or "qualified id" (dirName/myDoc)
      function findDocByLocalId(localId: string): SidebarItemDoc | undefined {
        return allItems.find(
          (item) => item.type === 'doc' && getLocalDocId(item.id) === localId,
        ) as SidebarItemDoc | undefined;
      }

      function findConventionalCategoryDocLink(): SidebarItemDoc | undefined {
        return allItems.find((item) => {
          if (item.type !== 'doc') {
            return false;
          }
          const doc = getDoc(item.id);
          return isCategoryIndex(toCategoryIndexMatcherParam(doc));
        }) as SidebarItemDoc | undefined;
      }

      function getCategoryLinkedDocId(): string | undefined {
        const link = categoryMetadata?.link;
        if (link !== undefined) {
          if (link && link.type === 'doc') {
            return findDocByLocalId(link.id)?.id || getDoc(link.id).id;
          }
          // If a link is explicitly specified, we won't apply conventions
          return undefined;
        }
        // Apply default convention to pick index.md, README.md or
        // <categoryName>.md as the category doc
        return findConventionalCategoryDocLink()?.id;
      }

      const categoryLinkedDocId = getCategoryLinkedDocId();

      const link: SidebarItemCategoryLink | undefined = categoryLinkedDocId
        ? {
            type: 'doc',
            id: categoryLinkedDocId, // We "remap" a potentially "local id" to a "qualified id"
          }
        : // TODO typing issue
          (categoryMetadata?.link as SidebarItemCategoryLink | undefined);

      // If a doc is linked, remove it from the category subItems
      const items = allItems.filter(
        (item) => !(item.type === 'doc' && item.id === categoryLinkedDocId),
      );

      return {
        type: 'category',
        label: categoryMetadata?.label ?? filename,
        collapsible:
          categoryMetadata?.collapsible ?? options.sidebarCollapsible,
        collapsed: categoryMetadata?.collapsed ?? options.sidebarCollapsed,
        position: categoryMetadata?.position ?? numberPrefix,
        ...(className !== undefined && {className}),
        items,
        ...(link && {link}),
      };
    }
    async function dirToItem(
      dir: Dir | null, // The directory item to be transformed.
      itemKey: string, // For docs, it's the doc ID; for categories, it's used to generate the next `relativePath`.
      fullPath: string, // `dir`'s full path relative to the autogen dir.
    ): Promise<WithPosition<SidebarItem>> {
      return dir
        ? createCategoryItem(dir, fullPath, itemKey)
        : createDocItem(itemKey.substring(docIdPrefix.length));
    }
    return Promise.all(
      Object.entries(fsModel).map(([key, content]) =>
        dirToItem(content, key, key),
      ),
    );
  }

  /**
   * Step 4. Recursively sort the categories/docs + remove the "position"
   * attribute from final output. Note: the "position" is only used to sort
   * "inside" a sidebar slice. It is not used to sort across multiple
   * consecutive sidebar slices (i.e. a whole category composed of multiple
   * autogenerated items)
   */
  function sortItems(sidebarItems: WithPosition<SidebarItem>[]): SidebarItem[] {
    const processedSidebarItems = sidebarItems.map((item) => {
      if (item.type === 'category') {
        return {...item, items: sortItems(item.items)};
      }
      return item;
    });
    const sortedSidebarItems = sortBy(
      processedSidebarItems,
      (item) => item.position,
    );
    return sortedSidebarItems.map(({position, ...item}) => item);
  }
  // TODO: the whole code is designed for pipeline operator
  const docs = getAutogenDocs();
  const fsModel = treeify(docs);
  const sidebarWithPosition = await generateSidebar(fsModel);
  const sortedSidebar = sortItems(sidebarWithPosition);
  return sortedSidebar;
};