feat(sitemap): add support for "lastmod" (#9954)

packages/docusaurus-utils/src/__tests__/gitUtils.test.ts

@@ -9,7 +9,7 @@ import fs from 'fs-extra';
 import path from 'path';
 import {createTempRepo} from '@testing-utils/git';
 import {FileNotTrackedError, getFileCommitDate} from '../gitUtils';
-import {getFileLastUpdate} from '../lastUpdateUtils';
+import {getGitLastUpdate} from '../lastUpdateUtils';
 
 /* eslint-disable no-restricted-properties */
 function initializeTempRepo() {
@@ -146,8 +146,9 @@ describe('getFileCommitDate', () => {
     const tempFilePath2 = path.join(repoDir, 'file2.md');
     await fs.writeFile(tempFilePath1, 'Lorem ipsum :)');
     await fs.writeFile(tempFilePath2, 'Lorem ipsum :)');
-    await expect(getFileLastUpdate(tempFilePath1)).resolves.toBeNull();
-    await expect(getFileLastUpdate(tempFilePath2)).resolves.toBeNull();
+    // TODO this is not the correct place to test "getGitLastUpdate"
+    await expect(getGitLastUpdate(tempFilePath1)).resolves.toBeNull();
+    await expect(getGitLastUpdate(tempFilePath2)).resolves.toBeNull();
     expect(consoleMock).toHaveBeenCalledTimes(1);
     expect(consoleMock).toHaveBeenLastCalledWith(
       expect.stringMatching(/not tracked by git./),

packages/docusaurus-utils/src/__tests__/lastUpdateUtils.test.ts

@@ -11,13 +11,12 @@ import path from 'path';
 import {createTempRepo} from '@testing-utils/git';
 import shell from 'shelljs';
 import {
-  getFileLastUpdate,
-  GIT_FALLBACK_LAST_UPDATE_AUTHOR,
-  GIT_FALLBACK_LAST_UPDATE_DATE,
+  getGitLastUpdate,
+  LAST_UPDATE_FALLBACK,
   readLastUpdateData,
 } from '@docusaurus/utils';
 
-describe('getFileLastUpdate', () => {
+describe('getGitLastUpdate', () => {
   const {repoDir} = createTempRepo();
 
   const existingFilePath = path.join(
@@ -25,15 +24,15 @@ describe('getFileLastUpdate', () => {
     '__fixtures__/simple-site/hello.md',
   );
   it('existing test file in repository with Git timestamp', async () => {
-    const lastUpdateData = await getFileLastUpdate(existingFilePath);
+    const lastUpdateData = await getGitLastUpdate(existingFilePath);
     expect(lastUpdateData).not.toBeNull();
 
-    const {author, timestamp} = lastUpdateData!;
-    expect(author).not.toBeNull();
-    expect(typeof author).toBe('string');
+    const {lastUpdatedAt, lastUpdatedBy} = lastUpdateData!;
+    expect(lastUpdatedBy).not.toBeNull();
+    expect(typeof lastUpdatedBy).toBe('string');
 
-    expect(timestamp).not.toBeNull();
-    expect(typeof timestamp).toBe('number');
+    expect(lastUpdatedAt).not.toBeNull();
+    expect(typeof lastUpdatedAt).toBe('number');
   });
 
   it('existing test file with spaces in path', async () => {
@@ -41,15 +40,15 @@ describe('getFileLastUpdate', () => {
       __dirname,
       '__fixtures__/simple-site/doc with space.md',
     );
-    const lastUpdateData = await getFileLastUpdate(filePathWithSpace);
+    const lastUpdateData = await getGitLastUpdate(filePathWithSpace);
     expect(lastUpdateData).not.toBeNull();
 
-    const {author, timestamp} = lastUpdateData!;
-    expect(author).not.toBeNull();
-    expect(typeof author).toBe('string');
+    const {lastUpdatedBy, lastUpdatedAt} = lastUpdateData!;
+    expect(lastUpdatedBy).not.toBeNull();
+    expect(typeof lastUpdatedBy).toBe('string');
 
-    expect(timestamp).not.toBeNull();
-    expect(typeof timestamp).toBe('number');
+    expect(lastUpdatedAt).not.toBeNull();
+    expect(typeof lastUpdatedAt).toBe('number');
   });
 
   it('non-existing file', async () => {
@@ -62,7 +61,7 @@ describe('getFileLastUpdate', () => {
       '__fixtures__',
       nonExistingFileName,
     );
-    await expect(getFileLastUpdate(nonExistingFilePath)).rejects.toThrow(
+    await expect(getGitLastUpdate(nonExistingFilePath)).rejects.toThrow(
       /An error occurred when trying to get the last update date/,
     );
     expect(consoleMock).toHaveBeenCalledTimes(0);
@@ -74,7 +73,7 @@ describe('getFileLastUpdate', () => {
     const consoleMock = jest
       .spyOn(console, 'warn')
       .mockImplementation(() => {});
-    const lastUpdateData = await getFileLastUpdate(existingFilePath);
+    const lastUpdateData = await getGitLastUpdate(existingFilePath);
     expect(lastUpdateData).toBeNull();
     expect(consoleMock).toHaveBeenLastCalledWith(
       expect.stringMatching(
@@ -92,7 +91,7 @@ describe('getFileLastUpdate', () => {
       .mockImplementation(() => {});
     const tempFilePath = path.join(repoDir, 'file.md');
     await fs.writeFile(tempFilePath, 'Lorem ipsum :)');
-    await expect(getFileLastUpdate(tempFilePath)).resolves.toBeNull();
+    await expect(getGitLastUpdate(tempFilePath)).resolves.toBeNull();
     expect(consoleMock).toHaveBeenCalledTimes(1);
     expect(consoleMock).toHaveBeenLastCalledWith(
       expect.stringMatching(/not tracked by git./),
@@ -113,7 +112,7 @@ describe('readLastUpdateData', () => {
       {date: testDate},
     );
     expect(lastUpdatedAt).toEqual(testTimestamp);
-    expect(lastUpdatedBy).toBe(GIT_FALLBACK_LAST_UPDATE_AUTHOR);
+    expect(lastUpdatedBy).toBe(LAST_UPDATE_FALLBACK.lastUpdatedBy);
   });
 
   it('read last author show author time', async () => {
@@ -123,7 +122,7 @@ describe('readLastUpdateData', () => {
       {author: testAuthor},
     );
     expect(lastUpdatedBy).toEqual(testAuthor);
-    expect(lastUpdatedAt).toBe(GIT_FALLBACK_LAST_UPDATE_DATE);
+    expect(lastUpdatedAt).toBe(LAST_UPDATE_FALLBACK.lastUpdatedAt);
   });
 
   it('read last all show author time', async () => {
@@ -160,7 +159,7 @@ describe('readLastUpdateData', () => {
       {showLastUpdateAuthor: true, showLastUpdateTime: false},
       {date: testDate},
     );
-    expect(lastUpdatedBy).toBe(GIT_FALLBACK_LAST_UPDATE_AUTHOR);
+    expect(lastUpdatedBy).toBe(LAST_UPDATE_FALLBACK.lastUpdatedBy);
     expect(lastUpdatedAt).toBeUndefined();
   });
 
@@ -180,7 +179,7 @@ describe('readLastUpdateData', () => {
       {showLastUpdateAuthor: true, showLastUpdateTime: false},
       {},
     );
-    expect(lastUpdatedBy).toBe(GIT_FALLBACK_LAST_UPDATE_AUTHOR);
+    expect(lastUpdatedBy).toBe(LAST_UPDATE_FALLBACK.lastUpdatedBy);
     expect(lastUpdatedAt).toBeUndefined();
   });
 
@@ -201,7 +200,7 @@ describe('readLastUpdateData', () => {
       {author: testAuthor},
     );
     expect(lastUpdatedBy).toBeUndefined();
-    expect(lastUpdatedAt).toEqual(GIT_FALLBACK_LAST_UPDATE_DATE);
+    expect(lastUpdatedAt).toEqual(LAST_UPDATE_FALLBACK.lastUpdatedAt);
   });
 
   it('read last author show time only - both front matter', async () => {

packages/docusaurus-utils/src/__tests__/pathUtils.test.ts

@@ -15,6 +15,7 @@ import {
   aliasedSitePath,
   toMessageRelativeFilePath,
   addTrailingPathSeparator,
+  aliasedSitePathToRelativePath,
 } from '../pathUtils';
 
 describe('isNameTooLong', () => {
@@ -185,6 +186,20 @@ describe('aliasedSitePath', () => {
   });
 });
 
+describe('aliasedSitePathToRelativePath', () => {
+  it('works', () => {
+    expect(aliasedSitePathToRelativePath('@site/site/relative/path')).toBe(
+      'site/relative/path',
+    );
+  });
+
+  it('is fail-fast', () => {
+    expect(() => aliasedSitePathToRelativePath('/site/relative/path')).toThrow(
+      /Unexpected, filePath is not site-aliased: \/site\/relative\/path/,
+    );
+  });
+});
+
 describe('addTrailingPathSeparator', () => {
   it('works', () => {
     expect(addTrailingPathSeparator('foo')).toEqual(

packages/docusaurus-utils/src/__tests__/routeUtils.test.ts

@@ -0,0 +1,33 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import {flattenRoutes} from '../routeUtils';
+import type {RouteConfig} from '@docusaurus/types';
+
+describe('flattenRoutes', () => {
+  it('returns flattened routes without parents', () => {
+    const routes: RouteConfig[] = [
+      {
+        path: '/docs',
+        component: '',
+        routes: [
+          {path: '/docs/someDoc', component: ''},
+          {path: '/docs/someOtherDoc', component: ''},
+        ],
+      },
+      {
+        path: '/community',
+        component: '',
+      },
+    ];
+    expect(flattenRoutes(routes)).toEqual([
+      routes[0]!.routes![0],
+      routes[0]!.routes![1],
+      routes[1],
+    ]);
+  });
+});

packages/docusaurus-utils/src/index.ts

@@ -91,6 +91,7 @@ export {
   posixPath,
   toMessageRelativeFilePath,
   aliasedSitePath,
+  aliasedSitePathToRelativePath,
   escapePath,
   addTrailingPathSeparator,
 } from './pathUtils';
@@ -118,12 +119,13 @@ export {
 export {isDraft, isUnlisted} from './contentVisibilityUtils';
 export {escapeRegexp} from './regExpUtils';
 export {askPreferredLanguage} from './cliUtils';
+export {flattenRoutes} from './routeUtils';
 
 export {
-  getFileLastUpdate,
+  getGitLastUpdate,
+  getLastUpdate,
+  readLastUpdateData,
+  LAST_UPDATE_FALLBACK,
   type LastUpdateData,
   type FrontMatterLastUpdate,
-  readLastUpdateData,
-  GIT_FALLBACK_LAST_UPDATE_AUTHOR,
-  GIT_FALLBACK_LAST_UPDATE_DATE,
 } from './lastUpdateUtils';

packages/docusaurus-utils/src/lastUpdateUtils.ts

@@ -14,22 +14,6 @@ import {
 } from './gitUtils';
 import type {PluginOptions} from '@docusaurus/types';
 
-export const GIT_FALLBACK_LAST_UPDATE_DATE = 1539502055000;
-
-export const GIT_FALLBACK_LAST_UPDATE_AUTHOR = 'Author';
-
-async function getGitLastUpdate(filePath: string): Promise<LastUpdateData> {
-  if (process.env.NODE_ENV !== 'production') {
-    // Use fake data in dev/test for faster development.
-    return {
-      lastUpdatedBy: GIT_FALLBACK_LAST_UPDATE_AUTHOR,
-      lastUpdatedAt: GIT_FALLBACK_LAST_UPDATE_DATE,
-    };
-  }
-  const {author, timestamp} = (await getFileLastUpdate(filePath)) ?? {};
-  return {lastUpdatedBy: author, lastUpdatedAt: timestamp};
-}
-
 export type LastUpdateData = {
   /** A timestamp in **milliseconds**, usually read from `git log` */
   lastUpdatedAt?: number;
@@ -37,20 +21,12 @@ export type LastUpdateData = {
   lastUpdatedBy?: string;
 };
 
-export type FrontMatterLastUpdate = {
-  author?: string;
-  /** Date can be any
-   * [parsable date string](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/parse).
-   */
-  date?: Date | string;
-};
-
 let showedGitRequirementError = false;
 let showedFileNotTrackedError = false;
 
-export async function getFileLastUpdate(
+export async function getGitLastUpdate(
   filePath: string,
-): Promise<{timestamp: number; author: string} | null> {
+): Promise<LastUpdateData | null> {
   if (!filePath) {
     return null;
   }
@@ -63,7 +39,7 @@ export async function getFileLastUpdate(
       includeAuthor: true,
     });
 
-    return {timestamp: result.timestamp, author: result.author};
+    return {lastUpdatedAt: result.timestamp, lastUpdatedBy: result.author};
   } catch (err) {
     if (err instanceof GitNotFoundError) {
       if (!showedGitRequirementError) {
@@ -87,11 +63,35 @@ export async function getFileLastUpdate(
   }
 }
 
+export const LAST_UPDATE_FALLBACK: LastUpdateData = {
+  lastUpdatedAt: 1539502055000,
+  lastUpdatedBy: 'Author',
+};
+
+export async function getLastUpdate(
+  filePath: string,
+): Promise<LastUpdateData | null> {
+  if (process.env.NODE_ENV !== 'production') {
+    // Use fake data in dev/test for faster development.
+    return LAST_UPDATE_FALLBACK;
+  }
+  return getGitLastUpdate(filePath);
+}
+
 type LastUpdateOptions = Pick<
   PluginOptions,
   'showLastUpdateAuthor' | 'showLastUpdateTime'
 >;
 
+export type FrontMatterLastUpdate = {
+  author?: string;
+  /**
+   * Date can be any
+   * [parsable date string](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/parse).
+   */
+  date?: Date | string;
+};
+
 export async function readLastUpdateData(
   filePath: string,
   options: LastUpdateOptions,
@@ -111,18 +111,18 @@ export async function readLastUpdateData(
   // We try to minimize git last update calls
   // We call it at most once
   // If all the data is provided as front matter, we do not call it
-  const getGitLastUpdateMemoized = _.memoize(() => getGitLastUpdate(filePath));
-  const getGitLastUpdateBy = () =>
-    getGitLastUpdateMemoized().then((update) => update.lastUpdatedBy);
-  const getGitLastUpdateAt = () =>
-    getGitLastUpdateMemoized().then((update) => update.lastUpdatedAt);
+  const getLastUpdateMemoized = _.memoize(() => getLastUpdate(filePath));
+  const getLastUpdateBy = () =>
+    getLastUpdateMemoized().then((update) => update?.lastUpdatedBy);
+  const getLastUpdateAt = () =>
+    getLastUpdateMemoized().then((update) => update?.lastUpdatedAt);
 
   const lastUpdatedBy = showLastUpdateAuthor
-    ? frontMatterAuthor ?? (await getGitLastUpdateBy())
+    ? frontMatterAuthor ?? (await getLastUpdateBy())
     : undefined;
 
   const lastUpdatedAt = showLastUpdateTime
-    ? frontMatterTimestamp ?? (await getGitLastUpdateAt())
+    ? frontMatterTimestamp ?? (await getLastUpdateAt())
     : undefined;
 
   return {

packages/docusaurus-utils/src/pathUtils.ts

@@ -92,6 +92,20 @@ export function aliasedSitePath(filePath: string, siteDir: string): string {
   return `@site/${relativePath}`;
 }
 
+/**
+ * Converts back the aliased site path (starting with "@site/...") to a relative path
+ *
+ * TODO method this is a workaround, we shouldn't need to alias/un-alias paths
+ *  we should refactor the codebase to not have aliased site paths everywhere
+ *  We probably only need aliasing for client-only paths required by Webpack
+ */
+export function aliasedSitePathToRelativePath(filePath: string): string {
+  if (filePath.startsWith('@site/')) {
+    return filePath.replace('@site/', '');
+  }
+  throw new Error(`Unexpected, filePath is not site-aliased: ${filePath}`);
+}
+
 /**
  * When you have a path like C:\X\Y
  * It is not safe to use directly when generating code

packages/docusaurus-utils/src/routeUtils.ts

@@ -0,0 +1,19 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import type {RouteConfig} from '@docusaurus/types';
+
+/**
+ * Recursively flatten routes and only return the "leaf routes"
+ * Parent routes are filtered out
+ */
+export function flattenRoutes(routeConfig: RouteConfig[]): RouteConfig[] {
+  function flatten(route: RouteConfig): RouteConfig[] {
+    return route.routes ? route.routes.flatMap(flatten) : [route];
+  }
+  return routeConfig.flatMap(flatten);
+}