perf(utils): implement git command queue (#11163)

Co-authored-by: slorber <749374+slorber@users.noreply.github.com>

packages/docusaurus-utils/package.json

@@ -33,6 +33,7 @@
     "lodash": "^4.17.21",
     "micromatch": "^4.0.5",
     "prompts": "^2.4.2",
+    "p-queue": "^6.6.2",
     "resolve-pathname": "^3.0.0",
     "tslib": "^2.6.0",
     "url-loader": "^4.1.1",

packages/docusaurus-utils/src/gitUtils.ts

@@ -7,8 +7,33 @@
 
 import path from 'path';
 import fs from 'fs-extra';
+import os from 'os';
 import _ from 'lodash';
 import execa from 'execa';
+import PQueue from 'p-queue';
+
+// Quite high/conservative concurrency value (it was previously "Infinity")
+// See https://github.com/facebook/docusaurus/pull/10915
+const DefaultGitCommandConcurrency =
+  // TODO Docusaurus v4: bump node, availableParallelism() now always exists
+  (typeof os.availableParallelism === 'function'
+    ? os.availableParallelism()
+    : os.cpus().length) * 4;
+
+const GitCommandConcurrencyEnv = process.env.DOCUSAURUS_GIT_COMMAND_CONCURRENCY
+  ? parseInt(process.env.DOCUSAURUS_GIT_COMMAND_CONCURRENCY, 10)
+  : undefined;
+
+const GitCommandConcurrency =
+  GitCommandConcurrencyEnv && GitCommandConcurrencyEnv > 0
+    ? GitCommandConcurrencyEnv
+    : DefaultGitCommandConcurrency;
+
+// We use a queue to avoid running too many concurrent Git commands at once
+// See https://github.com/facebook/docusaurus/issues/10348
+const GitCommandQueue = new PQueue({
+  concurrency: GitCommandConcurrency,
+});
 
 const realHasGitFn = () => {
   try {
@@ -129,10 +154,13 @@ export async function getFileCommitDate(
     file,
   )}"`;
 
-  const result = await execa(command, {
-    cwd: path.dirname(file),
-    shell: true,
-  });
+  const result = (await GitCommandQueue.add(() =>
+    execa(command, {
+      cwd: path.dirname(file),
+      shell: true,
+    }),
+  ))!;
+
   if (result.exitCode !== 0) {
     throw new Error(
       `Failed to retrieve the git history for file "${file}" with exit code ${result.exitCode}: ${result.stderr}`,

yarn.lock

@@ -13909,7 +13909,7 @@ p-pipe@3.1.0:
   resolved "https://registry.yarnpkg.com/p-pipe/-/p-pipe-3.1.0.tgz#48b57c922aa2e1af6a6404cb7c6bf0eb9cc8e60e"
   integrity sha512-08pj8ATpzMR0Y80x50yJHn37NF6vjrqHutASaX5LiH5npS9XPvrUmscd9MF5R4fuYRHOxQR1FfMIlF7AzwoPqw==
 
-p-queue@6.6.2:
+p-queue@6.6.2, p-queue@^6.6.2:
   version "6.6.2"
   resolved "https://registry.yarnpkg.com/p-queue/-/p-queue-6.6.2.tgz#2068a9dcf8e67dd0ec3e7a2bcb76810faa85e426"
   integrity sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==