diff --git a/packages/docusaurus-utils/package.json b/packages/docusaurus-utils/package.json index 7dde60ae3a..a8a74fa2ad 100644 --- a/packages/docusaurus-utils/package.json +++ b/packages/docusaurus-utils/package.json @@ -33,6 +33,7 @@ "lodash": "^4.17.21", "micromatch": "^4.0.5", "prompts": "^2.4.2", + "p-queue": "^6.6.2", "resolve-pathname": "^3.0.0", "tslib": "^2.6.0", "url-loader": "^4.1.1", diff --git a/packages/docusaurus-utils/src/gitUtils.ts b/packages/docusaurus-utils/src/gitUtils.ts index 104bf73634..fff2659922 100644 --- a/packages/docusaurus-utils/src/gitUtils.ts +++ b/packages/docusaurus-utils/src/gitUtils.ts @@ -7,8 +7,33 @@ import path from 'path'; import fs from 'fs-extra'; +import os from 'os'; import _ from 'lodash'; import execa from 'execa'; +import PQueue from 'p-queue'; + +// Quite high/conservative concurrency value (it was previously "Infinity") +// See https://github.com/facebook/docusaurus/pull/10915 +const DefaultGitCommandConcurrency = + // TODO Docusaurus v4: bump node, availableParallelism() now always exists + (typeof os.availableParallelism === 'function' + ? os.availableParallelism() + : os.cpus().length) * 4; + +const GitCommandConcurrencyEnv = process.env.DOCUSAURUS_GIT_COMMAND_CONCURRENCY + ? parseInt(process.env.DOCUSAURUS_GIT_COMMAND_CONCURRENCY, 10) + : undefined; + +const GitCommandConcurrency = + GitCommandConcurrencyEnv && GitCommandConcurrencyEnv > 0 + ? GitCommandConcurrencyEnv + : DefaultGitCommandConcurrency; + +// We use a queue to avoid running too many concurrent Git commands at once +// See https://github.com/facebook/docusaurus/issues/10348 +const GitCommandQueue = new PQueue({ + concurrency: GitCommandConcurrency, +}); const realHasGitFn = () => { try { @@ -129,10 +154,13 @@ export async function getFileCommitDate( file, )}"`; - const result = await execa(command, { - cwd: path.dirname(file), - shell: true, - }); + const result = (await GitCommandQueue.add(() => + execa(command, { + cwd: path.dirname(file), + shell: true, + }), + ))!; + if (result.exitCode !== 0) { throw new Error( `Failed to retrieve the git history for file "${file}" with exit code ${result.exitCode}: ${result.stderr}`, diff --git a/yarn.lock b/yarn.lock index 78765fc47a..d61d8a7ae5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13909,7 +13909,7 @@ p-pipe@3.1.0: resolved "https://registry.yarnpkg.com/p-pipe/-/p-pipe-3.1.0.tgz#48b57c922aa2e1af6a6404cb7c6bf0eb9cc8e60e" integrity sha512-08pj8ATpzMR0Y80x50yJHn37NF6vjrqHutASaX5LiH5npS9XPvrUmscd9MF5R4fuYRHOxQR1FfMIlF7AzwoPqw== -p-queue@6.6.2: +p-queue@6.6.2, p-queue@^6.6.2: version "6.6.2" resolved "https://registry.yarnpkg.com/p-queue/-/p-queue-6.6.2.tgz#2068a9dcf8e67dd0ec3e7a2bcb76810faa85e426" integrity sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==