From bdb129252c1d2bcde9071e74f5d975bbfbeff19e Mon Sep 17 00:00:00 2001 From: Endi Date: Mon, 14 Oct 2019 19:37:50 +0700 Subject: [PATCH] perf(v2): fork clean-webpack-plugin to reduce memory (#1839) * chore: fork clean-webpack-plugin * deps --- CHANGELOG-2.x.md | 3 + packages/docusaurus/package.json | 2 +- packages/docusaurus/src/commands/build.ts | 2 +- .../src/webpack/plugins/CleanWebpackPlugin.ts | 330 ++++++++++++++++++ yarn.lock | 9 +- 5 files changed, 336 insertions(+), 10 deletions(-) create mode 100644 packages/docusaurus/src/webpack/plugins/CleanWebpackPlugin.ts diff --git a/CHANGELOG-2.x.md b/CHANGELOG-2.x.md index 97e16901a4..b85b2de436 100644 --- a/CHANGELOG-2.x.md +++ b/CHANGELOG-2.x.md @@ -1,5 +1,8 @@ # Docusaurus 2 Changelog +## 2.0.0-alpha.28 +- Further reduce memory usage to avoid heap memory allocation failure. + ## 2.0.0-alpha.27 - Add `@theme/Tabs` which can be used to implement multi-language code tabs. diff --git a/packages/docusaurus/package.json b/packages/docusaurus/package.json index cb271a8761..1845944ee8 100644 --- a/packages/docusaurus/package.json +++ b/packages/docusaurus/package.json @@ -43,10 +43,10 @@ "chalk": "^2.4.2", "chokidar": "^3.0.2", "classnames": "^2.2.6", - "clean-webpack-plugin": "^2.0.1", "commander": "^2.20.0", "copy-webpack-plugin": "^5.0.4", "css-loader": "^3.1.0", + "del": "^4.1.1", "ejs": "^2.6.2", "express": "^4.17.1", "fs-extra": "^8.1.0", diff --git a/packages/docusaurus/src/commands/build.ts b/packages/docusaurus/src/commands/build.ts index de192fe2d3..e68b0dfbe9 100644 --- a/packages/docusaurus/src/commands/build.ts +++ b/packages/docusaurus/src/commands/build.ts @@ -6,7 +6,6 @@ */ import chalk from 'chalk'; -import CleanWebpackPlugin from 'clean-webpack-plugin'; import CopyWebpackPlugin from 'copy-webpack-plugin'; import fs from 'fs-extra'; import path from 'path'; @@ -20,6 +19,7 @@ import {CLIOptions, Props} from '@docusaurus/types'; import {createClientConfig} from '../webpack/client'; import {createServerConfig} from '../webpack/server'; import {applyConfigureWebpack} from '../webpack/utils'; +import CleanWebpackPlugin from '../webpack/plugins/CleanWebpackPlugin'; function compile(config: Configuration[]): Promise { return new Promise((resolve, reject) => { diff --git a/packages/docusaurus/src/webpack/plugins/CleanWebpackPlugin.ts b/packages/docusaurus/src/webpack/plugins/CleanWebpackPlugin.ts new file mode 100644 index 0000000000..9bc6d68267 --- /dev/null +++ b/packages/docusaurus/src/webpack/plugins/CleanWebpackPlugin.ts @@ -0,0 +1,330 @@ +/** + * Copyright (c) 2017-present, Facebook, Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import {Compiler, Stats} from 'webpack'; +import path from 'path'; +import {sync as delSync} from 'del'; + +export interface Options { + /** + * Simulate the removal of files + * + * default: false + */ + dry?: boolean; + + /** + * Write Logs to Console + * (Always enabled when dry is true) + * + * default: false + */ + verbose?: boolean; + + /** + * Automatically remove all unused webpack assets on rebuild + * + * default: true + */ + cleanStaleWebpackAssets?: boolean; + + /** + * Do not allow removal of current webpack assets + * + * default: true + */ + protectWebpackAssets?: boolean; + + /** + * Removes files once prior to Webpack compilation + * Not included in rebuilds (watch mode) + * + * Use !negative patterns to exclude files + * + * default: ['**\/*'] + */ + cleanOnceBeforeBuildPatterns?: string[]; + + /** + * Removes files after every build (including watch mode) that match this pattern. + * Used for files that are not created directly by Webpack. + * + * Use !negative patterns to exclude files + * + * default: disabled + */ + cleanAfterEveryBuildPatterns?: string[]; + + /** + * Allow clean patterns outside of process.cwd() + * + * requires dry option to be explicitly set + * + * default: false + */ + dangerouslyAllowCleanPatternsOutsideProject?: boolean; +} + +class CleanWebpackPlugin { + private readonly dry: boolean; + private readonly verbose: boolean; + private readonly cleanStaleWebpackAssets: boolean; + private readonly protectWebpackAssets: boolean; + private readonly cleanAfterEveryBuildPatterns: string[]; + private readonly cleanOnceBeforeBuildPatterns: string[]; + private readonly dangerouslyAllowCleanPatternsOutsideProject: boolean; + private currentAssets: string[]; + private initialClean: boolean; + private outputPath: string; + + constructor(options: Options = {}) { + if (typeof options !== 'object' || Array.isArray(options) === true) { + throw new Error(`clean-webpack-plugin only accepts an options object. See: + https://github.com/johnagan/clean-webpack-plugin#options-and-defaults-optional`); + } + + // @ts-ignore + if (options.allowExternal) { + throw new Error( + 'clean-webpack-plugin: `allowExternal` option no longer supported. Use `dangerouslyAllowCleanPatternsOutsideProject`', + ); + } + + if ( + options.dangerouslyAllowCleanPatternsOutsideProject === true && + options.dry !== true && + options.dry !== false + ) { + // eslint-disable-next-line no-console + console.warn( + 'clean-webpack-plugin: dangerouslyAllowCleanPatternsOutsideProject requires dry: false to be explicitly set. Enabling dry mode', + ); + } + + this.dangerouslyAllowCleanPatternsOutsideProject = + options.dangerouslyAllowCleanPatternsOutsideProject === true || false; + + this.dry = + options.dry === true || options.dry === false + ? options.dry + : this.dangerouslyAllowCleanPatternsOutsideProject === true || false; + + this.verbose = this.dry === true || options.verbose === true || false; + + this.cleanStaleWebpackAssets = + options.cleanStaleWebpackAssets === true || + options.cleanStaleWebpackAssets === false + ? options.cleanStaleWebpackAssets + : true; + + this.protectWebpackAssets = + options.protectWebpackAssets === true || + options.protectWebpackAssets === false + ? options.protectWebpackAssets + : true; + + this.cleanAfterEveryBuildPatterns = Array.isArray( + options.cleanAfterEveryBuildPatterns, + ) + ? options.cleanAfterEveryBuildPatterns + : []; + + this.cleanOnceBeforeBuildPatterns = Array.isArray( + options.cleanOnceBeforeBuildPatterns, + ) + ? options.cleanOnceBeforeBuildPatterns + : ['**/*']; + + /** + * Store webpack build assets + */ + this.currentAssets = []; + + /** + * Only used with cleanOnceBeforeBuildPatterns + */ + this.initialClean = false; + + this.outputPath = ''; + + this.apply = this.apply.bind(this); + this.handleInitial = this.handleInitial.bind(this); + this.handleDone = this.handleDone.bind(this); + this.removeFiles = this.removeFiles.bind(this); + } + + apply(compiler: Compiler) { + if (!compiler.options.output || !compiler.options.output.path) { + // eslint-disable-next-line no-console + console.warn( + 'clean-webpack-plugin: options.output.path not defined. Plugin disabled...', + ); + + return; + } + + this.outputPath = compiler.options.output.path; + + /** + * webpack 4+ comes with a new plugin system. + * + * Check for hooks in-order to support old plugin system + */ + const hooks = compiler.hooks; + + if (this.cleanOnceBeforeBuildPatterns.length !== 0) { + if (hooks) { + hooks.compile.tap('clean-webpack-plugin', () => { + this.handleInitial(); + }); + } else { + compiler.plugin('compile', () => { + this.handleInitial(); + }); + } + } + + if (hooks) { + hooks.done.tap('clean-webpack-plugin', stats => { + this.handleDone(stats); + }); + } else { + compiler.plugin('done', stats => { + this.handleDone(stats); + }); + } + } + + /** + * Initially remove files from output directory prior to build. + * + * Only happens once. + * + * Warning: It is recommended to initially clean your build directory outside of webpack to minimize unexpected behavior. + */ + handleInitial() { + if (this.initialClean) { + return; + } + + this.initialClean = true; + + this.removeFiles(this.cleanOnceBeforeBuildPatterns); + } + + handleDone(stats: Stats) { + /** + * Do nothing if there is a webpack error + */ + if (stats.hasErrors()) { + if (this.verbose) { + // eslint-disable-next-line no-console + console.warn('clean-webpack-plugin: pausing due to webpack errors'); + } + + return; + } + + /** + * Fetch Webpack's output asset files + */ + const statsAssets = + stats.toJson( + { + all: false, + assets: true, + }, + true, + ).assets || []; + const assets = statsAssets.map((asset: {name: string}) => { + return asset.name; + }); + + /** + * Get all files that were in the previous build but not the current + * + * (relies on del's cwd: outputPath option) + */ + const staleFiles = this.currentAssets.filter(previousAsset => { + const assetCurrent = assets.includes(previousAsset) === false; + + return assetCurrent; + }); + + /** + * Save assets for next compilation + */ + this.currentAssets = assets.sort(); + + const removePatterns: string[] = []; + + /** + * Remove unused webpack assets + */ + if (this.cleanStaleWebpackAssets === true && staleFiles.length !== 0) { + removePatterns.push(...staleFiles); + } + + /** + * Remove cleanAfterEveryBuildPatterns + */ + if (this.cleanAfterEveryBuildPatterns.length !== 0) { + removePatterns.push(...this.cleanAfterEveryBuildPatterns); + } + + if (removePatterns.length !== 0) { + this.removeFiles(removePatterns); + } + } + + removeFiles(patterns: string[]) { + try { + const deleted = delSync(patterns, { + force: this.dangerouslyAllowCleanPatternsOutsideProject, + // Change context to build directory + cwd: this.outputPath, + dryRun: this.dry, + dot: true, + ignore: this.protectWebpackAssets ? this.currentAssets : [], + }); + + /** + * Log if verbose is enabled + */ + if (this.verbose) { + deleted.forEach(file => { + const filename = path.relative(process.cwd(), file); + + const message = this.dry ? 'dry' : 'removed'; + + /** + * Use console.warn over .log + * https://github.com/webpack/webpack/issues/1904 + * https://github.com/johnagan/clean-webpack-plugin/issues/11 + */ + // eslint-disable-next-line no-console + console.warn(`clean-webpack-plugin: ${message} ${filename}`); + }); + } + } catch (error) { + const needsForce = /Cannot delete files\/folders outside the current working directory\./.test( + error.message, + ); + + if (needsForce) { + const message = + 'clean-webpack-plugin: Cannot delete files/folders outside the current working directory. Can be overridden with the `dangerouslyAllowCleanPatternsOutsideProject` option.'; + + throw new Error(message); + } + + throw error; + } + } +} + +export default CleanWebpackPlugin; diff --git a/yarn.lock b/yarn.lock index e09ff5e39b..720c18cdfa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4486,13 +4486,6 @@ clean-stack@^2.0.0: resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b" integrity sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A== -clean-webpack-plugin@^2.0.1: - version "2.0.2" - resolved "https://registry.yarnpkg.com/clean-webpack-plugin/-/clean-webpack-plugin-2.0.2.tgz#805a19ff20d46a06125298a25eb31142ecad2166" - integrity sha512-pi1111o4OBd9qvacbgs+NRqClfVPKVIc66B4d8kx6Ho/L+i9entQ/NpK600CsTYTPu3kWvKwwyKarsYMvC2xeA== - dependencies: - del "^4.0.0" - cli-cursor@^2.0.0, cli-cursor@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-2.1.0.tgz#b35dac376479facc3e94747d41d0d0f5238ffcb5" @@ -5543,7 +5536,7 @@ define-property@^2.0.2: is-descriptor "^1.0.2" isobject "^3.0.1" -del@^4.0.0, del@^4.1.1: +del@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/del/-/del-4.1.1.tgz#9e8f117222ea44a31ff3a156c049b99052a9f0b4" integrity sha512-QwGuEUouP2kVwQenAsOof5Fv8K9t3D8Ca8NxcXKrIpEHjTXK5J2nXLdP+ALI1cgv8wj7KuwBhTwBkOZSJKM5XQ==