From 34411e12e56fc768495b94db2f85373eefb4835e Mon Sep 17 00:00:00 2001 From: Lucas Correia Date: Tue, 15 Jun 2021 13:39:06 -0300 Subject: [PATCH] fix(v2): truncate docuhash return value in order to avoid ERRNAMETOOLONG error (#4899) * fix: truncate docuhash return value in order to avoid ERRNAMETOOLONG error * chore: add deep file path test page to website * refactor: reorganize docuHash/pathUtils code and tests * chore: git support longpaths on v2 windows tests workflow Co-authored-by: slorber --- .github/workflows/v2-tests-windows.yml | 2 + .../src/__tests__/docuHash.test.ts | 30 +++++++ .../src/__tests__/index.test.ts | 33 -------- .../src/__tests__/pathUtils.test.ts | 82 +++++++++++++++++++ packages/docusaurus-utils/src/docuHash.ts | 28 +++++++ packages/docusaurus-utils/src/index.ts | 22 ++--- packages/docusaurus-utils/src/pathUtils.ts | 48 +++++++++++ .../bar/foo/bar/foo/bar/foo/bar/test-file.md | 7 ++ 8 files changed, 202 insertions(+), 50 deletions(-) create mode 100644 packages/docusaurus-utils/src/__tests__/docuHash.test.ts create mode 100644 packages/docusaurus-utils/src/__tests__/pathUtils.test.ts create mode 100644 packages/docusaurus-utils/src/docuHash.ts create mode 100644 packages/docusaurus-utils/src/pathUtils.ts create mode 100644 website/src/pages/deep-file-path-test/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/test-file.md diff --git a/.github/workflows/v2-tests-windows.yml b/.github/workflows/v2-tests-windows.yml index c156779fd2..1754b5e8ce 100644 --- a/.github/workflows/v2-tests-windows.yml +++ b/.github/workflows/v2-tests-windows.yml @@ -17,6 +17,8 @@ jobs: matrix: node: ['12', '14'] steps: + - name: Support longpaths + run: git config --system core.longpaths true - uses: actions/checkout@v2 - name: Use Node.js ${{ matrix.node }} uses: actions/setup-node@v2 diff --git a/packages/docusaurus-utils/src/__tests__/docuHash.test.ts b/packages/docusaurus-utils/src/__tests__/docuHash.test.ts new file mode 100644 index 0000000000..81c76507a0 --- /dev/null +++ b/packages/docusaurus-utils/src/__tests__/docuHash.test.ts @@ -0,0 +1,30 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import {docuHash} from '../docuHash'; + +describe('docuHash', () => { + test('docuHash works', () => { + const asserts: Record = { + '': '-d41', + '/': 'index', + '/foo-bar': 'foo-bar-096', + '/foo/bar': 'foo-bar-1df', + '/endi/lie': 'endi-lie-9fa', + '/endi-lie': 'endi-lie-fd3', + '/yangshun/tay': 'yangshun-tay-48d', + '/yangshun-tay': 'yangshun-tay-f3b', + '/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar': + 'foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo--d46', + '/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/test1-test2': + 'foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-test-1-test--787', + }; + Object.keys(asserts).forEach((file) => { + expect(docuHash(file)).toBe(asserts[file]); + }); + }); +}); diff --git a/packages/docusaurus-utils/src/__tests__/index.test.ts b/packages/docusaurus-utils/src/__tests__/index.test.ts index 6a06a70510..de065102ae 100644 --- a/packages/docusaurus-utils/src/__tests__/index.test.ts +++ b/packages/docusaurus-utils/src/__tests__/index.test.ts @@ -8,8 +8,6 @@ import path from 'path'; import { fileToPath, - simpleHash, - docuHash, genComponentName, genChunkName, idx, @@ -68,37 +66,6 @@ describe('load utils', () => { }); }); - test('simpleHash', () => { - const asserts: Record = { - '': 'd41', - '/foo-bar': '096', - '/foo/bar': '1df', - '/endi/lie': '9fa', - '/endi-lie': 'fd3', - '/yangshun/tay': '48d', - '/yangshun-tay': 'f3b', - }; - Object.keys(asserts).forEach((file) => { - expect(simpleHash(file, 3)).toBe(asserts[file]); - }); - }); - - test('docuHash', () => { - const asserts: Record = { - '': '-d41', - '/': 'index', - '/foo-bar': 'foo-bar-096', - '/foo/bar': 'foo-bar-1df', - '/endi/lie': 'endi-lie-9fa', - '/endi-lie': 'endi-lie-fd3', - '/yangshun/tay': 'yangshun-tay-48d', - '/yangshun-tay': 'yangshun-tay-f3b', - }; - Object.keys(asserts).forEach((file) => { - expect(docuHash(file)).toBe(asserts[file]); - }); - }); - test('fileToPath', () => { const asserts: Record = { 'index.md': '/', diff --git a/packages/docusaurus-utils/src/__tests__/pathUtils.test.ts b/packages/docusaurus-utils/src/__tests__/pathUtils.test.ts new file mode 100644 index 0000000000..007beff12a --- /dev/null +++ b/packages/docusaurus-utils/src/__tests__/pathUtils.test.ts @@ -0,0 +1,82 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import {simpleHash, isNameTooLong, shortName} from '../pathUtils'; + +describe('pathUtils', () => { + test('simpleHash', () => { + const asserts: Record = { + '': 'd41', + '/foo-bar': '096', + '/foo/bar': '1df', + '/endi/lie': '9fa', + '/endi-lie': 'fd3', + '/yangshun/tay': '48d', + '/yangshun-tay': 'f3b', + '/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar': + 'd46', + '/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/test1-test2': + '787', + }; + Object.keys(asserts).forEach((file) => { + expect(simpleHash(file, 3)).toBe(asserts[file]); + }); + }); + + test('isNameTooLong', () => { + const asserts: Record = { + '': false, + 'foo-bar-096': false, + 'foo-bar-1df': false, + 'endi-lie-9fa': false, + 'endi-lie-fd3': false, + 'yangshun-tay-48d': false, + 'yangshun-tay-f3b': false, + 'foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-d46': true, + 'foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-test-1-test-2-787': true, + }; + Object.keys(asserts).forEach((path) => { + expect(isNameTooLong(path)).toBe(asserts[path]); + }); + }); + + describe('shortName', () => { + test('works', () => { + const asserts: Record = { + '': '', + 'foo-bar': 'foo-bar', + 'endi-lie': 'endi-lie', + 'yangshun-tay': 'yangshun-tay', + 'foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar': + 'foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-', + 'foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-test-1-test-2': + 'foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-test-1-test-', + }; + Object.keys(asserts).forEach((file) => { + expect(shortName(file)).toBe(asserts[file]); + }); + }); + + // Based on https://github.com/gatsbyjs/gatsby/pull/21518/files + + const SHORT_PATH = `/short/path/without/trailing/slash`; + const VERY_LONG_PATH = `/${`x`.repeat(256)}/`; + const VERY_LONG_PATH_NON_LATIN = `/${`あ`.repeat(255)}/`; + + it(`Truncates long paths correctly`, () => { + const truncatedPathLatin = shortName(VERY_LONG_PATH); + const truncatedPathNonLatin = shortName(VERY_LONG_PATH_NON_LATIN); + expect(truncatedPathLatin.length).toBeLessThanOrEqual(255); + expect(truncatedPathNonLatin.length).toBeLessThanOrEqual(255); + }); + + it(`Does not truncate short paths`, () => { + const truncatedPath = shortName(SHORT_PATH); + expect(truncatedPath).toEqual(SHORT_PATH); + }); + }); +}); diff --git a/packages/docusaurus-utils/src/docuHash.ts b/packages/docusaurus-utils/src/docuHash.ts new file mode 100644 index 0000000000..63e9f07d3f --- /dev/null +++ b/packages/docusaurus-utils/src/docuHash.ts @@ -0,0 +1,28 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import {kebabCase} from 'lodash'; + +import {shortName, isNameTooLong, simpleHash} from './pathUtils'; + +/** + * Given an input string, convert to kebab-case and append a hash. + * Avoid str collision. + * Also removes part of the string if its larger than the allowed + * filename per OS. Avoids ERRNAMETOOLONG error. + */ +export function docuHash(str: string): string { + if (str === '/') { + return 'index'; + } + const shortHash = simpleHash(str, 3); + const parsedPath = `${kebabCase(str)}-${shortHash}`; + if (isNameTooLong(parsedPath)) { + return `${shortName(kebabCase(str))}-${shortHash}`; + } + return parsedPath; +} diff --git a/packages/docusaurus-utils/src/index.ts b/packages/docusaurus-utils/src/index.ts index a826826168..10954d14fd 100644 --- a/packages/docusaurus-utils/src/index.ts +++ b/packages/docusaurus-utils/src/index.ts @@ -8,7 +8,7 @@ import chalk from 'chalk'; import path from 'path'; import {createHash} from 'crypto'; -import {camelCase, kebabCase, mapValues} from 'lodash'; +import {camelCase, mapValues} from 'lodash'; import escapeStringRegexp from 'escape-string-regexp'; import fs from 'fs-extra'; import {URL} from 'url'; @@ -22,6 +22,8 @@ import { import resolvePathnameUnsafe from 'resolve-pathname'; import {posixPath as posixPathImport} from './posixPath'; +import {simpleHash} from './pathUtils'; +import {docuHash} from './docuHash'; export const posixPath = posixPathImport; @@ -29,6 +31,8 @@ export * from './codeTranslationsUtils'; export * from './markdownParser'; export * from './markdownLinks'; export * from './escapePath'; +export * from './docuHash'; +export {simpleHash} from './pathUtils'; const fileHash = new Map(); export async function generate( @@ -98,22 +102,6 @@ export function encodePath(userpath: string): string { .join('/'); } -export function simpleHash(str: string, length: number): string { - return createHash('md5').update(str).digest('hex').substr(0, length); -} - -/** - * Given an input string, convert to kebab-case and append a hash. - * Avoid str collision. - */ -export function docuHash(str: string): string { - if (str === '/') { - return 'index'; - } - const shortHash = simpleHash(str, 3); - return `${kebabCase(str)}-${shortHash}`; -} - /** * Convert first string character to the upper case. * E.g: docusaurus -> Docusaurus diff --git a/packages/docusaurus-utils/src/pathUtils.ts b/packages/docusaurus-utils/src/pathUtils.ts new file mode 100644 index 0000000000..91c82a1d1d --- /dev/null +++ b/packages/docusaurus-utils/src/pathUtils.ts @@ -0,0 +1,48 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +// Based on https://github.com/gatsbyjs/gatsby/pull/21518/files + +import {createHash} from 'crypto'; + +// MacOS (APFS) and Windows (NTFS) filename length limit = 255 chars, Others = 255 bytes +const MAX_PATH_SEGMENT_CHARS = 255; +const MAX_PATH_SEGMENT_BYTES = 255; +// Space for appending things to the string like file extensions and so on +const SPACE_FOR_APPENDING = 10; + +const isMacOs = process.platform === `darwin`; +const isWindows = process.platform === `win32`; + +export const isNameTooLong = (str: string): boolean => { + return isMacOs || isWindows + ? str.length + SPACE_FOR_APPENDING > MAX_PATH_SEGMENT_CHARS // MacOS (APFS) and Windows (NTFS) filename length limit (255 chars) + : Buffer.from(str).length + SPACE_FOR_APPENDING > MAX_PATH_SEGMENT_BYTES; // Other (255 bytes) +}; + +export const shortName = (str: string): string => { + if (isMacOs || isWindows) { + const overflowingChars = str.length - MAX_PATH_SEGMENT_CHARS; + return str.slice( + 0, + str.length - overflowingChars - SPACE_FOR_APPENDING - 1, + ); + } + const strBuffer = Buffer.from(str); + const overflowingBytes = + Buffer.byteLength(strBuffer) - MAX_PATH_SEGMENT_BYTES; + return strBuffer + .slice( + 0, + Buffer.byteLength(strBuffer) - overflowingBytes - SPACE_FOR_APPENDING - 1, + ) + .toString(); +}; + +export function simpleHash(str: string, length: number): string { + return createHash('md5').update(str).digest('hex').substr(0, length); +} diff --git a/website/src/pages/deep-file-path-test/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/test-file.md b/website/src/pages/deep-file-path-test/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/test-file.md new file mode 100644 index 0000000000..9756c5b668 --- /dev/null +++ b/website/src/pages/deep-file-path-test/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/test-file.md @@ -0,0 +1,7 @@ +--- +title: Markdown page example +--- + +# Markdown page example + +You don't need React to write simple standalone pages.