mirror of
https://github.com/facebook/docusaurus.git
synced 2025-05-12 00:27:21 +02:00
fix(v2): truncate docuhash return value in order to avoid ERRNAMETOOLONG error (#4899)
* fix: truncate docuhash return value in order to avoid ERRNAMETOOLONG error * chore: add deep file path test page to website * refactor: reorganize docuHash/pathUtils code and tests * chore: git support longpaths on v2 windows tests workflow Co-authored-by: slorber <lorber.sebastien@gmail.com>
This commit is contained in:
Lucas Correia
GitHub
parent
3d95a3e6b1
commit
34411e12e5
8 changed files with 202 additions and 50 deletions
30
packages/docusaurus-utils/src/__tests__/docuHash.test.ts
Normal file
30
packages/docusaurus-utils/src/__tests__/docuHash.test.ts
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
/**
|
||||
* Copyright (c) Facebook, Inc. and its affiliates.
|
||||
*
|
||||
* This source code is licensed under the MIT license found in the
|
||||
* LICENSE file in the root directory of this source tree.
|
||||
*/
|
||||
|
||||
import {docuHash} from '../docuHash';
|
||||
|
||||
describe('docuHash', () => {
|
||||
test('docuHash works', () => {
|
||||
const asserts: Record<string, string> = {
|
||||
'': '-d41',
|
||||
'/': 'index',
|
||||
'/foo-bar': 'foo-bar-096',
|
||||
'/foo/bar': 'foo-bar-1df',
|
||||
'/endi/lie': 'endi-lie-9fa',
|
||||
'/endi-lie': 'endi-lie-fd3',
|
||||
'/yangshun/tay': 'yangshun-tay-48d',
|
||||
'/yangshun-tay': 'yangshun-tay-f3b',
|
||||
'/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar':
|
||||
'foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo--d46',
|
||||
'/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/test1-test2':
|
||||
'foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-test-1-test--787',
|
||||
};
|
||||
Object.keys(asserts).forEach((file) => {
|
||||
expect(docuHash(file)).toBe(asserts[file]);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -8,8 +8,6 @@
|
|||
import path from 'path';
|
||||
import {
|
||||
fileToPath,
|
||||
simpleHash,
|
||||
docuHash,
|
||||
genComponentName,
|
||||
genChunkName,
|
||||
idx,
|
||||
|
|
@ -68,37 +66,6 @@ describe('load utils', () => {
|
|||
});
|
||||
});
|
||||
|
||||
test('simpleHash', () => {
|
||||
const asserts: Record<string, string> = {
|
||||
'': 'd41',
|
||||
'/foo-bar': '096',
|
||||
'/foo/bar': '1df',
|
||||
'/endi/lie': '9fa',
|
||||
'/endi-lie': 'fd3',
|
||||
'/yangshun/tay': '48d',
|
||||
'/yangshun-tay': 'f3b',
|
||||
};
|
||||
Object.keys(asserts).forEach((file) => {
|
||||
expect(simpleHash(file, 3)).toBe(asserts[file]);
|
||||
});
|
||||
});
|
||||
|
||||
test('docuHash', () => {
|
||||
const asserts: Record<string, string> = {
|
||||
'': '-d41',
|
||||
'/': 'index',
|
||||
'/foo-bar': 'foo-bar-096',
|
||||
'/foo/bar': 'foo-bar-1df',
|
||||
'/endi/lie': 'endi-lie-9fa',
|
||||
'/endi-lie': 'endi-lie-fd3',
|
||||
'/yangshun/tay': 'yangshun-tay-48d',
|
||||
'/yangshun-tay': 'yangshun-tay-f3b',
|
||||
};
|
||||
Object.keys(asserts).forEach((file) => {
|
||||
expect(docuHash(file)).toBe(asserts[file]);
|
||||
});
|
||||
});
|
||||
|
||||
test('fileToPath', () => {
|
||||
const asserts: Record<string, string> = {
|
||||
'index.md': '/',
|
||||
|
|
|
|||
82
packages/docusaurus-utils/src/__tests__/pathUtils.test.ts
Normal file
82
packages/docusaurus-utils/src/__tests__/pathUtils.test.ts
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
/**
|
||||
* Copyright (c) Facebook, Inc. and its affiliates.
|
||||
*
|
||||
* This source code is licensed under the MIT license found in the
|
||||
* LICENSE file in the root directory of this source tree.
|
||||
*/
|
||||
|
||||
import {simpleHash, isNameTooLong, shortName} from '../pathUtils';
|
||||
|
||||
describe('pathUtils', () => {
|
||||
test('simpleHash', () => {
|
||||
const asserts: Record<string, string> = {
|
||||
'': 'd41',
|
||||
'/foo-bar': '096',
|
||||
'/foo/bar': '1df',
|
||||
'/endi/lie': '9fa',
|
||||
'/endi-lie': 'fd3',
|
||||
'/yangshun/tay': '48d',
|
||||
'/yangshun-tay': 'f3b',
|
||||
'/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar':
|
||||
'd46',
|
||||
'/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/test1-test2':
|
||||
'787',
|
||||
};
|
||||
Object.keys(asserts).forEach((file) => {
|
||||
expect(simpleHash(file, 3)).toBe(asserts[file]);
|
||||
});
|
||||
});
|
||||
|
||||
test('isNameTooLong', () => {
|
||||
const asserts: Record<string, boolean> = {
|
||||
'': false,
|
||||
'foo-bar-096': false,
|
||||
'foo-bar-1df': false,
|
||||
'endi-lie-9fa': false,
|
||||
'endi-lie-fd3': false,
|
||||
'yangshun-tay-48d': false,
|
||||
'yangshun-tay-f3b': false,
|
||||
'foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-d46': true,
|
||||
'foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-test-1-test-2-787': true,
|
||||
};
|
||||
Object.keys(asserts).forEach((path) => {
|
||||
expect(isNameTooLong(path)).toBe(asserts[path]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('shortName', () => {
|
||||
test('works', () => {
|
||||
const asserts: Record<string, string> = {
|
||||
'': '',
|
||||
'foo-bar': 'foo-bar',
|
||||
'endi-lie': 'endi-lie',
|
||||
'yangshun-tay': 'yangshun-tay',
|
||||
'foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar':
|
||||
'foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-',
|
||||
'foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-test-1-test-2':
|
||||
'foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-test-1-test-',
|
||||
};
|
||||
Object.keys(asserts).forEach((file) => {
|
||||
expect(shortName(file)).toBe(asserts[file]);
|
||||
});
|
||||
});
|
||||
|
||||
// Based on https://github.com/gatsbyjs/gatsby/pull/21518/files
|
||||
|
||||
const SHORT_PATH = `/short/path/without/trailing/slash`;
|
||||
const VERY_LONG_PATH = `/${`x`.repeat(256)}/`;
|
||||
const VERY_LONG_PATH_NON_LATIN = `/${`あ`.repeat(255)}/`;
|
||||
|
||||
it(`Truncates long paths correctly`, () => {
|
||||
const truncatedPathLatin = shortName(VERY_LONG_PATH);
|
||||
const truncatedPathNonLatin = shortName(VERY_LONG_PATH_NON_LATIN);
|
||||
expect(truncatedPathLatin.length).toBeLessThanOrEqual(255);
|
||||
expect(truncatedPathNonLatin.length).toBeLessThanOrEqual(255);
|
||||
});
|
||||
|
||||
it(`Does not truncate short paths`, () => {
|
||||
const truncatedPath = shortName(SHORT_PATH);
|
||||
expect(truncatedPath).toEqual(SHORT_PATH);
|
||||
});
|
||||
});
|
||||
});
|
||||
28
packages/docusaurus-utils/src/docuHash.ts
Normal file
28
packages/docusaurus-utils/src/docuHash.ts
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
/**
|
||||
* Copyright (c) Facebook, Inc. and its affiliates.
|
||||
*
|
||||
* This source code is licensed under the MIT license found in the
|
||||
* LICENSE file in the root directory of this source tree.
|
||||
*/
|
||||
|
||||
import {kebabCase} from 'lodash';
|
||||
|
||||
import {shortName, isNameTooLong, simpleHash} from './pathUtils';
|
||||
|
||||
/**
|
||||
* Given an input string, convert to kebab-case and append a hash.
|
||||
* Avoid str collision.
|
||||
* Also removes part of the string if its larger than the allowed
|
||||
* filename per OS. Avoids ERRNAMETOOLONG error.
|
||||
*/
|
||||
export function docuHash(str: string): string {
|
||||
if (str === '/') {
|
||||
return 'index';
|
||||
}
|
||||
const shortHash = simpleHash(str, 3);
|
||||
const parsedPath = `${kebabCase(str)}-${shortHash}`;
|
||||
if (isNameTooLong(parsedPath)) {
|
||||
return `${shortName(kebabCase(str))}-${shortHash}`;
|
||||
}
|
||||
return parsedPath;
|
||||
}
|
||||
|
|
@ -8,7 +8,7 @@
|
|||
import chalk from 'chalk';
|
||||
import path from 'path';
|
||||
import {createHash} from 'crypto';
|
||||
import {camelCase, kebabCase, mapValues} from 'lodash';
|
||||
import {camelCase, mapValues} from 'lodash';
|
||||
import escapeStringRegexp from 'escape-string-regexp';
|
||||
import fs from 'fs-extra';
|
||||
import {URL} from 'url';
|
||||
|
|
@ -22,6 +22,8 @@ import {
|
|||
import resolvePathnameUnsafe from 'resolve-pathname';
|
||||
|
||||
import {posixPath as posixPathImport} from './posixPath';
|
||||
import {simpleHash} from './pathUtils';
|
||||
import {docuHash} from './docuHash';
|
||||
|
||||
export const posixPath = posixPathImport;
|
||||
|
||||
|
|
@ -29,6 +31,8 @@ export * from './codeTranslationsUtils';
|
|||
export * from './markdownParser';
|
||||
export * from './markdownLinks';
|
||||
export * from './escapePath';
|
||||
export * from './docuHash';
|
||||
export {simpleHash} from './pathUtils';
|
||||
|
||||
const fileHash = new Map();
|
||||
export async function generate(
|
||||
|
|
@ -98,22 +102,6 @@ export function encodePath(userpath: string): string {
|
|||
.join('/');
|
||||
}
|
||||
|
||||
export function simpleHash(str: string, length: number): string {
|
||||
return createHash('md5').update(str).digest('hex').substr(0, length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Given an input string, convert to kebab-case and append a hash.
|
||||
* Avoid str collision.
|
||||
*/
|
||||
export function docuHash(str: string): string {
|
||||
if (str === '/') {
|
||||
return 'index';
|
||||
}
|
||||
const shortHash = simpleHash(str, 3);
|
||||
return `${kebabCase(str)}-${shortHash}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert first string character to the upper case.
|
||||
* E.g: docusaurus -> Docusaurus
|
||||
|
|
|
|||
48
packages/docusaurus-utils/src/pathUtils.ts
Normal file
48
packages/docusaurus-utils/src/pathUtils.ts
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
/**
|
||||
* Copyright (c) Facebook, Inc. and its affiliates.
|
||||
*
|
||||
* This source code is licensed under the MIT license found in the
|
||||
* LICENSE file in the root directory of this source tree.
|
||||
*/
|
||||
|
||||
// Based on https://github.com/gatsbyjs/gatsby/pull/21518/files
|
||||
|
||||
import {createHash} from 'crypto';
|
||||
|
||||
// MacOS (APFS) and Windows (NTFS) filename length limit = 255 chars, Others = 255 bytes
|
||||
const MAX_PATH_SEGMENT_CHARS = 255;
|
||||
const MAX_PATH_SEGMENT_BYTES = 255;
|
||||
// Space for appending things to the string like file extensions and so on
|
||||
const SPACE_FOR_APPENDING = 10;
|
||||
|
||||
const isMacOs = process.platform === `darwin`;
|
||||
const isWindows = process.platform === `win32`;
|
||||
|
||||
export const isNameTooLong = (str: string): boolean => {
|
||||
return isMacOs || isWindows
|
||||
? str.length + SPACE_FOR_APPENDING > MAX_PATH_SEGMENT_CHARS // MacOS (APFS) and Windows (NTFS) filename length limit (255 chars)
|
||||
: Buffer.from(str).length + SPACE_FOR_APPENDING > MAX_PATH_SEGMENT_BYTES; // Other (255 bytes)
|
||||
};
|
||||
|
||||
export const shortName = (str: string): string => {
|
||||
if (isMacOs || isWindows) {
|
||||
const overflowingChars = str.length - MAX_PATH_SEGMENT_CHARS;
|
||||
return str.slice(
|
||||
0,
|
||||
str.length - overflowingChars - SPACE_FOR_APPENDING - 1,
|
||||
);
|
||||
}
|
||||
const strBuffer = Buffer.from(str);
|
||||
const overflowingBytes =
|
||||
Buffer.byteLength(strBuffer) - MAX_PATH_SEGMENT_BYTES;
|
||||
return strBuffer
|
||||
.slice(
|
||||
0,
|
||||
Buffer.byteLength(strBuffer) - overflowingBytes - SPACE_FOR_APPENDING - 1,
|
||||
)
|
||||
.toString();
|
||||
};
|
||||
|
||||
export function simpleHash(str: string, length: number): string {
|
||||
return createHash('md5').update(str).digest('hex').substr(0, length);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue