Add versioning functionality

This commit is contained in:
Frank Li 2017-08-03 10:25:01 -07:00
parent 58452ea963
commit 3598dffc58
12 changed files with 568 additions and 132 deletions

View file

@ -13,7 +13,7 @@ const Marked = require("./Marked.js");
class Doc extends React.Component {
render() {
let editLink =
this.props.config.editUrl &&
!this.props.version && this.props.config.editUrl &&
<a
className="edit-page-link button"
href={
@ -28,7 +28,7 @@ class Doc extends React.Component {
</a>;
if (this.props.language != "en") {
editLink =
this.props.config.recruitingLink &&
!this.props.version && this.props.config.recruitingLink &&
<a
className="edit-page-link button"
href={this.props.config.recruitingLink + "/" + this.props.language}

View file

@ -33,6 +33,7 @@ class DocsLayout extends React.Component {
}
description={content.trim().split("\n")[0]}
language={metadata.language}
version={metadata.version}
>
<div className="docMainWrapper wrapper">
<DocsSidebar metadata={metadata} />

View file

@ -78,6 +78,7 @@ class Site extends React.Component {
section={this.props.section}
title={this.props.config.title}
language={this.props.language}
version={this.props.version}
/>
<div className="navPusher">
{this.props.children}

View file

@ -92,14 +92,22 @@ class HeaderNav extends React.Component {
}
makeInternalLinks(link) {
const linkWithLang = link.href.replace(
let updatedLink = link.href.replace(
/\/LANGUAGE\//,
"/" + this.props.language + "/"
);
if (this.props.version) {
updatedLink = updatedLink.replace(
/\/VERSION\//,
"/" + this.props.version + "/"
);
} else {
updatedLink = updatedLink.replace(/\/VERSION\//, "/");
}
return (
<li key={link.section}>
<a
href={linkWithLang}
href={updatedLink}
className={link.section === this.props.section ? "active" : ""}
>
{translation[this.props.language]
@ -111,14 +119,10 @@ class HeaderNav extends React.Component {
}
makeExternalLinks(link) {
const linkWithLang = link.href.replace(
/\/LANGUAGE\//,
"/" + this.props.language + "/"
);
return (
<li key={link.section}>
<a
href={linkWithLang}
href={link.href}
className={link.section === this.props.section ? "active" : ""}
target={siteConfig.externalLinkTarget || "_self"}
>

View file

@ -78,8 +78,9 @@ class SideNav extends React.Component {
? i18n["localized-strings"][sbTitle] || sbTitle
: sbTitle;
} else {
const id = metadata.original_id || metadata.localized_id;
localizedString = i18n
? i18n["localized-strings"][metadata.localized_id] || metadata.title
? i18n["localized-strings"][id] || metadata.title
: metadata.title;
}
return localizedString;

View file

@ -20,6 +20,9 @@ function execute() {
const Site = require("../core/Site.js");
const siteConfig = require(CWD + "/siteConfig.js");
const translate = require("./translate.js");
const versionFallback = require("./versionFallback.js");
const ENABLE_TRANSLATION = fs.existsSync(CWD + "/languages.js");
let languages;
if (fs.existsSync(CWD + "/languages.js")) {
languages = require(CWD + "/languages.js");
@ -77,35 +80,53 @@ function execute() {
readMetadata.generateDocsMetadata();
const Metadata = require("../core/metadata.js");
let mdToHtml = {};
for (let i = 0; i < Metadata.length; i++) {
const metadata = Metadata[i];
if (metadata.language !== "en") {
continue;
const mdToHtml = {};
Object.keys(Metadata).forEach(id => {
const metadata = Metadata[id];
if (metadata.language !== "en" || metadata.version) {
return;
}
mdToHtml[metadata.source] = siteConfig.baseUrl + metadata.permalink;
let htmlLink = siteConfig.baseUrl + metadata.permalink;
if (htmlLink.includes("/docs/en/")) {
htmlLink = htmlLink.replace("/docs/en/", "/docs/en/VERSION/");
} else {
htmlLink = htmlLink.replace("/docs/", "/docs/VERSION/");
}
mdToHtml[metadata.source] = htmlLink;
});
const DocsLayout = require("../core/DocsLayout.js");
fs.removeSync(CWD + "/build");
// create html files for all English docs
let files = glob.sync(CWD + "/../docs/**");
files.forEach(file => {
// console.log(file);
let language = "en";
// create html files for all docs
Object.keys(Metadata).forEach(id => {
const metadata = Metadata[id];
const extension = path.extname(file);
if (extension === ".md" || extension === ".markdown") {
const result = readMetadata.processMetadata(file);
if (!result) {
let file;
if (metadata.version) {
if (ENABLE_TRANSLATION) {
file =
CWD + "/versioned_docs/" + metadata.language + "/" + metadata.source;
} else {
file = CWD + "/versioned_docs/" + metadata.source;
}
} else {
if (metadata.language === "en") {
file = CWD + "/../docs/" + metadata.source;
} else {
file =
CWD + "/translated_docs/" + metadata.language + "/" + metadata.source;
}
}
if (!fs.existsSync(file)) {
return;
}
let rawContent = readMetadata.extractMetadata(fs.readFileSync(file, "utf8"))
.rawContent;
const metadata = result.metadata;
let rawContent = result.rawContent;
const language = metadata.language;
/* generate table of contents if appropriate */
if (rawContent && rawContent.indexOf(TABLE_OF_CONTENTS_TOKEN) != -1) {
@ -114,11 +135,19 @@ function execute() {
/* replace any links to markdown files to their website html links */
Object.keys(mdToHtml).forEach(function(key, index) {
let link = mdToHtml[key];
link = link.replace("/en/", "/" + language + "/");
link = link.replace(
"/VERSION/",
metadata.version ? "/" + metadata.version + "/" : "/"
);
rawContent = rawContent.replace(new RegExp(key, "g"), link);
});
rawContent = rawContent.replace(
new RegExp(key, "g"),
mdToHtml[key].replace("/en/", "/" + language + "/")
/\]\(assets\//g,
"](" + siteConfig.baseUrl + "docs/assets/"
);
});
const docComp = (
<DocsLayout metadata={metadata} language={language} config={siteConfig}>
@ -126,65 +155,11 @@ function execute() {
</DocsLayout>
);
const str = renderToStaticMarkup(docComp);
let targetFile =
const targetFile =
CWD + "/build/" + siteConfig.projectName + "/" + metadata.permalink;
// console.log(targetFile);
writeFileAndCreateFolder(targetFile, str);
}
});
// create html files for all non-English docs
if (languages.length > 1) {
files = glob.sync(CWD + "/translated_docs/**");
files.forEach(file => {
let language = "en";
const regexSubFolder = /translated_docs\/(.*)\/.*/;
const match = regexSubFolder.exec(file);
if (match) {
language = match[1];
}
if (enabledLanguages.indexOf(language) === -1) {
return;
}
const extension = path.extname(file);
if (extension !== ".md" && extension !== ".markdown") {
return;
}
const result = readMetadata.processMetadata(file);
if (!result) {
return;
}
const metadata = result.metadata;
let rawContent = result.rawContent;
/* generate table of contents if appropriate */
if (rawContent && rawContent.indexOf(TABLE_OF_CONTENTS_TOKEN) != -1) {
rawContent = insertTableOfContents(rawContent);
}
/* replace any links to markdown files to their website html links */
Object.keys(mdToHtml).forEach(function(key, index) {
rawContent = rawContent.replace(new RegExp(key, "g"), mdToHtml[key]);
});
const docComp = (
<DocsLayout metadata={metadata} language={language} config={siteConfig}>
{rawContent}
</DocsLayout>
);
const str = renderToStaticMarkup(docComp);
let targetFile =
CWD + "/build/" + siteConfig.projectName + "/" + metadata.permalink;
// console.log(targetFile);
writeFileAndCreateFolder(targetFile, str);
});
}
/* copy docs assets if they exist */
if (fs.existsSync(CWD + "/../docs/assets")) {

View file

@ -34,8 +34,12 @@ function readCategories(sidebar) {
for (let k = 0; k < enabledLanguages.length; ++k) {
const language = enabledLanguages[k];
const metadatas = Metadata.filter(metadata => {
return metadata.sidebar === sidebar && metadata.language === language;
const metadatas = [];
Object.keys(Metadata).forEach(id => {
const metadata = Metadata[id];
if (metadata.sidebar === sidebar && metadata.language === language) {
metadatas.push(metadata);
}
});
// Build a hashmap of article_id -> metadata

View file

@ -14,6 +14,7 @@ const fs = require("fs");
const os = require("os");
const glob = require("glob");
const siteConfig = require(CWD + "/siteConfig.js");
const versionFallback = require("./versionFallback");
let languages;
if (fs.existsSync(CWD + "/languages.js")) {
languages = require(CWD + "/languages.js");
@ -28,7 +29,9 @@ if (fs.existsSync(CWD + "/languages.js")) {
}
function readSidebar() {
const allSidebars = require(CWD + "/sidebar.json");
let allSidebars = require(CWD + "/sidebar.json");
Object.assign(allSidebars, versionFallback.sidebarData());
const order = {};
Object.keys(allSidebars).forEach(sidebar => {
@ -148,7 +151,7 @@ function generateDocsMetadata() {
enabledLanguages.push(lang.tag);
});
const metadatas = [];
const metadatas = {};
/* metadata for english files */
let files = glob.sync(CWD + "/../docs/**");
@ -163,7 +166,7 @@ function generateDocsMetadata() {
return;
}
let metadata = res.metadata;
metadatas.push(metadata);
metadatas[metadata.id] = metadata;
}
});
@ -188,10 +191,32 @@ function generateDocsMetadata() {
return;
}
let metadata = res.metadata;
metadatas.push(metadata);
metadatas[metadata.id] = metadata;
}
});
versionData = versionFallback.docData();
versionData.forEach(metadata => {
const id = metadata.localized_id;
metadata.sidebar = order[id].sidebar;
metadata.category = order[id].category;
if (order[id].next) {
metadata.next_id = order[id].next.replace(
"version-" + metadata.version + "-",
""
);
metadata.next = metadata.language + "-" + order[id].next;
}
if (order[id].previous) {
metadata.previous_id = order[id].previous.replace(
"version-" + metadata.version + "-",
""
);
metadata.previous = metadata.language + "-" + order[id].previous;
}
metadatas[metadata.id] = metadata;
});
fs.writeFileSync(
__dirname + "/../core/metadata.js",
"/**\n" +
@ -245,6 +270,7 @@ function generateBlogMetadata() {
}
module.exports = {
readSidebar,
extractMetadata,
processMetadata,
generateDocsMetadata,

View file

@ -20,6 +20,7 @@ function execute(port) {
const mkdirp = require("mkdirp");
const glob = require("glob");
const translate = require("./translate.js");
const versionFallback = require("./versionFallback");
const CWD = process.cwd();
const ENABLE_TRANSLATION = fs.existsSync(CWD + "/languages.js");
@ -127,37 +128,59 @@ function execute(port) {
purgeCache(CWD + "/siteConfig.js");
siteConfig = require(CWD + "/siteConfig.js");
let url = req.path.toString().replace(siteConfig.baseUrl, "");
reloadMetadata();
// links is a map from a permalink to an id
let links = {};
for (let i = 0; i < Metadata.length; i++) {
const metadata = Metadata[i];
if (metadata.language === "en") {
links[metadata.permalink] = CWD + "/../docs/" + metadata.source;
Object.keys(Metadata).forEach(id => {
const metadata = Metadata[id];
links[metadata.permalink] = id;
});
const mdToHtml = {};
Object.keys(Metadata).forEach(id => {
const metadata = Metadata[id];
if (metadata.language !== "en" || metadata.version) {
return;
}
let htmlLink = siteConfig.baseUrl + metadata.permalink;
if (htmlLink.includes("/docs/en/")) {
htmlLink = htmlLink.replace("/docs/en/", "/docs/en/VERSION/");
} else {
links[metadata.permalink] =
htmlLink = htmlLink.replace("/docs/", "/docs/VERSION/");
}
mdToHtml[metadata.source] = htmlLink;
});
const metadata = Metadata[links[url]];
const language = metadata.language;
let file;
if (metadata.version) {
if (ENABLE_TRANSLATION) {
file =
CWD + "/versioned_docs/" + metadata.language + "/" + metadata.source;
} else {
file = CWD + "/versioned_docs/" + metadata.source;
}
} else {
if (metadata.language === "en") {
file = CWD + "/../docs/" + metadata.source;
} else {
file =
CWD + "/translated_docs/" + metadata.language + "/" + metadata.source;
}
}
let mdToHtml = {};
for (let i = 0; i < Metadata.length; i++) {
const metadata = Metadata[i];
if (metadata.language !== "en") {
continue;
}
mdToHtml[metadata.source] = siteConfig.baseUrl + metadata.permalink;
}
let file = links[req.path.toString().replace(siteConfig.baseUrl, "")];
if (!fs.existsSync(file)) {
next();
return;
}
const result = readMetadata.processMetadata(file);
const metadata = result.metadata;
const language = metadata.language;
let rawContent = result.rawContent;
let rawContent = readMetadata.extractMetadata(fs.readFileSync(file, "utf8"))
.rawContent;
/* generate table of contents if appropriate */
if (rawContent && rawContent.indexOf(TABLE_OF_CONTENTS_TOKEN) !== -1) {
@ -166,10 +189,13 @@ function execute(port) {
/* replace any links to markdown files to their website html links */
Object.keys(mdToHtml).forEach(function(key, index) {
rawContent = rawContent.replace(
new RegExp(key, "g"),
mdToHtml[key].replace("/en/", "/" + language + "/")
let link = mdToHtml[key];
link = link.replace("/en/", "/" + language + "/");
link = link.replace(
"/VERSION/",
metadata.version ? "/" + metadata.version + "/" : "/"
);
rawContent = rawContent.replace(new RegExp(key, "g"), link);
});
rawContent = rawContent.replace(
@ -187,6 +213,7 @@ function execute(port) {
res.send(renderToStaticMarkup(docComp));
});
/* handle all requests for blog pages and posts */
app.get(/blog\/.*html$/, (req, res) => {
purgeCache(CWD + "/siteConfig.js");

View file

@ -0,0 +1,278 @@
/**
* Copyright (c) 2017-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
const CWD = process.cwd();
const semver = require("semver");
const glob = require("glob");
const fs = require("fs");
const path = require("path");
const diff = require("diff");
const assert = require("assert");
const siteConfig = require(CWD + "/siteConfig.js");
const ENABLE_TRANSLATION = fs.existsSync(CWD + "/languages.js");
let languages;
if (fs.existsSync(CWD + "/languages.js")) {
languages = require(CWD + "/languages.js");
} else {
languages = [
{
enabled: true,
name: "English",
tag: "en"
}
];
}
/*****************************************************************/
// included to prevent cyclical dependency with readMetadata.js
function splitHeader(content) {
const lines = content.split("\n");
let i = 1;
for (; i < lines.length - 1; ++i) {
if (lines[i] === "---") {
break;
}
}
return {
header: lines.slice(1, i + 1).join("\n"),
content: lines.slice(i + 1).join("\n")
};
}
// Extract markdown metadata header
function extractMetadata(content) {
const metadata = {};
const both = splitHeader(content);
const lines = both.header.split("\n");
for (let i = 0; i < lines.length - 1; ++i) {
const keyvalue = lines[i].split(":");
const key = keyvalue[0].trim();
let value = keyvalue.slice(1).join(":").trim();
// Handle the case where you have "Community #10"
try {
value = JSON.parse(value);
} catch (e) {}
metadata[key] = value;
}
return { metadata, rawContent: both.content };
}
/*****************************************************************/
/* preprocessing */
const versions = [];
const versionFolder = ENABLE_TRANSLATION
? CWD + "/versioned_docs/en/"
: CWD + "/versioned_docs/";
let files = glob.sync(versionFolder + "*");
files.forEach(file => {
if (!fs.lstatSync(file).isDirectory()) {
return;
}
const version = file.split("version-")[1];
versions.push(version);
});
versions.sort(semver.rcompare);
const available = {};
const versionFiles = {};
files = glob.sync(versionFolder + "**");
files.forEach(file => {
const ext = path.extname(file);
if (ext !== ".md" && ext !== ".markdown") {
return;
}
const res = extractMetadata(fs.readFileSync(file, "utf8"));
const metadata = res.metadata;
if (!(metadata.original_id in available)) {
available[metadata.original_id] = new Set();
}
const version = metadata.id.split("-")[1];
available[metadata.original_id].add(version);
if (!(version in versionFiles)) {
versionFiles[version] = {};
}
versionFiles[version][metadata.original_id] = file;
});
function docVersion(id, req_version) {
for (let i = 0; i < versions.length; i++) {
if (semver.gt(versions[i], req_version)) {
continue;
}
if (!available[id]) {
return null;
}
if (available[id].has(versions[i])) {
return versions[i];
}
}
return null;
}
function diffLatestDoc(file, id) {
if (versions.length === 0) {
return true;
}
const latest = versions[0];
const version = docVersion(id, latest);
if (!version) {
return true;
}
const latestFile = versionFiles[version][id];
if (!latestFile || !fs.existsSync(latestFile)) {
return true;
}
const diffs = diff.diffChars(
extractMetadata(fs.readFileSync(latestFile, "utf8")).rawContent,
extractMetadata(fs.readFileSync(file, "utf8")).rawContent
);
diffs.forEach(part => {
if (part.added || part.removed) {
return true;
}
});
return false;
}
function processVersionMetadata(file, version, useVersion, language) {
const metadata = extractMetadata(fs.readFileSync(file, "utf8")).metadata;
metadata.source = "version-" + useVersion + "/" + path.basename(file);
if (!ENABLE_TRANSLATION && !siteConfig.useEnglishUrl) {
metadata.permalink =
"docs/" + version + "/" + metadata.original_id + ".html";
} else {
metadata.permalink =
"docs/" + language + "/" + version + "/" + metadata.original_id + ".html";
}
metadata.id = metadata.id.replace(
"version-" + useVersion + "-",
"version-" + version + "-"
);
metadata.localized_id = metadata.id;
metadata.id = language + "-" + metadata.id;
metadata.language = language;
metadata.version = version;
return metadata;
}
function docData() {
const files = glob.sync(CWD + "/versioned_docs/**");
allIds = new Set();
Object.keys(versionFiles).forEach(version => {
Object.keys(versionFiles[version]).forEach(id => {
allIds.add(id);
});
});
const metadatas = [];
languages.filter(language => language.enabled).forEach(language => {
versions.forEach(version => {
allIds.forEach(id => {
const useVersion = docVersion(id, version);
if (!useVersion) {
return;
}
const file = versionFiles[useVersion][id];
metadatas.push(
processVersionMetadata(file, version, useVersion, language.tag)
);
});
});
});
return metadatas;
}
function sidebarVersion(req_version) {
for (let i = 0; i < versions.length; i++) {
if (semver.gt(versions[i], req_version)) {
continue;
}
if (
fs.existsSync(
CWD + "/versioned_sidebars/version-" + versions[i] + "-sidebar.json"
)
) {
return versions[i];
}
}
return null;
}
function diffLatestSidebar() {
if (versions.length === 0) {
return true;
}
const latest = versions[0];
const version = sidebarVersion(latest);
const latestSidebar =
CWD + "/versioned_sidebars/version-" + version + "-sidebar.json";
if (!fs.existsSync(latestSidebar)) {
return true;
}
const currentSidebar = CWD + "/sidebar.json";
if (!fs.existsSync(currentSidebar)) {
// TO DO: error message
}
// compare for equality between latest version sidebar with version prefixes
// stripped and current sidebar
return (
JSON.stringify(JSON.parse(fs.readFileSync(latestSidebar, "utf8"))).replace(
new RegExp("version-" + version + "-", "g"),
""
) !== JSON.stringify(JSON.parse(fs.readFileSync(currentSidebar, "utf8")))
);
}
function sidebarData() {
const allSidebars = {};
for (let i = 0; i < versions.length; i++) {
const version = sidebarVersion(versions[i]);
const sidebar = JSON.parse(
fs
.readFileSync(
CWD + "/versioned_sidebars/version-" + version + "-sidebar.json",
"utf8"
)
.replace(
new RegExp("version-" + version + "-", "g"),
"version-" + versions[i] + "-"
)
);
Object.assign(allSidebars, sidebar);
}
return allSidebars;
}
module.exports = {
docVersion,
diffLatestDoc,
processVersionMetadata,
docData,
sidebarVersion,
diffLatestSidebar,
sidebarData
};

116
lib/version.js Normal file
View file

@ -0,0 +1,116 @@
#!/usr/bin/env node
/**
* Copyright (c) 2017-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
const CWD = process.cwd();
const glob = require("glob");
const fs = require("fs-extra");
const path = require("path");
const mkdirp = require("mkdirp");
const semver = require("semver");
const readMetadata = require("./server/readMetadata.js");
const versionFallback = require("./server/versionFallback.js");
const ENABLE_TRANSLATION = fs.existsSync(CWD + "/languages.js");
let version;
const program = require("commander");
program
.arguments("<version>")
.action(ver => {
version = ver;
})
.parse(process.argv);
if (typeof version === "undefined") {
console.error(
"No version number specified!\nPass the version you wish to create as an argument.\nEx: 1.0.0"
);
process.exit(1);
}
if (!(version = semver.valid(version))) {
console.error(
"Invalid version!\nSpecify a valid version following the specifications at http://semver.org/."
);
process.exit(1);
}
function makeHeader(metadata) {
let header = "---\n";
Object.keys(metadata).forEach(key => {
header += key + ": " + metadata[key] + "\n";
});
header += "---\n";
return header;
}
let versionFolder = CWD + "/versioned_docs/version-" + version;
if (ENABLE_TRANSLATION) {
versionFolder = CWD + "/versioned_docs/en/version-" + version;
}
mkdirp.sync(versionFolder);
let files = glob.sync(CWD + "/../docs/*");
files.forEach(file => {
const ext = path.extname(file);
if (ext !== ".md" && ext !== ".markdown") {
return;
}
const res = readMetadata.extractMetadata(fs.readFileSync(file, "utf8"));
let metadata = res.metadata;
let rawContent = res.rawContent;
if (!metadata.id) {
return;
}
if (!versionFallback.diffLatestDoc(file, metadata.id)) {
return;
}
metadata.original_id = metadata.id;
metadata.id = "version-" + version + "-" + metadata.id;
let targetFile =
CWD + "/versioned_docs/version-" + version + "/" + path.basename(file);
if (ENABLE_TRANSLATION) {
targetFile = CWD + "/versioned_docs/en/version-" + version + "/" + path.basename(file);
}
fs.writeFileSync(targetFile, makeHeader(metadata) + rawContent, "utf8");
});
if (versionFallback.diffLatestSidebar()) {
mkdirp(CWD + "/versioned_sidebars");
const sidebar = JSON.parse(fs.readFileSync(CWD + "/sidebar.json", "utf8"));
const versioned = {};
Object.keys(sidebar).forEach(sb => {
const version_sb = "version-" + version + "-" + sb;
versioned[version_sb] = {};
const categories = sidebar[sb];
Object.keys(categories).forEach(category => {
versioned[version_sb][category] = [];
const ids = categories[category];
ids.forEach((id, index) => {
versioned[version_sb][category].push("version-" + version + "-" + id);
});
});
});
fs.writeFileSync(
CWD + "/versioned_sidebars/version-" + version + "-sidebar.json",
JSON.stringify(versioned, null, 2),
"utf8"
);
}

View file

@ -12,6 +12,7 @@
"babylon": "^6.17.4",
"classnames": "^2.2.5",
"commander": "^2.11.0",
"diff": "^3.3.0",
"express": "^4.15.3",
"fs-extra": "^3.0.1",
"glob": "^7.1.2",
@ -19,6 +20,7 @@
"react": "^15.5.4",
"react-dom": "^15.5.4",
"request": "^2.81.0",
"semver": "^5.4.1",
"shelljs": "^0.7.8"
},
"name": "docusaurus",
@ -28,6 +30,7 @@
"docusaurus-build": "./lib/build-files.js",
"docusaurus-publish": "./lib/publish-gh-pages.js",
"docusaurus-examples": "./lib/copy-examples.js",
"docusaurus-write-translations": "./lib/write-translations.js"
"docusaurus-write-translations": "./lib/write-translations.js",
"docusaurus-version": "./lib/version.js"
}
}