tools: produce JSON documentation using unified/remark/rehype

PR-URL: #21697 Reviewed-By: Vse Mozhet Byt <[email protected]> Reviewed-By: Rich Trott <[email protected]> Reviewed-By: Trivikram Kamat <[email protected]>

Author: rubys

.gitignore

@@ -107,6 +107,9 @@ deps/npm/node_modules/.bin/
 /*.pkg
 /SHASUMS*.txt*
 
+# api docs artifacts
+tools/doc/node_modules
+
 # test artifacts
 tools/remark-cli/node_modules
 tools/remark-preset-lint-node/node_modules

Makefile

@@ -320,7 +320,7 @@ ifeq ($(OSTYPE),aix)
 DOCBUILDSTAMP_PREREQS := $(DOCBUILDSTAMP_PREREQS) out/$(BUILDTYPE)/node.exp
 endif
 
-test/addons/.docbuildstamp: $(DOCBUILDSTAMP_PREREQS)
+test/addons/.docbuildstamp: $(DOCBUILDSTAMP_PREREQS) tools/doc/node_modules
 $(RM) -r test/addons/??_*/
  [ -x$(NODE) ] &&$(NODE)$<|| node $<
  touch $@
@@ -633,15 +633,15 @@ available-node = \
 run-npm-install = $(PWD)/$(NPM) install --production --no-package-lock
 run-npm-ci = $(PWD)/$(NPM) ci
 
-gen-json = tools/doc/generate.js --format=json $< > $@
-gen-html = tools/doc/generate.js --node-version=$(FULLVERSION) --format=html \
- --analytics=$(DOCS_ANALYTICS)$< > $@
+tools/doc/node_modules/js-yaml/package.json:
+cd tools/doc &&$(call available-node,$(run-npm-install))
 
-out/doc/api/%.json: doc/api/%.md tools/doc/generate.js tools/doc/json.js
-$(call available-node, $(gen-json))
+gen-api = tools/doc/generate.js --node-version=$(FULLVERSION)\
+ --analytics=$(DOCS_ANALYTICS)$< --output-directory=out/doc/api
 
-out/doc/api/%.html: doc/api/%.md tools/doc/generate.js tools/doc/html.js
-$(call available-node, $(gen-html))
+out/doc/api/%.jsonout/doc/api/%.html: doc/api/%.md tools/doc/generate.js \
+ tools/doc/html.js tools/doc/json.js
+$(call available-node, $(gen-api))
 
 out/doc/api/all.html: $(apidocs_html) tools/doc/allhtml.js
 $(call available-node, tools/doc/allhtml.js)

doc/api/https.md

@@ -123,7 +123,7 @@ changes:
  pr-url: https://github.com/nodejs/node/pull/10638
  description: The `options` parameter can be a WHATWG `URL` object.
 -->
--`url`{string | URL}
+*`url`{string | URL}
 *`options`{Object | string | URL} Accepts the same `options` as
 [`https.request()`][], with the `method` always set to `GET`.
 *`callback`{Function}
@@ -174,7 +174,7 @@ changes:
  pr-url: https://github.com/nodejs/node/pull/10638
  description: The `options` parameter can be a WHATWG `URL` object.
 -->
--`url`{string | URL}
+*`url`{string | URL}
 *`options`{Object | string | URL} Accepts all `options` from
 [`http.request()`][], with some differences in default values:
 -`protocol`**Default:**`'https:'`

test/doctool/test-doctool-html.js

@@ -11,7 +11,34 @@ try{
 constassert=require('assert');
 const{ readFile }=require('fs');
 constfixtures=require('../common/fixtures');
-consttoHTML=require('../../tools/doc/html.js');
+consthtml=require('../../tools/doc/html.js');
+constpath=require('path');
+
+module.paths.unshift(
+path.join(__dirname,'..','..','tools','doc','node_modules'));
+constunified=require('unified');
+constmarkdown=require('remark-parse');
+constremark2rehype=require('remark-rehype');
+constraw=require('rehype-raw');
+consthtmlStringify=require('rehype-stringify');
+
+functiontoHTML({ input, filename, nodeVersion, analytics },cb){
+constcontent=unified()
+.use(markdown)
+.use(html.firstHeader)
+.use(html.preprocessText)
+.use(html.preprocessElements,{ filename })
+.use(html.buildToc,{ filename })
+.use(remark2rehype,{allowDangerousHTML: true})
+.use(raw)
+.use(htmlStringify)
+.processSync(input);
+
+html.toHTML(
+{ input, content, filename, nodeVersion, analytics },
+cb
+);
+}
 
 // Test data is a list of objects with two properties.
 // The file property is the file path.
@@ -80,7 +107,6 @@ testData.forEach(({file, html, analyticsId }) =>{
 
 readFile(file,'utf8',common.mustCall((err,input)=>{
 assert.ifError(err);
-
 toHTML(
 {
 input: input,

test/doctool/test-doctool-json.js

@@ -10,9 +10,27 @@ try{
 
 constassert=require('assert');
 constfs=require('fs');
+constpath=require('path');
 constfixtures=require('../common/fixtures');
 constjson=require('../../tools/doc/json.js');
 
+module.paths.unshift(
+path.join(__dirname,'..','..','tools','doc','node_modules'));
+constunified=require('unified');
+constmarkdown=require('remark-parse');
+
+functiontoJSON(input,filename,cb){
+functionnullCompiler(){
+this.Compiler=(tree)=>tree;
+}
+
+unified()
+.use(markdown)
+.use(json.jsonAPI,{ filename })
+.use(nullCompiler)
+.process(input,cb);
+}
+
 // Outputs valid json with the expected fields when given simple markdown
 // Test data is a list of objects with two properties.
 // The file property is the file path.
@@ -21,15 +39,16 @@ const testData = [
 {
 file: fixtures.path('sample_document.md'),
 json: {
+type: 'module',
 source: 'foo',
 modules: [{
 textRaw: 'Sample Markdown',
 name: 'sample_markdown',
 modules: [{
 textRaw: 'Seussian Rhymes',
 name: 'seussian_rhymes',
-desc: '<ol>\n<li>fish</li>\n<li><p>fish</p>\n</li>\n<li>'+
-'<p>Red fish</p>\n</li>\n<li>Blue fish</li>\n</ol>\n',
+desc: '<ol>\n<li>fish</li>\n<li>fish</li>\n</ol>\n'+
+'<ul>\n<li>Red fish</li>\n<li>Blue fish</li>\n</ul>',
 type: 'module',
 displayName: 'Seussian Rhymes'
 }],
@@ -41,6 +60,7 @@ const testData = [
 {
 file: fixtures.path('order_of_end_tags_5873.md'),
 json: {
+type: 'module',
 source: 'foo',
 modules: [{
 textRaw: 'Title',
@@ -55,15 +75,10 @@ const testData = [
 signatures: [
 {
 params: [{
-textRaw: '`array`{Array}',
+textRaw: '`array`{Array}',
 name: 'array',
 type: 'Array'
 }]
-},
-{
-params: [{
-name: 'array'
-}]
 }
 ]
 }],
@@ -78,6 +93,7 @@ const testData = [
 {
 file: fixtures.path('doc_with_yaml.md'),
 json: {
+type: 'module',
 source: 'foo',
 modules: [
 {
@@ -92,7 +108,7 @@ const testData = [
 changes: []
 },
 desc: '<p>Describe <code>Foobar</code> in more detail '+
-'here.</p>\n',
+'here.</p>',
 type: 'module',
 displayName: 'Foobar'
 },
@@ -110,7 +126,7 @@ const testData = [
 ]
 },
 desc: '<p>Describe <code>Foobar II</code> in more detail '+
-'here. fg(1)</p>\n',
+'here. fg(1)</p>',
 type: 'module',
 displayName: 'Foobar II'
 },
@@ -123,15 +139,15 @@ const testData = [
 changes: []
 },
 desc: '<p>Describe <code>Deprecated thingy</code> in more '+
-'detail here. fg(1p)</p>\n',
+'detail here. fg(1p)</p>',
 type: 'module',
 displayName: 'Deprecated thingy'
 },
 {
 textRaw: 'Something',
 name: 'something',
 desc: '<!-- This is not a metadata comment -->\n<p>'+
-'Describe <code>Something</code> in more detail here.</p>\n',
+'Describe <code>Something</code> in more detail here.</p>',
 type: 'module',
 displayName: 'Something'
 }
@@ -147,9 +163,9 @@ const testData = [
 testData.forEach((item)=>{
 fs.readFile(item.file,'utf8',common.mustCall((err,input)=>{
 assert.ifError(err);
-json(input,'foo',common.mustCall((err,output)=>{
+toJSON(input,'foo',common.mustCall((err,output)=>{
 assert.ifError(err);
-assert.deepStrictEqual(output,item.json);
+assert.deepStrictEqual(output.json,item.json);
 }));
 }));
 });

tools/doc/generate.js

@@ -22,53 +22,72 @@
 'use strict';
 
 constfs=require('fs');
+constpath=require('path');
+constunified=require('unified');
+constmarkdown=require('remark-parse');
+constremark2rehype=require('remark-rehype');
+constraw=require('rehype-raw');
+consthtmlStringify=require('rehype-stringify');
+
+consthtml=require('./html');
+constjson=require('./json');
 
 // Parse the args.
 // Don't use nopt or whatever for this. It's simple enough.
 
 constargs=process.argv.slice(2);
-letformat='json';
 letfilename=null;
 letnodeVersion=null;
 letanalytics=null;
+letoutputDir=null;
 
 args.forEach(function(arg){
 if(!arg.startsWith('--')){
 filename=arg;
-}elseif(arg.startsWith('--format=')){
-format=arg.replace(/^--format=/,'');
 }elseif(arg.startsWith('--node-version=')){
 nodeVersion=arg.replace(/^--node-version=/,'');
 }elseif(arg.startsWith('--analytics=')){
 analytics=arg.replace(/^--analytics=/,'');
+}elseif(arg.startsWith('--output-directory=')){
+outputDir=arg.replace(/^--output-directory=/,'');
 }
 });
 
 nodeVersion=nodeVersion||process.version;
 
 if(!filename){
 thrownewError('No input file specified');
+}elseif(!outputDir){
+thrownewError('No output directory specified');
 }
 
+
 fs.readFile(filename,'utf8',(er,input)=>{
 if(er)thrower;
-switch(format){
-case'json':
-require('./json.js')(input,filename,(er,obj)=>{
-if(er)thrower;
-console.log(JSON.stringify(obj,null,2));
-});
-break;
 
-case'html':
-require('./html')({ input, filename, nodeVersion, analytics },
-(err,html)=>{
-if(err)throwerr;
-console.log(html);
-});
-break;
+constcontent=unified()
+.use(markdown)
+.use(json.jsonAPI,{ filename })
+.use(html.firstHeader)
+.use(html.preprocessText)
+.use(html.preprocessElements,{ filename })
+.use(html.buildToc,{ filename })
+.use(remark2rehype,{allowDangerousHTML: true})
+.use(raw)
+.use(htmlStringify)
+.processSync(input);
 
-default:
-thrownewError(`Invalid format: ${format}`);
-}
+constbasename=path.basename(filename,'.md');
+
+html.toHTML(
+{ input, content, filename, nodeVersion, analytics },
+(err,html)=>{
+consttarget=path.join(outputDir,`${basename}.html`);
+if(err)throwerr;
+fs.writeFileSync(target,html);
+}
+);
+
+consttarget=path.join(outputDir,`${basename}.json`);
+fs.writeFileSync(target,JSON.stringify(content.json,null,2));
 });

tools/doc/html.js

@@ -29,11 +29,13 @@ const visit = require('unist-util-visit');
 constmarkdown=require('remark-parse');
 constremark2rehype=require('remark-rehype');
 constraw=require('rehype-raw');
-consthtml=require('rehype-stringify');
+consthtmlStringify=require('rehype-stringify');
 constpath=require('path');
 consttypeParser=require('./type-parser.js');
 
-module.exports=toHTML;
+module.exports={
+ toHTML, firstHeader, preprocessText, preprocessElements, buildToc
+};
 
 constdocPath=path.resolve(__dirname,'..','..','doc');
 
@@ -54,26 +56,15 @@ const gtocHTML = unified()
 .use(remark2rehype,{allowDangerousHTML: true})
 .use(raw)
 .use(navClasses)
-.use(html)
+.use(htmlStringify)
 .processSync(gtocMD).toString();
 
 consttemplatePath=path.join(docPath,'template.html');
 consttemplate=fs.readFileSync(templatePath,'utf8');
 
-functiontoHTML({ input, filename, nodeVersion, analytics },cb){
+functiontoHTML({ input,content,filename, nodeVersion, analytics },cb){
 filename=path.basename(filename,'.md');
 
-constcontent=unified()
-.use(markdown)
-.use(firstHeader)
-.use(preprocessText)
-.use(preprocessElements,{ filename })
-.use(buildToc,{ filename })
-.use(remark2rehype,{allowDangerousHTML: true})
-.use(raw)
-.use(html)
-.processSync(input);
-
 constid=filename.replace(/\W+/g,'-');
 
 letHTML=template.replace('__ID__',id)
@@ -296,7 +287,7 @@ function parseYAML(text){
 .use(markdown)
 .use(remark2rehype,{allowDangerousHTML: true})
 .use(raw)
-.use(html)
+.use(htmlStringify)
 .processSync(change.description).toString();
 
 result+=`<tr><td>${change.version}</td>\n`+
@@ -381,7 +372,7 @@ function buildToc({filename }){
 .use(markdown)
 .use(remark2rehype,{allowDangerousHTML: true})
 .use(raw)
-.use(html)
+.use(htmlStringify)
 .processSync(toc).toString();
 };
 }