From 0968b8ffbae62db8fcbb2f724d61bf24642072e2 Mon Sep 17 00:00:00 2001 From: Jieiku Date: Wed, 27 Sep 2023 12:48:48 -0700 Subject: Fixes for Offline Site Build --- package_abridge.js | 326 +++++++++++++++++++++++++++-------------------------- 1 file changed, 165 insertions(+), 161 deletions(-) (limited to 'package_abridge.js') diff --git a/package_abridge.js b/package_abridge.js index eb02098..4f12bb5 100644 --- a/package_abridge.js +++ b/package_abridge.js @@ -4,17 +4,12 @@ const TOML = require('fast-toml'); const UglifyJS = require('uglify-js'); const jsonminify = require("jsonminify"); const replace = require('replace-in-file'); -//const crypto = require('crypto'); (disabled for now, because unused) - -// check if abridge is used directly or as a theme. -bpath = ''; -if (fs.existsSync('./themes')) { - bpath = 'themes/abridge/'; -} +const util = require("util"); +const { exec } = require("child_process"); +const execPromise = util.promisify(exec); const tomlString = String(fs.readFileSync('config.toml')); const data = TOML.parse(tomlString); - const js_prestyle = data.extra.js_prestyle; const js_switcher = data.extra.js_switcher; const js_email_encode = data.extra.js_email_encode; @@ -25,7 +20,7 @@ const uglyurls = data.extra.uglyurls; const js_bundle = data.extra.js_bundle; const offline = data.extra.offline; const online_url = data.extra.online_url; -const sitePath = path.resolve('public');// used for offline builds +const online_indexformat = data.extra.online_indexformat; const pwa = data.extra.pwa; const pwa_VER = data.extra.pwa_VER; const pwa_NORM_TTL = data.extra.pwa_NORM_TTL; @@ -36,23 +31,169 @@ const pwa_TTL_EXEMPT = data.extra.pwa_TTL_EXEMPT; const pwa_cache_all = data.extra.pwa_cache_all; const pwa_BASE_CACHE_FILES = data.extra.pwa_BASE_CACHE_FILES; -if (offline === false) { - replace.sync({files: 'config.toml', from: /base_url.*=.*/g, to: "base_url=\""+online_url+"\""}); -} else if (offline === true) { - replace.sync({files: 'config.toml', from: /base_url.*=.*/g, to: "base_url=\""+sitePath+"\""}); +async function execWrapper(cmd) { + const { stdout, stderr } = await execPromise(cmd); + if (stdout) { + console.log(stdout); + } + if (stderr) { + console.log('Error: '+stderr); + } } -base_url = data.base_url; -if (base_url.slice(-1) == "/") { - base_url = base_url.slice(0, -1); -} +async function abridge() { + if (offline === false) { + replace.sync({files: 'config.toml', from: /base_url.*=.*/g, to: "base_url = \""+online_url+"\""}); + replace.sync({files: 'config.toml', from: /index_format.*=.*/g, to: "index_format = \""+online_indexformat+"\""}); + } else if (offline === true) { + replace.sync({files: 'config.toml', from: /base_url.*=.*/g, to: "base_url = \""+__dirname+"\/public\""}); + replace.sync({files: 'config.toml', from: /index_format.*=.*/g, to: "index_format = \"elasticlunr_javascript\""}); + } + + console.log('Zola Build to generate files for minification:'); + await execWrapper('zola build'); + + // check if abridge is used directly or as a theme. + bpath = ''; + if (fs.existsSync('./themes')) { + bpath = 'themes/abridge/'; + } -// disabled for now, because unused -// function checksum(input) { -// var hash = crypto.createHash('sha384').update(input, 'utf8'); -// var hashBase64 = hash.digest('base64'); -// return 'sha384-' + hashBase64; -// } + base_url = data.base_url; + if (base_url.slice(-1) == "/") { + base_url = base_url.slice(0, -1); + } + + if (search_library === 'elasticlunr') { + if (fs.existsSync('content/static/stork_toml.md')) { + replace.sync({files: 'content/static/stork_toml.md', from: /draft.*=.*/g, to: "draft = true"}); + } + if (fs.existsSync('content/static/tinysearch_json.md')) { + replace.sync({files: 'content/static/tinysearch_json.md', from: /draft.*=.*/g, to: "draft = true"}); + } + } else if (search_library === 'tinysearch') { + if (!fs.existsSync('content/static/tinysearch_json.md')) {// 'content/static/tinysearch_json.md' file is missing, copy from abridge theme. + fs.copyFileSync(bpath+'content/static/tinysearch_json.md', 'content/static/tinysearch_json.md',fs.constants.COPYFILE_EXCL); + } + if (fs.existsSync('content/static/stork_toml.md')) { + replace.sync({files: 'content/static/stork_toml.md', from: /draft.*=.*/g, to: "draft = true"}); + } + if (fs.existsSync('content/static/tinysearch_json.md')) { + replace.sync({files: 'content/static/tinysearch_json.md', from: /draft.*=.*/g, to: "draft = false"}); + } + // zola build && mkdir -p tmp && tinysearch --optimize --path tmp public/data_tinysearch/index.html && rsync -avz tmp/*.wasm static/ && rm -rf tmp + } else if (search_library === 'stork') { + + if (!fs.existsSync('content/static/stork_toml.md')) {// 'content/static/stork_toml.md' file is missing, copy from abridge theme. + fs.copyFileSync(bpath+'content/static/stork_toml.md', 'content/static/stork_toml.md',fs.constants.COPYFILE_EXCL); + } + if (fs.existsSync('content/static/stork_toml.md')) { + replace.sync({files: 'content/static/stork_toml.md', from: /draft.*=.*/g, to: "draft = false"}); + } + if (fs.existsSync('content/static/tinysearch_json.md')) { + replace.sync({files: 'content/static/tinysearch_json.md', from: /draft.*=.*/g, to: "draft = true"}); + } + // zola build && stork build --input public/data_stork/index.html --output static/stork.st + } + + if (pwa) {// Update pwa settings, file list, and hashes. + // update from abridge theme. + fs.copyFileSync(bpath+'static/sw.js', 'static/sw.js'); + fs.copyFileSync(bpath+'static/js/sw_load.js', 'static/js/sw_load.js'); + // Update settings in PWA javascript file, using options parsed from config.toml. sw.min.js?v=3.10.0", "++" + if (fs.existsSync('static/js/sw_load.js')) { + sw_load_min = '.js?v='; + if (js_bundle) { + sw_load_min = '.min.js?v='; + } + replace.sync({files: 'static/js/sw_load.js', from: /sw.*v=.*/g, to: "sw"+sw_load_min+pwa_VER+"\","}); + } + if (fs.existsSync('static/sw.js')) { + replace.sync({files: 'static/sw.js', from: /NORM_TTL.*=.*/g, to: "NORM_TTL = "+pwa_NORM_TTL+";"}); + replace.sync({files: 'static/sw.js', from: /LONG_TTL.*=.*/g, to: "LONG_TTL = "+pwa_LONG_TTL+";"}); + replace.sync({files: 'static/sw.js', from: /TTL_NORM.*=.*/g, to: "TTL_NORM = ["+pwa_TTL_NORM+"];"}); + replace.sync({files: 'static/sw.js', from: /TTL_LONG.*=.*/g, to: "TTL_LONG = ["+pwa_TTL_LONG+"];"}); + replace.sync({files: 'static/sw.js', from: /TTL_EXEMPT.*=.*/g, to: "TTL_EXEMPT = ["+pwa_TTL_EXEMPT+"];"}); + } + + if (pwa_cache_all) { + // Generate array from the list of files, for the entire site. + + var dir = 'public'; + try { + fs.mkdirSync(dir); + } catch(e) { + if (e.code != 'EEXIST') throw e; + } + const path = './public/'; + cache = 'this.BASE_CACHE_FILES = ['; + files = fs.readdirSync(path, { recursive: true, withFileTypes: false }) + .forEach( + (file) => { + // check if is directory, if not then add the path/file + if (!fs.lstatSync(path+file).isDirectory()) { + // format output + item = "/"+file.replace(/index\.html$/i,'');// strip index.html from path + item = item.replace(/^\/sw(\.min)?\.js/i,'');// dont cache service worker + + // if formatted output is not empty line then append it to cache var + if (item != '') {// skip empty lines + cache = cache+"'"+item+"',"; + } + } + } + ); + cache = cache.slice(0, -1)+'];'// remove the last comma and close the array + } else if (pwa_BASE_CACHE_FILES) { + cache = 'this.BASE_CACHE_FILES = ['+pwa_BASE_CACHE_FILES+'];'; + } + + // update the BASE_CACHE_FILES variable in the sw.js service worker file + results = replace.sync({ + files: 'static/sw.js', + from: /this\.BASE_CACHE_FILES =.*/g, + to: cache, + countMatches: true, + }); + } + + if (bpath === '') {// abridge used directly + // These are truely static js files, so they should only need to be updated by abridge maintainer or contributors. + minify(['static/js/theme.js']); + minify(['static/js/theme_light.js']); + minify(['static/js/katex.min.js','static/js/mathtex-script-type.min.js','static/js/katex-auto-render.min.js','static/js/katexoptions.js'],'static/js/katexbundle.min.js'); + minify(['static/js/elasticlunr.min.js','static/js/search.js'],'static/js/search_elasticlunr.min.js'); + minify(['static/js/stork.js','static/js/stork_config.js'],'static/js/search_stork.min.js'); + minify(['static/js/tinysearch.js'],'static/js/search_tinysearch.min.js'); + minify(['static/js/prestyle.js','static/js/theme_button.js','static/js/email.js','static/js/codecopy.js','static/js/sw_load.js'],'static/js/abridge_nosearch.min.js'); + minify(['static/js/prestyle.js','static/js/theme_button.js','static/js/email.js','static/js/codecopy.js'],'static/js/abridge_nosearch_nopwa.min.js'); + minify(['static/js/sw_load.js']); + minify(['static/sw.js']); + } else if (pwa) { + minify(['static/js/sw_load.js']); + minify(['static/sw.js']); + } + + // if manifest.json is present, then minify it. + if (fs.existsSync('static/manifest.json')) { + let out; + try { + out = JSON.minify(fs.readFileSync('static/manifest.json', {encoding:"utf-8"})); + } catch(err) { + console.log(err); + } + fs.writeFileSync('static/manifest.min.json', out); + } + + abridge_bundle = bundle(bpath,js_prestyle,js_switcher,js_email_encode,js_copycode,search_library,index_format,uglyurls,false); + minify(abridge_bundle,'static/js/abridge_nopwa.min.js'); + + abridge_bundle = bundle(bpath,js_prestyle,js_switcher,js_email_encode,js_copycode,search_library,index_format,uglyurls,pwa); + minify(abridge_bundle,'static/js/abridge.min.js'); + + console.log('Zola Build to generate new integrity hashes for the previously minified files:'); + await execWrapper('zola build'); +} function bundle(bpath,js_prestyle,js_switcher,js_email_encode,js_copycode,search_library,index_format,uglyurls,pwa) { minify_files = []; @@ -124,141 +265,4 @@ function minify(fileA,outfile) { fs.writeFileSync(outfile, result.code); } -if (search_library === 'elasticlunr') { - if (fs.existsSync('content/static/stork_toml.md')) { - replace.sync({files: 'content/static/stork_toml.md', from: /draft.*=.*/g, to: "draft = true"}); - } - if (fs.existsSync('content/static/tinysearch_json.md')) { - replace.sync({files: 'content/static/tinysearch_json.md', from: /draft.*=.*/g, to: "draft = true"}); - } -} else if (search_library === 'tinysearch') { - if (!fs.existsSync('content/static/tinysearch_json.md')) {// 'content/static/tinysearch_json.md' file is missing, copy from abridge theme. - fs.copyFileSync(bpath+'content/static/tinysearch_json.md', 'content/static/tinysearch_json.md',fs.constants.COPYFILE_EXCL); - } - if (fs.existsSync('content/static/stork_toml.md')) { - replace.sync({files: 'content/static/stork_toml.md', from: /draft.*=.*/g, to: "draft = true"}); - } - if (fs.existsSync('content/static/tinysearch_json.md')) { - replace.sync({files: 'content/static/tinysearch_json.md', from: /draft.*=.*/g, to: "draft = false"}); - } - // zola build && mkdir -p tmp && tinysearch --optimize --path tmp public/data_tinysearch/index.html && rsync -avz tmp/*.wasm static/ && rm -rf tmp -} else if (search_library === 'stork') { - - if (!fs.existsSync('content/static/stork_toml.md')) {// 'content/static/stork_toml.md' file is missing, copy from abridge theme. - fs.copyFileSync(bpath+'content/static/stork_toml.md', 'content/static/stork_toml.md',fs.constants.COPYFILE_EXCL); - } - if (fs.existsSync('content/static/stork_toml.md')) { - replace.sync({files: 'content/static/stork_toml.md', from: /draft.*=.*/g, to: "draft = false"}); - } - if (fs.existsSync('content/static/tinysearch_json.md')) { - replace.sync({files: 'content/static/tinysearch_json.md', from: /draft.*=.*/g, to: "draft = true"}); - } - // zola build && stork build --input public/data_stork/index.html --output static/stork.st -} - -if (pwa) {// Update pwa settings, file list, and hashes. - // update from abridge theme. - fs.copyFileSync(bpath+'static/sw.js', 'static/sw.js'); - fs.copyFileSync(bpath+'static/js/sw_load.js', 'static/js/sw_load.js'); - // Update settings in PWA javascript file, using options parsed from config.toml. sw.min.js?v=3.10.0", "++" - if (fs.existsSync('static/js/sw_load.js')) { - sw_load_min = '.js?v='; - if (js_bundle) { - sw_load_min = '.min.js?v='; - } - replace.sync({files: 'static/js/sw_load.js', from: /sw.*v=.*/g, to: "sw"+sw_load_min+pwa_VER+"\","}); - } - if (fs.existsSync('static/sw.js')) { - replace.sync({files: 'static/sw.js', from: /NORM_TTL.*=.*/g, to: "NORM_TTL = "+pwa_NORM_TTL+";"}); - replace.sync({files: 'static/sw.js', from: /LONG_TTL.*=.*/g, to: "LONG_TTL = "+pwa_LONG_TTL+";"}); - replace.sync({files: 'static/sw.js', from: /TTL_NORM.*=.*/g, to: "TTL_NORM = ["+pwa_TTL_NORM+"];"}); - replace.sync({files: 'static/sw.js', from: /TTL_LONG.*=.*/g, to: "TTL_LONG = ["+pwa_TTL_LONG+"];"}); - replace.sync({files: 'static/sw.js', from: /TTL_EXEMPT.*=.*/g, to: "TTL_EXEMPT = ["+pwa_TTL_EXEMPT+"];"}); - } - - /// Generate hashes for PWA BASE_CACHE_FILES (disabled, seems unsupported, errors on install) - // buff = fs.readFileSync('public/abridge.css'); - // const h_abridge_css = crypto.createHash('sha256').update(buff).digest('hex').slice(0,20); - // buff = fs.readFileSync('public/js/abridge.min.js'); - // const h_abridge_js = crypto.createHash('sha256').update(buff).digest('hex').slice(0,20); - // buff = fs.readFileSync('public/js/abridge_nosearch.min.js'); - // const h_abridge_nosearch_js = crypto.createHash('sha256').update(buff).digest('hex').slice(0,20); - // buff = fs.readFileSync('public/js/katexbundle.min.js'); - // const h_katexbundle_js = crypto.createHash('sha256').update(buff).digest('hex').slice(0,20); - - if (pwa_cache_all) { - // Generate array from the list of files, for the entire site. - const path = './public/'; - cache = 'this.BASE_CACHE_FILES = ['; - files = fs.readdirSync(path, { recursive: true, withFileTypes: false }) - .forEach( - (file) => { - // check if is directory, if not then add the path/file - if (!fs.lstatSync(path+file).isDirectory()) { - // format output - item = "/"+file.replace(/index\.html$/i,'');// strip index.html from path - item = item.replace(/^\/sw(\.min)?\.js/i,'');// dont cache service worker - - // look for files ending in .js, if its not preceded by .min then skip it. - - // if formatted output is not empty line then append it to cache var - if (item != '') {// skip empty lines - cache = cache+"'"+item+"',"; - } - } - } - ); - cache = cache.slice(0, -1)+'];'// remove the last comma and close the array - } else if (pwa_BASE_CACHE_FILES) { - cache = 'this.BASE_CACHE_FILES = ['+pwa_BASE_CACHE_FILES+'];'; - } - - /// add cachebust hashes (disabled, seems unsupported, errors on install) - // cache = cache.replace(/abridge\.css/i,'abridge.css?h='+h_abridge_css); - // cache = cache.replace(/abridge\.min\.js/i,'abridge.min.js?h='+h_abridge_js); - // cache = cache.replace(/abridge_nosearch\.min\.js/i,'abridge_nosearch.min.js?h='+h_abridge_nosearch_js); - // cache = cache.replace(/katexbundle\.min\.js/i,'katexbundle.min.js?h='+h_katexbundle_js); - // console.log(cache); - - // update the BASE_CACHE_FILES variable in the sw.js service worker file - results = replace.sync({ - files: 'static/sw.js', - from: /this\.BASE_CACHE_FILES =.*/g, - to: cache, - countMatches: true, - }); -} - -if (bpath === '') {// abridge used directly - // These are truely static js files, so they should only need to be updated by abridge maintainer or contributors. - minify(['static/js/theme.js']); - minify(['static/js/theme_light.js']); - minify(['static/js/katex.min.js','static/js/mathtex-script-type.min.js','static/js/katex-auto-render.min.js','static/js/katexoptions.js'],'static/js/katexbundle.min.js'); - minify(['static/js/elasticlunr.min.js','static/js/search.js'],'static/js/search_elasticlunr.min.js'); - minify(['static/js/stork.js','static/js/stork_config.js'],'static/js/search_stork.min.js'); - minify(['static/js/tinysearch.js'],'static/js/search_tinysearch.min.js'); - minify(['static/js/prestyle.js','static/js/theme_button.js','static/js/email.js','static/js/codecopy.js','static/js/sw_load.js'],'static/js/abridge_nosearch.min.js'); - minify(['static/js/prestyle.js','static/js/theme_button.js','static/js/email.js','static/js/codecopy.js'],'static/js/abridge_nosearch_nopwa.min.js'); - minify(['static/js/sw_load.js']); - minify(['static/sw.js']); -} else if (pwa) { - minify(['static/js/sw_load.js']); - minify(['static/sw.js']); -} - -// if manifest.json is present, then minify it. -if (fs.existsSync('static/manifest.json')) { - let out; - try { - out = JSON.minify(fs.readFileSync('static/manifest.json', {encoding:"utf-8"})); - } catch(err) { - console.log(err); - } - fs.writeFileSync('static/manifest.min.json', out); -} - -abridge_bundle = bundle(bpath,js_prestyle,js_switcher,js_email_encode,js_copycode,search_library,index_format,uglyurls,false); -minify(abridge_bundle,'static/js/abridge_nopwa.min.js'); - -abridge_bundle = bundle(bpath,js_prestyle,js_switcher,js_email_encode,js_copycode,search_library,index_format,uglyurls,pwa); -minify(abridge_bundle,'static/js/abridge.min.js'); +abridge(); -- cgit v1.3.1