Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactoring source code to make easy testing #209

Merged
merged 1 commit into from
Mar 12, 2023
Merged

Refactoring source code to make easy testing #209

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 0 additions & 19 deletions lib/classes.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,25 +9,6 @@ import { log } from './logger.js'
import { Simulation } from './types.js'
import { getIdAndLanguage } from './common.js'

export class Base64Entity {
public data: string
public mimeType: string

constructor(encoded) {
const decoded = encoded.match(/^data:([A-Za-z-+\/]+);base64,(.+)$/)
this.mimeType = decoded && decoded[1]
this.data = decoded && decoded[2]
}

isEmpty(): boolean {
return !this.data || this.data.length === 0 || !this.mimeType
}

isImage(): boolean {
return this.mimeType.split('/')[0] === 'image'
}
}

export class SimulationsList {
public items: Simulation[] = []
private readonly language: string
Expand Down
1,455 changes: 698 additions & 757 deletions package-lock.json
View file
Open in desktop

Large diffs are not rendered by default.

10 changes: 5 additions & 5 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
"imagemin-svgo": "^10.0.1",
"iso-639-1": "^2.1.15",
"iso-639-3": "^3.0.1",
"jest": "^29.3.1",
"jest": "^29.5.0",
"js-yaml": "^4.1.0",
"md5": "^2.3.0",
"ncp": "^2.0.0",
Expand All @@ -50,7 +50,7 @@
"snyk": "^1.1066.0",
"sweetalert2": "^11.6.15",
"tiny-async-pool": "^1.3.0",
"ts-jest": "^29.0.3",
"ts-jest": "^29.0.5",
"ts-node": "^10.9.1",
"typescript": "^4.9.3",
"winston": "^3.8.2",
Expand All @@ -68,10 +68,10 @@
},
"scripts": {
"setup": "ts-node-esm -- steps/setup.ts",
"get": "ts-node-esm -- steps/get.ts",
"transform": "ts-node-esm -- steps/transform.ts",
"get": "ts-node-esm -- steps/get/index.ts",
"transform": "ts-node-esm -- steps/transform/index.ts",
"export-prebuild": "npm run build && node_modules/browserify/bin/cmd.js res/js/index.js -t babelify -o res/js/dist.js",
"export": "npm run export-prebuild && ts-node-esm -- steps/export.ts",
"export": "npm run export-prebuild && ts-node-esm -- steps/export/index.ts",
"start": "npm run setup && npm run get && npm run transform && npm run export",
"test": "npm run export-prebuild && node --experimental-vm-modules node_modules/jest/bin/jest.js --verbose --runInBand --forceExit",
"build": "tsc",
Expand Down
153 changes: 21 additions & 132 deletions steps/export.ts → steps/export/converters.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,118 +1,26 @@
import * as fs from 'fs'
import glob from 'glob'
import * as path from 'path'
import yargs from 'yargs'
import { promisify } from 'util'
import rimraf from 'rimraf'
import * as dotenv from 'dotenv'
import * as cheerio from 'cheerio'
import { ZimArticle, ZimCreator } from '@openzim/libzim'

import { log } from '../lib/logger.js'
import { Target } from '../lib/types.js'
import welcome from '../lib/welcome.js'
import { Catalog } from '../lib/classes.js'
import { getISO6393 } from '../lib/common.js'
import { log } from '../../lib/logger.js'
import { Target } from '../../lib/types.js'
import { Catalog } from '../../lib/classes.js'
import { getISO6393 } from '../../lib/common.js'
import { Presets, SingleBar } from 'cli-progress'
import { hideBin } from 'yargs/helpers'
import { fileURLToPath } from 'url'
import { catalogJs } from '../res/templates/catalog.js'
import { catalogJs } from '../../res/templates/catalog.js'
import Banana from 'banana-i18n'
import { iso6393To1 } from 'iso-639-3'

dotenv.config()
import { options, rimrafPromised, extractResources, loadLanguages, getNamespaceByExt } from './utils.js'
import { fileURLToPath } from 'url'

const __filename = fileURLToPath(import.meta.url)
const __dirname = path.dirname(__filename)

const { argv } = yargs(hideBin(process.argv)).array('includeLanguages').array('excludeLanguages').boolean('mulOnly')

const langsFile = await fs.promises.readFile(path.join(__dirname, '../state/get/languages.json'))
const langs = JSON.parse(langsFile.toString())

const languages = Object.entries(langs).reduce((acc, [key, value]) => {
if (argv.includeLanguages && !((argv.includeLanguages as string[]) || []).includes(key)) return acc
if (argv.excludeLanguages && ((argv.excludeLanguages as string[]) || []).includes(key)) return acc // noinspection RedundantIfStatementJS
return { ...acc, [key]: value }
}, {})

const catalogsDir = 'state/get/catalogs'
const inDir = 'state/transform/'
const outDir = 'state/export/'
const resDir = 'res/'

const verbose = process.env.PHET_VERBOSE_ERRORS !== undefined ? process.env.PHET_VERBOSE_ERRORS === 'true' : false

const rimrafPromised = promisify(rimraf)

const namespaces = {
// ico: '-',
js: '-',
css: '-',
svg: 'I',
png: 'I',
jpg: 'I',
jpeg: 'I',
html: 'A',
}

const getNamespaceByExt = (ext: string): string => namespaces[ext] || '-'

const getKiwixPrefix = (ext: string): string => `../${getNamespaceByExt(ext)}/`

const addKiwixPrefixes = function addKiwixPrefixes(file) {
const resources = file.match(/[0-9a-f]{32}\.(svg|jpg|jpeg|png|js)/g) || []
return resources.reduce((file, resName) => {
const ext = path.extname(resName).slice(1)
return file.replace(resName, `${getKiwixPrefix(ext)}${resName}`)
}, file)
}
const languages = await loadLanguages()

const extractResources = async (target, targetDir: string): Promise<void> => {
const bar = new SingleBar({}, Presets.shades_classic)
const files = glob.sync(`${inDir}/*_@(${target.languages.join('|')}).html`, {})
bar.start(files.length, 0)

for (const file of files) {
try {
let html = await fs.promises.readFile(file, 'utf8')
const $ = cheerio.load(html)

const filesToExtract = $('[src]')
.toArray()
.map((a) => $(a).attr('src'))
await fs.promises.copyFile(`${file.split('_')[0]}.png`, `${targetDir}${path.basename(file).split('_')[0]}.png`)

await Promise.all(
filesToExtract.map(async (fileName) => {
if (fileName.length > 40 || fileName.search(/this\.image/) !== -1) return
const ext = path.extname(fileName).slice(1)
html = html.replace(fileName, `${getKiwixPrefix(ext)}${fileName}`)

let file = await fs.promises.readFile(`${inDir}${fileName}`, 'utf8')
file = addKiwixPrefixes(file)
return fs.promises.writeFile(`${targetDir}${path.basename(fileName)}`, file, 'utf8')
}),
)
await fs.promises.writeFile(`${targetDir}${path.basename(file)}`, html, 'utf8')
} catch (e) {
if (verbose) {
log.error(`Failed to extract resources from: ${file}`)
log.error(e)
} else {
log.warn(`Unable to extract resources from: ${file}. Skipping it.`)
}
} finally {
bar.increment()
if (!process.stdout.isTTY) log.info(` + ${path.basename(file)}`)
}
}
bar.stop()
}

const loadTranslations = async (locale: string) => {
export const loadTranslations = async (locale: string) => {
try {
const translations = await fs.promises.readFile(path.join(__dirname, `../res/js/i18n/${locale}.json`))
const translations = await fs.promises.readFile(path.join(__dirname, `../../res/js/i18n/${locale}.json`))
return JSON.parse(translations.toString())
} catch (err) {
if (err?.name.includes('SyntaxError')) {
Expand All @@ -122,29 +30,29 @@ const loadTranslations = async (locale: string) => {
}
}

const exportTarget = async (target: Target, bananaI18n: Banana) => {
const targetDir = `${outDir}${target.output}/`
export const exportTarget = async (target: Target, bananaI18n: Banana) => {
const targetDir = `${options.outDir}${target.output}/`

await rimrafPromised(targetDir)
await fs.promises.mkdir(targetDir)
await extractResources(target, targetDir)

const catalog = new Catalog({ target, languages, catalogsDir })
const catalog = new Catalog({ target, languages, catalogsDir: options.catalogsDir })
await catalog.init()
if (catalog.isEmpty()) {
log.info(`Skipping ${target.output}.zim (empty)`)
return
}

// Generate index file
await fs.promises.copyFile(resDir + 'template.html', targetDir + 'index.html')
await fs.promises.copyFile(options.resDir + 'template.html', targetDir + 'index.html')

// Generate catalog JS
await fs.promises.writeFile(targetDir + 'catalog.js', catalogJs(catalog, target.output), 'utf8')

await Promise.all(
glob
.sync(`${resDir}/**/*`, {
.sync(`${options.resDir}/**/*`, {
ignore: ['*/templates/*', '*.ts', '*/template.html'],
nodir: true,
})
Expand Down Expand Up @@ -213,11 +121,11 @@ const exportTarget = async (target: Target, bananaI18n: Banana) => {
log.info('Created.')
}

const convertTranslations = async () => {
export const convertTranslations = async () => {
log.info('Converting translations to JS')

const bar = new SingleBar({}, Presets.shades_classic)
const translationsFolder = path.join(__dirname, '../res/js/i18n/')
const translationsFolder = path.join(__dirname, '../../res/js/i18n/')
const files = glob.sync(`${translationsFolder}**/*.json`, {})
bar.start(files.length, 0)

Expand All @@ -227,7 +135,7 @@ const convertTranslations = async () => {
const jsTranslations = `window.phetTranslations = ${translations};`
await fs.promises.writeFile(`${file}.js`, jsTranslations, 'utf8')
} catch (e) {
if (verbose) {
if (options.verbose) {
log.error(`Failed to extract translations from: ${file}`)
log.error(e)
} else {
Expand All @@ -242,7 +150,7 @@ const convertTranslations = async () => {
log.info('Converted.')
}

const exportData = async () => {
export const prepareTargets = () => {
const now = new Date()
const datePostfix = `${now.getUTCFullYear()}-${(now.getUTCMonth() + 1).toString().padStart(2, '0')}`

Expand All @@ -256,7 +164,7 @@ const exportData = async () => {
}),
},
]
if (!argv.mulOnly) {
if (!options.mulOnly) {
for (const lang of Object.keys(languages)) {
targets.push({
output: `phet_${lang.toLowerCase().replace('_', '-')}_all_${datePostfix}`,
Expand All @@ -266,24 +174,5 @@ const exportData = async () => {
}
}

await convertTranslations()

const defaultTranslations = await loadTranslations('en')
const banana = new Banana('en', { messages: defaultTranslations })
for (const target of targets) {
await exportTarget(target, banana)
}
return targets
}

;(async () => {
welcome('export')
await exportData()
log.info('Done.')
})().catch((err: Error) => {
if (err && err.message) {
log.error(err.message)
} else {
log.error(`An unidentified error occured ${err}`)
}
yargs(hideBin(process.argv)).exit(1, err)
})
26 changes: 26 additions & 0 deletions steps/export/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import yargs from 'yargs'
import { log } from '../../lib/logger.js'
import welcome from '../../lib/welcome.js'
import { hideBin } from 'yargs/helpers'
import Banana from 'banana-i18n'
import { prepareTargets, convertTranslations, loadTranslations, exportTarget } from './converters.js'
;(async () => {
welcome('export')
const targets = prepareTargets()

await convertTranslations()

const defaultTranslations = await loadTranslations('en')
const banana = new Banana('en', { messages: defaultTranslations })
for (const target of targets) {
await exportTarget(target, banana)
}
log.info('Done.')
})().catch((err: Error) => {
if (err && err.message) {
log.error(err.message)
} else {
log.error(`An unidentified error occured ${err}`)
}
yargs(hideBin(process.argv)).exit(1, err)
})
Loading