Skip to content

Commit

Permalink
Refactoring source code to make easy testing
Browse files Browse the repository at this point in the history
  • Loading branch information
@pavel-karatsiuba
pavel-karatsiuba committed Mar 8, 2023
1 parent eb6cfd8 commit 94c0000
Show file tree
Hide file tree
Showing 16 changed files with 1,168 additions and 1,190 deletions.
19 changes: 0 additions & 19 deletions lib/classes.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,25 +9,6 @@ import { log } from './logger.js'
import { Simulation } from './types.js'
import { getIdAndLanguage } from './common.js'

export class Base64Entity {
public data: string
public mimeType: string

constructor(encoded) {
const decoded = encoded.match(/^data:([A-Za-z-+\/]+);base64,(.+)$/)
this.mimeType = decoded && decoded[1]
this.data = decoded && decoded[2]
}

isEmpty(): boolean {
return !this.data || this.data.length === 0 || !this.mimeType
}

isImage(): boolean {
return this.mimeType.split('/')[0] === 'image'
}
}

export class SimulationsList {
public items: Simulation[] = []
private readonly language: string
Expand Down
1,455 changes: 698 additions & 757 deletions package-lock.json
View file

Large diffs are not rendered by default.

10 changes: 5 additions & 5 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
"imagemin-svgo": "^10.0.1",
"iso-639-1": "^2.1.15",
"iso-639-3": "^3.0.1",
"jest": "^29.3.1",
"jest": "^29.5.0",
"js-yaml": "^4.1.0",
"md5": "^2.3.0",
"ncp": "^2.0.0",
Expand All @@ -50,7 +50,7 @@
"snyk": "^1.1066.0",
"sweetalert2": "^11.6.15",
"tiny-async-pool": "^1.3.0",
"ts-jest": "^29.0.3",
"ts-jest": "^29.0.5",
"ts-node": "^10.9.1",
"typescript": "^4.9.3",
"winston": "^3.8.2",
Expand All @@ -68,10 +68,10 @@
},
"scripts": {
"setup": "ts-node-esm -- steps/setup.ts",
"get": "ts-node-esm -- steps/get.ts",
"transform": "ts-node-esm -- steps/transform.ts",
"get": "ts-node-esm -- steps/get/index.ts",
"transform": "ts-node-esm -- steps/transform/index.ts",
"export-prebuild": "npm run build && node_modules/browserify/bin/cmd.js res/js/index.js -t babelify -o res/js/dist.js",
"export": "npm run export-prebuild && ts-node-esm -- steps/export.ts",
"export": "npm run export-prebuild && ts-node-esm -- steps/export/index.ts",
"start": "npm run setup && npm run get && npm run transform && npm run export",
"test": "npm run export-prebuild && node --experimental-vm-modules node_modules/jest/bin/jest.js --verbose --runInBand --forceExit",
"build": "tsc",
Expand Down
153 changes: 21 additions & 132 deletions steps/export.ts → steps/export/converters.ts
View file
Original file line number Diff line number Diff line change
@@ -1,118 +1,26 @@
import * as fs from 'fs'
import glob from 'glob'
import * as path from 'path'
import yargs from 'yargs'
import { promisify } from 'util'
import rimraf from 'rimraf'
import * as dotenv from 'dotenv'
import * as cheerio from 'cheerio'
import { ZimArticle, ZimCreator } from '@openzim/libzim'

import { log } from '../lib/logger.js'
import { Target } from '../lib/types.js'
import welcome from '../lib/welcome.js'
import { Catalog } from '../lib/classes.js'
import { getISO6393 } from '../lib/common.js'
import { log } from '../../lib/logger.js'
import { Target } from '../../lib/types.js'
import { Catalog } from '../../lib/classes.js'
import { getISO6393 } from '../../lib/common.js'
import { Presets, SingleBar } from 'cli-progress'
import { hideBin } from 'yargs/helpers'
import { fileURLToPath } from 'url'
import { catalogJs } from '../res/templates/catalog.js'
import { catalogJs } from '../../res/templates/catalog.js'
import Banana from 'banana-i18n'
import { iso6393To1 } from 'iso-639-3'

dotenv.config()
import { options, rimrafPromised, extractResources, loadLanguages, getNamespaceByExt } from './utils.js'
import { fileURLToPath } from 'url'

const __filename = fileURLToPath(import.meta.url)
const __dirname = path.dirname(__filename)

const { argv } = yargs(hideBin(process.argv)).array('includeLanguages').array('excludeLanguages').boolean('mulOnly')

const langsFile = await fs.promises.readFile(path.join(__dirname, '../state/get/languages.json'))
const langs = JSON.parse(langsFile.toString())

const languages = Object.entries(langs).reduce((acc, [key, value]) => {
if (argv.includeLanguages && !((argv.includeLanguages as string[]) || []).includes(key)) return acc
if (argv.excludeLanguages && ((argv.excludeLanguages as string[]) || []).includes(key)) return acc // noinspection RedundantIfStatementJS
return { ...acc, [key]: value }
}, {})

const catalogsDir = 'state/get/catalogs'
const inDir = 'state/transform/'
const outDir = 'state/export/'
const resDir = 'res/'

const verbose = process.env.PHET_VERBOSE_ERRORS !== undefined ? process.env.PHET_VERBOSE_ERRORS === 'true' : false

const rimrafPromised = promisify(rimraf)

const namespaces = {
// ico: '-',
js: '-',
css: '-',
svg: 'I',
png: 'I',
jpg: 'I',
jpeg: 'I',
html: 'A',
}

const getNamespaceByExt = (ext: string): string => namespaces[ext] || '-'

const getKiwixPrefix = (ext: string): string => `../${getNamespaceByExt(ext)}/`

const addKiwixPrefixes = function addKiwixPrefixes(file) {
const resources = file.match(/[0-9a-f]{32}\.(svg|jpg|jpeg|png|js)/g) || []
return resources.reduce((file, resName) => {
const ext = path.extname(resName).slice(1)
return file.replace(resName, `${getKiwixPrefix(ext)}${resName}`)
}, file)
}
const languages = await loadLanguages()

const extractResources = async (target, targetDir: string): Promise<void> => {
const bar = new SingleBar({}, Presets.shades_classic)
const files = glob.sync(`${inDir}/*_@(${target.languages.join('|')}).html`, {})
bar.start(files.length, 0)

for (const file of files) {
try {
let html = await fs.promises.readFile(file, 'utf8')
const $ = cheerio.load(html)

const filesToExtract = $('[src]')
.toArray()
.map((a) => $(a).attr('src'))
await fs.promises.copyFile(`${file.split('_')[0]}.png`, `${targetDir}${path.basename(file).split('_')[0]}.png`)

await Promise.all(
filesToExtract.map(async (fileName) => {
if (fileName.length > 40 || fileName.search(/this\.image/) !== -1) return
const ext = path.extname(fileName).slice(1)
html = html.replace(fileName, `${getKiwixPrefix(ext)}${fileName}`)

let file = await fs.promises.readFile(`${inDir}${fileName}`, 'utf8')
file = addKiwixPrefixes(file)
return fs.promises.writeFile(`${targetDir}${path.basename(fileName)}`, file, 'utf8')
}),
)
await fs.promises.writeFile(`${targetDir}${path.basename(file)}`, html, 'utf8')
} catch (e) {
if (verbose) {
log.error(`Failed to extract resources from: ${file}`)
log.error(e)
} else {
log.warn(`Unable to extract resources from: ${file}. Skipping it.`)
}
} finally {
bar.increment()
if (!process.stdout.isTTY) log.info(` + ${path.basename(file)}`)
}
}
bar.stop()
}

const loadTranslations = async (locale: string) => {
export const loadTranslations = async (locale: string) => {
try {
const translations = await fs.promises.readFile(path.join(__dirname, `../res/js/i18n/${locale}.json`))
const translations = await fs.promises.readFile(path.join(__dirname, `../../res/js/i18n/${locale}.json`))
return JSON.parse(translations.toString())
} catch (err) {
if (err?.name.includes('SyntaxError')) {
Expand All @@ -122,29 +30,29 @@ const loadTranslations = async (locale: string) => {
}
}

const exportTarget = async (target: Target, bananaI18n: Banana) => {
const targetDir = `${outDir}${target.output}/`
export const exportTarget = async (target: Target, bananaI18n: Banana) => {
const targetDir = `${options.outDir}${target.output}/`

await rimrafPromised(targetDir)
await fs.promises.mkdir(targetDir)
await extractResources(target, targetDir)

const catalog = new Catalog({ target, languages, catalogsDir })
const catalog = new Catalog({ target, languages, catalogsDir: options.catalogsDir })
await catalog.init()
if (catalog.isEmpty()) {
log.info(`Skipping ${target.output}.zim (empty)`)
return
}

// Generate index file
await fs.promises.copyFile(resDir + 'template.html', targetDir + 'index.html')
await fs.promises.copyFile(options.resDir + 'template.html', targetDir + 'index.html')

// Generate catalog JS
await fs.promises.writeFile(targetDir + 'catalog.js', catalogJs(catalog, target.output), 'utf8')

await Promise.all(
glob
.sync(`${resDir}/**/*`, {
.sync(`${options.resDir}/**/*`, {
ignore: ['*/templates/*', '*.ts', '*/template.html'],
nodir: true,
})
Expand Down Expand Up @@ -213,11 +121,11 @@ const exportTarget = async (target: Target, bananaI18n: Banana) => {
log.info('Created.')
}

const convertTranslations = async () => {
export const convertTranslations = async () => {
log.info('Converting translations to JS')

const bar = new SingleBar({}, Presets.shades_classic)
const translationsFolder = path.join(__dirname, '../res/js/i18n/')
const translationsFolder = path.join(__dirname, '../../res/js/i18n/')
const files = glob.sync(`${translationsFolder}**/*.json`, {})
bar.start(files.length, 0)

Expand All @@ -227,7 +135,7 @@ const convertTranslations = async () => {
const jsTranslations = `window.phetTranslations = ${translations};`
await fs.promises.writeFile(`${file}.js`, jsTranslations, 'utf8')
} catch (e) {
if (verbose) {
if (options.verbose) {
log.error(`Failed to extract translations from: ${file}`)
log.error(e)
} else {
Expand All @@ -242,7 +150,7 @@ const convertTranslations = async () => {
log.info('Converted.')
}

const exportData = async () => {
export const prepareTargets = () => {
const now = new Date()
const datePostfix = `${now.getUTCFullYear()}-${(now.getUTCMonth() + 1).toString().padStart(2, '0')}`

Expand All @@ -256,7 +164,7 @@ const exportData = async () => {
}),
},
]
if (!argv.mulOnly) {
if (!options.mulOnly) {
for (const lang of Object.keys(languages)) {
targets.push({
output: `phet_${lang.toLowerCase().replace('_', '-')}_${datePostfix}`,
Expand All @@ -266,24 +174,5 @@ const exportData = async () => {
}
}

await convertTranslations()

const defaultTranslations = await loadTranslations('en')
const banana = new Banana('en', { messages: defaultTranslations })
for (const target of targets) {
await exportTarget(target, banana)
}
return targets
}

;(async () => {
welcome('export')
await exportData()
log.info('Done.')
})().catch((err: Error) => {
if (err && err.message) {
log.error(err.message)
} else {
log.error(`An unidentified error occured ${err}`)
}
yargs(hideBin(process.argv)).exit(1, err)
})
26 changes: 26 additions & 0 deletions steps/export/index.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import yargs from 'yargs'
import { log } from '../../lib/logger.js'
import welcome from '../../lib/welcome.js'
import { hideBin } from 'yargs/helpers'
import Banana from 'banana-i18n'
import { prepareTargets, convertTranslations, loadTranslations, exportTarget } from './converters.js'
;(async () => {
welcome('export')
const targets = prepareTargets()

await convertTranslations()

const defaultTranslations = await loadTranslations('en')
const banana = new Banana('en', { messages: defaultTranslations })
for (const target of targets) {
await exportTarget(target, banana)
}
log.info('Done.')
})().catch((err: Error) => {
if (err && err.message) {
log.error(err.message)
} else {
log.error(`An unidentified error occured ${err}`)
}
yargs(hideBin(process.argv)).exit(1, err)
})
Loading

0 comments on commit 94c0000

Please sign in to comment.