diff --git a/.gitignore b/.gitignore index 8047755..dad9c10 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ _cache/ +build/ bin/ node_modules/ .DS_Store public/langs/* +profile_result.txt diff --git a/__tests__/mappings.js b/__tests__/mappings/mappings.test.js similarity index 85% rename from __tests__/mappings.js rename to __tests__/mappings/mappings.test.js index f5a0d56..f991d58 100644 --- a/__tests__/mappings.js +++ b/__tests__/mappings/mappings.test.js @@ -8,7 +8,8 @@ describe('mappings', () => { const csvFileName = `${lang}.csv` describe(csvFileName, () => { const encoding = { encoding: 'utf-8' }; - const csv = fs.readFileSync(`./mappings/${csvFileName}`, encoding) + const csvPath = __dirname + `/../../mappings/${csvFileName}`; + const csv = fs.readFileSync(csvPath, encoding) .split('\n') // Make array of lines .slice(1, -1); // Cut off the header and newline it(`should have a definition for all rules`, () => { @@ -18,12 +19,12 @@ describe('mappings', () => { // Extract types from tree_matcher_specs.js file for this lang const treeMatcherTypes = require( - `../language_configs/${lang}.tree_matcher_specs.js` + `../../language_configs/${lang}.tree_matcher_specs.js` ).reduce((arr, spec) => arr.concat(spec.provides_tags), []); // Now create a sorted array of rules in the config files, and // the tree matcher specs file, then compare the two - Object.keys(require(`../language_configs/${lang}.rules.js`)) + Object.keys(require(`../../language_configs/${lang}.rules.js`)) .concat(treeMatcherTypes) .sort((a, b) => a.localeCompare(b)) .forEach((rule, i) => expect(rule).toEqual(tokens[i])); diff --git a/__tests__/transformers/collapse.test.js b/__tests__/transformers/collapse.test.js new file mode 100644 index 0000000..4c04dc4 --- /dev/null +++ b/__tests__/transformers/collapse.test.js @@ -0,0 +1,63 @@ +describe('collapse.js', () => { + const collapse = require('../../src/transformers/collapse.js'); + const { makeNode } = require('../../src/utils/utils.js'); + + const start = 0; + const end = 5; + const noDetail = []; + + const bottomNode = makeNode('bottom_node', start, end, noDetail, []); + const middleNode = makeNode('middle_node', start, end, noDetail, [ bottomNode ]); + const topNode = makeNode('top_node', start, end, noDetail, [ middleNode ]); + + it(`is a function`, () => { + expect(typeof(collapse)).toEqual('function'); + }); + + it(`recursively removes nodes from the tree if those nodes: + • are flagged as collapsible in the runtime config, and + • have single children who occupy the exact same range`, () => { + const langRuntimeConfig1 = { + 'rules': { + 'top_node': { 'collapse': true }, + 'middle_node': { 'collapse': true }, + 'bottom_node': { 'collapse': false }, + }, + }; + expect(collapse(langRuntimeConfig1, topNode)).toBe(bottomNode); + + const langRuntimeConfig2 = { + 'rules': { + 'top_node': { 'collapse': true }, + 'middle_node': { 'collapse': false }, + 'bottom_node': { 'collapse': true } + }, + }; + expect(collapse(langRuntimeConfig2, topNode)).toBe(middleNode); + }); + + it(`does NOT remove nodes from the tree if they are NOT flagged as collapsible`, + () => { + const langRuntimeConfig = { + 'rules': { + 'top_node': {}, + 'middle_node': { 'collapse': true }, + 'bottom_node': { 'collapse': true }, + }, + }; + expect(collapse(langRuntimeConfig, topNode)).toBe(topNode); + }); + + it(`does NOT remove nodes from the tree if they ARE flagged as collapsible, + but whose children do NOT occupt identical ranges`, () => { + const childNode = makeNode('child', 0, 3, noDetail, []); + const parentNode = makeNode('parent', 0, 5, noDetail, [ childNode ]); + const langRuntimeConfig = { + 'rules': { + 'parent': { 'collapse': true }, + 'child': { 'collapse': true }, + }, + }; + expect(collapse(langRuntimeConfig, parentNode)).toBe(parentNode); + }); +}); diff --git a/__tests__/transformers/simplify_node.test.js b/__tests__/transformers/simplify_node.test.js new file mode 100644 index 0000000..987a18a --- /dev/null +++ b/__tests__/transformers/simplify_node.test.js @@ -0,0 +1,63 @@ +describe(`simplify_node.js`, () => { + const simplify_node = require('../../src/transformers/simplify_node.js'); + const { + makeAntlrNode, + makeAntlrTerminal, + makeNode, + makeTerminal, + } = require('../../src/utils/utils.js'); + const lang_runtime_config = { + symbol_name_map: [ + '.BLAH', + '.FOOBAR', + '.QUXBAZ', + ], + rule_name_map: [ + 'floop_type', + 'groop_loop', + ], + }; + + it(`is a function`, () => { + expect(typeof(simplify_node)).toEqual('function'); + }); + + it(`recursively extracts the following properties from non-terminal + input nodes: + • type + • begin + • end + • tags + • children`, () => { + // Different forms for starts and stops are intentionally + // used here, as both can be found in real ANTLR nodes. + const input_node = ( + makeAntlrNode(0, { start: 0 }, { stop: 5 }, [ + makeAntlrNode(1, { start: 0, stop: 4 }, undefined, undefined), + ])); + + const { rule_name_map } = lang_runtime_config; + const expected = ( + makeNode(rule_name_map[0], 0, 5 + 1, [], [ + makeNode(rule_name_map[1], 0, 4 + 1, [], []), + ])); + expect(simplify_node(lang_runtime_config, input_node)).toEqual(expected); + }); + + it(`extracts the following properties from terminal input nodes: + • type + • begin + • end + • tags + • children + • text`, () => { + const input_terminal = ( + makeAntlrTerminal(0, 0, 5, 'foobar') + ); + const { symbol_name_map } = lang_runtime_config; + const expected = ( + makeTerminal(symbol_name_map[0 + 2], 0, 6, 'foobar', []) + ); + expect(simplify_node(lang_runtime_config, input_terminal)).toEqual(expected); + }); +}); diff --git a/__tests__/tree/code/rename_random.sh b/__tests__/tree/code/rename_random.sh new file mode 100755 index 0000000..f521155 --- /dev/null +++ b/__tests__/tree/code/rename_random.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +mv "$1" "snippet.`node --print "Math.random().toString(36).substr(2, 8)"`.${1##*.}" diff --git a/__tests__/tree/code/snippet.hixfx981.rb b/__tests__/tree/code/snippet.hixfx981.rb new file mode 100644 index 0000000..a198637 --- /dev/null +++ b/__tests__/tree/code/snippet.hixfx981.rb @@ -0,0 +1,24 @@ +class Unit + # A list of all the units we know. + @@units = { } + + # Access to @@units. + def Unit.exists(n) + return @@units.has_key?(n) + end + def Unit.named(n) + return @@units[n] + end + + # If this were Java, I'd define an abstract function isbase() which tells + # if this object is a BaseUnit or not. + def initialize(name) + @name = name + @@units[name] = self + @@units[name + 's'] = self + end + attr_reader :name + def alias(*names) + names.each { |n| @@units[n] = self } + end +end diff --git a/__tests__/tree/code/snippet.k17eu4f2.java b/__tests__/tree/code/snippet.k17eu4f2.java new file mode 100644 index 0000000..5dedd6b --- /dev/null +++ b/__tests__/tree/code/snippet.k17eu4f2.java @@ -0,0 +1,5 @@ +public class HelloWorldApp { + public static void main(String[] args) { + System.out.println("Hello World!"); + } +} diff --git a/__tests__/tree/code/snippet.kt29xnfw.py b/__tests__/tree/code/snippet.kt29xnfw.py new file mode 100644 index 0000000..5696edc --- /dev/null +++ b/__tests__/tree/code/snippet.kt29xnfw.py @@ -0,0 +1,7 @@ +import sys +try: + total = sum(int(arg) for arg in sys.argv[1:]) + print('sum =', total) +except ValueError: + print('Please supply integer arguments') + diff --git a/__tests__/tree/code/snippet.voc84cjo.py b/__tests__/tree/code/snippet.voc84cjo.py new file mode 100644 index 0000000..ec7780c --- /dev/null +++ b/__tests__/tree/code/snippet.voc84cjo.py @@ -0,0 +1 @@ +print('Hello, world!') diff --git a/__tests__/tree/compare.test.js b/__tests__/tree/compare.test.js new file mode 100644 index 0000000..0045bd1 --- /dev/null +++ b/__tests__/tree/compare.test.js @@ -0,0 +1,163 @@ +const path = require('path'); +const fs = require('fs-promise'); +const antlr = require('antlr4'); +const expect_error = require('../../src/utils/expect_error.js'); +const run_antlr = require('../../src/utils/run_antlr.js'); +const run_java = require('../../src/utils/run_java.js'); +const webpack = require('webpack'); + +const config = require('../../config'); + +const prepareJava = async function(lang_compile_config, lang_runtime_config) { + return await run_antlr(lang_compile_config, lang_runtime_config, 'Java'); +} + +const genTreeViaJava = async function(lang_runtime_config, antlr_result, code) { + const { + build_dir + } = antlr_result; + + const java_sources = [ + lang_runtime_config.language + 'Lexer.java', + lang_runtime_config.language + 'Parser.java', + ]; + + // Add CLASSPATH to environment + const environment = Object.create(process.env); + const classpath = environment.CLASSPATH ? environment.CLASSPATH.split(':') : []; + classpath.unshift(path.resolve(__dirname, '../../bin/antlr-4.6-complete.jar')); + classpath.unshift('.'); + environment.CLASSPATH = classpath.join(':'); + + const result = await run_java( + java_sources, + 'org.antlr.v4.gui.TestRig', + [lang_runtime_config.language, lang_runtime_config.entry_rule, '-tree'], + { + 'cwd': build_dir, + 'env': environment, + //'stdio': ['pipe', 'pipe', process.stderr], + }, + code + ); + + if (result.code) { + throw result.stderr.toString(); + } else { + return result.stdout.toString(); + } +}; + +const prepareJs = async function(lang_compile_config, lang_runtime_config) { + const output_path = path.resolve(config.build_path, 'output'); + + const webpack_config = await require('../../webpack.config.js')({ + 'langs': lang_runtime_config.language, + 'optimize': 0, + 'libraryTarget': 'commonjs2', + 'outputPath': output_path, + 'enable_debug': false, + }); + + if (webpack_config.length !== 1) { + throw new Error('Unexpected webpack output'); + } + + const compiler = webpack(webpack_config); + + await new Promise(function(resolve, reject) { + compiler.run(function(err, stats) { + if (err) { + reject(err); + } else { + resolve(stats); + } + }); + }); + + return require(path.resolve(output_path, webpack_config[0].output.filename)); +} + +const genTreeViaJs = async function(lang_runtime_config, js_parser, code) { + const tree = js_parser(code, function(err) { + throw err; + }, { + 'return_antlr_tree': true, + }); + + return tree.toStringTree(tree.parser.ruleNames); +}; + + +let prepare_cache = {}; + +const compare = async function(language_name, code_file) { + const lang_compile_config = require(`../../language_configs/${language_name}.compile.js`); + const lang_runtime_config = require(`../../language_configs/${language_name}.runtime.js`); + + if (typeof prepare_cache[language_name] === 'undefined') { + prepare_cache[language_name] = Promise.all([ + prepareJava(lang_compile_config, lang_runtime_config), + prepareJs(lang_compile_config, lang_runtime_config), + ]); + } + + const t1 = Date.now(); + + const res = await prepare_cache[language_name]; + const antlr_result = res[0]; + const js_parser = res[1]; + const t2 = Date.now(); + + const code = await fs.readFile(code_file, {'encoding': 'utf8'}); + const t3 = Date.now(); + + const treeViaJava = await genTreeViaJava(lang_runtime_config, antlr_result, code); + const t4 = Date.now(); + + const treeViaJs = await genTreeViaJs(lang_runtime_config, js_parser, code); + const t5 = Date.now(); + + console.log('compare(' + code_file + ') prepare: ' + (t2 - t1) + 'ms'); + console.log('compare(' + code_file + ') readFile: ' + (t3 - t2) + 'ms'); + console.log('compare(' + code_file + ') genJava: ' + (t4 - t3) + 'ms'); + console.log('compare(' + code_file + ') genJs: ' + (t5 - t4) + 'ms'); + + return treeViaJava.trim() === treeViaJs.trim(); +}; + + + +describe('grammars-v4/', () => { + let prev_timeout; + beforeAll(() => { + prev_timeout = jasmine.DEFAULT_TIMEOUT_INTERVAL; + jasmine.DEFAULT_TIMEOUT_INTERVAL = 60000; + return fs.remove(config.build_path).catch(err => { + console.error(err); + }); + }); + afterAll(() => { + jasmine.DEFAULT_TIMEOUT_INTERVAL = prev_timeout; + }); + + const test_snippet = function(lang_key, description, code_filename) { + it(description, () => { + expect.assertions(1); + return compare(lang_key, __dirname + '/code/' + code_filename).then(data => { + expect(data).toBeTruthy(); + }).catch(err => { + console.error(err); + }); + }); + }; + + describe('grammars-v4/python3/', () => { + test_snippet('python3', 'handles basic hello worlds', 'snippet.voc84cjo.py'); + test_snippet('python3', 'handles a script adding the input arguments', 'snippet.kt29xnfw.py'); + }); + + describe('grammars-v4/java8/', () => { + test_snippet('java8', 'handles basic hello worlds', 'snippet.k17eu4f2.java'); + }); +}); diff --git a/__tests__/utils/__snapshots__/utils.test.js.snap b/__tests__/utils/__snapshots__/utils.test.js.snap new file mode 100644 index 0000000..f159457 --- /dev/null +++ b/__tests__/utils/__snapshots__/utils.test.js.snap @@ -0,0 +1,14 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`test-utils.js makeNode returns the expected object 1`] = ` +Object { + "ast_type": "Some Type", + "begin": 0, + "children": Array [], + "detail": Array [], + "end": 5, + "tags": Array [ + "Some Type", + ], +} +`; diff --git a/__tests__/utils/utils.test.js b/__tests__/utils/utils.test.js new file mode 100644 index 0000000..5e2bd5c --- /dev/null +++ b/__tests__/utils/utils.test.js @@ -0,0 +1,10 @@ +const { makeNode } = require('../../src/utils/utils.js'); + +describe(`test-utils.js`, () => { + describe(`makeNode`, () => { + it(`returns the expected object`, () => { + const node = makeNode('Some Type', 0, 5, [], []); + expect(node).toMatchSnapshot(); + }); + }); +}); diff --git a/app/compile.js b/app/compile.js deleted file mode 100644 index 7204dd5..0000000 --- a/app/compile.js +++ /dev/null @@ -1,129 +0,0 @@ -let path = require('path'); -let fs = require('fs-promise'); -let child_process = require('child-process-promise'); -let antlr = require('antlr4'); - -let config = require('../config.js'); -let expect_error = require('./expect_error.js'); -let tree_matcher = require('./tree_matcher.js'); -let java_func_data_generator = require('./java_func_data_generator.js'); - -let array_diff = function(a, b) { - return a.filter(function(i) {return b.indexOf(i) === -1;}); -}; - -module.exports = function(lang_compile_config, lang_runtime_config) { - // Figure out the language key - let language_key = lang_runtime_config.language.toLowerCase(); - - // Figure out the path to the grammar file - let g4_path = lang_compile_config.grammar_path; - if (!g4_path) { - g4_path = path.resolve(__dirname, '..', 'grammars-v4', language_key, lang_compile_config.grammar_file); - } - - // Figure out the path to the cache directory - let cache_dir = config.resolve_cache_dir(lang_runtime_config); - let cache_g4_path = path.resolve(cache_dir, lang_runtime_config.language + '.g4'); - - - let compile_promise = async function() { - // Make sure the cache directory exists - await fs.mkdir(config.cache_path).catch(expect_error('EEXIST', function() {})); - - // Make sure the language cache directory exists - await fs.mkdir(cache_dir).catch(expect_error('EEXIST', function() {})); - - // Copies the g4 file into the cache directory - await fs.copy(g4_path, cache_g4_path); - - // Prepare options to the antlr compiler that generates the antlr lexer and antlr parser - let cmd = 'java'; - let args = [ - '-Xmx500M', - '-cp', '../../bin/antlr-4.6-complete.jar', - 'org.antlr.v4.Tool', - '-long-messages', - lang_compile_config.generate_listener ? '-listener' : '-no-listener', - lang_compile_config.generate_visitor ? '-visitor' : '-no-visitor', - '-Dlanguage=JavaScript', - lang_runtime_config.language + '.g4', - ]; - let opts = { - 'cwd': cache_dir, - 'stdio': ['ignore', process.stdout, process.stderr], - }; - - // Call antlr - await child_process.spawn(cmd, args, opts); - - if (lang_compile_config.needs_java_func_data) { - await fs.stat(config.cache_path + '/java_func_data') - .catch(expect_error('ENOENT', java_func_data_generator)); - } - - // Make sure the generated parser has the same rules as our config file. - let parser_classname = lang_runtime_config.language + 'Parser'; - let ParserClass = require(cache_dir + '/' + parser_classname + '.js')[parser_classname]; - let parser = new ParserClass(); - - // Create an array of symbol (terminal) names - let symbol_name_map = ['_EPSILON', '_EOF', '_INVALID'] - .concat(parser.symbolicNames.slice(1)) - .map(function(val) {return val ? '.' + val : undefined;}); - - // Create the list of rule names (both terminals and non-terminals) - let parser_rules = parser.ruleNames.concat(symbol_name_map.filter(Boolean)); - let config_rules = Object.keys(lang_runtime_config.rules); - - // Make sure the parser doesn't have extra rules - let config_missing = array_diff(parser_rules, config_rules); - if (config_missing.length) { - throw new Error('Missing rules ' + JSON.stringify(config_missing)); - } - - // Make sure our config doesn't have extra rules - let config_extra = array_diff(config_rules, parser_rules); - if (config_extra.length) { - throw new Error('Extra rules ' + JSON.stringify(config_extra)); - } - - // Generate the runtime config modifier - let code = ''; - code += '// This function is generated by app/compile.js.\n'; - code += '// Do not attempt to make changes. They will be over-written.\n\n'; - - // This is a function that modifies the lang_runtime_config - code += 'module.exports = function(lang_runtime_config) {\n'; - - // It adds a symbol_name_map array - code += 'lang_runtime_config.symbol_name_map = ' + JSON.stringify(symbol_name_map, null, 2) + ';\n'; - - // And a rule_name_map array - code += 'lang_runtime_config.rule_name_map = ' + JSON.stringify(parser.ruleNames, null, 2) + ';\n'; - - // And a tree_matcher function - if (lang_compile_config.tree_matcher_specs) { - let generator = await tree_matcher.make_generator(lang_compile_config, lang_runtime_config); - let tree_matchers = lang_compile_config.tree_matcher_specs.map(generator); - code += 'lang_runtime_config.tree_matcher = function(root) {\n' + tree_matchers.join('\n') + '\n};\n'; - } - - code += '};'; - - // Write the runtime config modifier - let modifier_path = path.resolve(cache_dir, 'runtime_config_modifier.js'); - await fs.writeFile(modifier_path, code); - }; - - // Stat the cache directory, which is the standard way if checking if it exists. - return fs.stat(cache_dir) - .catch(expect_error('ENOENT', compile_promise)) - .then(function() { - // In either case, return an object describing the results. - return { - // Currently, this description is just where the compiled files are stored. - 'cache_dir': cache_dir - }; - }); -}; diff --git a/app/transformers/simplify_node.js b/app/transformers/simplify_node.js deleted file mode 100644 index d09888e..0000000 --- a/app/transformers/simplify_node.js +++ /dev/null @@ -1,32 +0,0 @@ -let TerminalNodeImpl = require('antlr4/tree/Tree.js').TerminalNodeImpl; - -module.exports = function(lang_runtime_config, root) { - - // Takes an antlr node generated by the antlr parser, and outputs our simplified node. - let simplify_node = function(node) { - if (node instanceof TerminalNodeImpl) { - let ast_type = lang_runtime_config.symbol_name_map[node.symbol.type + 2]; - return { - 'ast_type': ast_type, - 'tags': [ast_type], - 'begin': node.symbol.start, - 'end': node.symbol.stop + 1, - 'text': node.symbol.text, - 'detail': [], - 'children': [], - }; - } else { - let ast_type = lang_runtime_config.rule_name_map[node.ruleIndex]; - return { - 'ast_type': ast_type, - 'tags': [ast_type], - 'begin': node.start.start, - 'end': (node.stop ? node.stop : node.start).stop + 1, - 'detail': [], - 'children': node.children ? node.children.map(simplify_node) : [], - }; - } - }; - - return simplify_node(root); -}; diff --git a/app/tree_matcher_parser/lang_config.compile.js b/app/tree_matcher_parser/lang_config.compile.js deleted file mode 100644 index e305562..0000000 --- a/app/tree_matcher_parser/lang_config.compile.js +++ /dev/null @@ -1,5 +0,0 @@ -let path = require('path'); - -module.exports = { - 'grammar_path': path.resolve(__dirname, 'TreeMatcher.g4'), -}; diff --git a/config.js b/config.js index c2fceec..c328458 100644 --- a/config.js +++ b/config.js @@ -1,11 +1,13 @@ -let path = require('path'); +const path = require('path'); +const os = require('os'); module.exports = { - 'app_path': path.resolve(__dirname, 'app'), + 'app_path': path.resolve(__dirname, 'src'), 'lang_configs_path': path.resolve(__dirname, 'language_configs'), - 'cache_path': path.resolve(__dirname, '_cache'), - 'resolve_cache_dir': function(lang_runtime_config) { - let language_key = lang_runtime_config.language.toLowerCase(); - return path.resolve(this.cache_path, language_key); + //'build_path': path.resolve(os.tmpdir(), 'codesplain_build'), + 'build_path': path.resolve(__dirname, 'build'), + 'resolve_build_dir': function(lang_runtime_config, target_language) { + const language_key = lang_runtime_config.language.toLowerCase(); + return path.resolve(this.build_path, language_key + '_to_' + target_language); }, }; diff --git a/grammars-v4 b/grammars-v4 index 727ec61..96255d4 160000 --- a/grammars-v4 +++ b/grammars-v4 @@ -1 +1 @@ -Subproject commit 727ec61bfa476363400a7162f6c681a783ddc573 +Subproject commit 96255d4c1bfcc377263842d35cbba92620154275 diff --git a/language_configs/java8.compile.js b/language_configs/java8.compile.js index 0615069..051210c 100644 --- a/language_configs/java8.compile.js +++ b/language_configs/java8.compile.js @@ -1,7 +1,10 @@ // Configuration that can only be read by the compiler. module.exports = { - 'grammar_file': 'Java8.JavaScriptTarget.g4', + 'grammar_files': { + 'Java': 'Java8.g4', + 'JavaScript': 'Java8.JavaScriptTarget.g4', + }, 'needs_java_func_data': true, 'tree_matcher_specs': require('./java8.tree_matcher_specs.js'), }; diff --git a/language_configs/python3.compile.js b/language_configs/python3.compile.js index 72ea244..c3b92b9 100644 --- a/language_configs/python3.compile.js +++ b/language_configs/python3.compile.js @@ -1,6 +1,9 @@ // Configuration that can only be read by the compiler. module.exports = { - 'grammar_file': 'Python3.JavaScriptTarget.g4', + 'grammar_files': { + 'Java': 'Python3.g4', + 'JavaScript': 'Python3.JavaScriptTarget.g4', + }, 'tree_matcher_specs': require('./python3.tree_matcher_specs.js'), }; diff --git a/language_configs/python3.rules.js b/language_configs/python3.rules.js index 49399f9..766542d 100644 --- a/language_configs/python3.rules.js +++ b/language_configs/python3.rules.js @@ -67,7 +67,6 @@ module.exports = { 'term': {'collapse': true}, 'factor': {'collapse': true}, 'power': {'collapse': true}, - 'trailed_atom': {'collapse': true}, 'atom': {}, 'testlist_comp': {}, 'trailer': {}, diff --git a/language_configs/python3.tree_matcher_specs.js b/language_configs/python3.tree_matcher_specs.js index 3e83b75..022c52f 100644 --- a/language_configs/python3.tree_matcher_specs.js +++ b/language_configs/python3.tree_matcher_specs.js @@ -4,7 +4,7 @@ module.exports = [ // The pattern specifies which nodes to match. // When a node is matched, actor will be executed. - pattern: 'for_stmt [.FOR, /.NAME:iter, .IN, /trailed_atom [/.NAME="range", trailer\ + pattern: 'for_stmt [.FOR, /.NAME:iter, .IN, /power [/.NAME="range", trailer\ [.OPEN_PAREN, arglist [/.DECIMAL_INTEGER:begin, .COMMA,\ /.DECIMAL_INTEGER:end], .CLOSE_PAREN]], .COLON, suite]', @@ -32,7 +32,7 @@ module.exports = [ }, { provides_tags: ['function_call'], - pattern: 'trailed_atom [/.NAME:name, trailer [.OPEN_PAREN, arglist:args, .CLOSE_PAREN]]', + pattern: 'power [/.NAME:name, trailer [.OPEN_PAREN, arglist:args, .CLOSE_PAREN]]', actor: function() { root.tags.push('function_call'); diff --git a/make b/make index fbcfa13..a704730 100755 --- a/make +++ b/make @@ -14,14 +14,14 @@ DEBUG=0 # Help function help { echo "Usage: $0 [-l|--lang lang] [-m|--minify] [-d|--debug] [-h|--help]" - echo " Makes the parser with the current configuration and places in './public/lang'" + echo " Makes the parser with the current configuration and places in './build/parsers'" echo " -l|--lang [lang]: Which language the parser is for" echo " -m|--minify: Optimizes the parser, will not be human-readable" echo " -d|--debug: Enables debugging info" echo " -h|--help: Prints this help text" } -#Ensure npm installed +# Ensure npm installed NPM="$(which npm)" if [ "$NPM" == "" ]; then echo "npm not installed" diff --git a/mappings/python3.csv b/mappings/python3.csv index d72a8a1..4ca9e3a 100644 --- a/mappings/python3.csv +++ b/mappings/python3.csv @@ -63,7 +63,6 @@ arith_expr,1,Arithmetic Expression,#FFA500 term,0,, factor,0,, power,0,, -trailed_atom,0,, atom,0,, testlist_comp,0,, trailer,0,, @@ -112,8 +111,8 @@ integer,1,Integers,#ABF7C6 .NOT,0,, .IS,0,, .NONE,0,, -.TRUE,0,, -.FALSE,0,, +.TRUE,1,Boolean (True),#FF9933 +.FALSE,1,Boolean (False),#E67300 .CLASS,0,, .YIELD,0,, .DEL,0,, diff --git a/package.json b/package.json index 6815205..e51daea 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,12 @@ "jest": "^19.0.2" }, "jest": { - "moduleFileExtensions": ["js", "json", "jsx", "node", "csv"] + "moduleFileExtensions": [ + "js", + "json", + "jsx", + "node", + "csv" + ] } } diff --git a/profile_test b/profile_test new file mode 100755 index 0000000..a22534a --- /dev/null +++ b/profile_test @@ -0,0 +1,24 @@ +#!/bin/sh + +# Exit if error +set -e + +# Remove any profile logs from previous runs +rm -f isolate-*-v8.log + +# Run profiler +node --prof ./node_modules/.bin/jest + +# The profiler outputs 3 files. +# Find the one that contains results from running /Users/joelwalker/codesplain/build/output/ +match=$(grep --fixed-strings --files-with-matches '/Users/joelwalker/codesplain/build/output/' isolate-*-v8.log) +[ $(wc -l <<< "$match") = "1" ] || { + echo 'Did not find exactly one file with search string in profile output' + echo "$match" + exit 1 +} + +node --prof-process "$match" > profile_result.txt +echo "Wrote profile_result.txt" + +rm isolate-*-v8.log diff --git a/publish b/publish index df94595..79c0531 100755 --- a/publish +++ b/publish @@ -1,5 +1,13 @@ #!/usr/bin/env bash +################################################################################ +# This script builds the parsers and pushes them to their S3 bucket. It can be # +# used in dev mode, to push to the dev bucket: # +# ./publish dev # +# or release mode, to push to a prod bucket with the specified release version:# +# ./publish release v0.1 # +################################################################################ + # Exit if error set -e diff --git a/src/compile.js b/src/compile.js new file mode 100644 index 0000000..055696d --- /dev/null +++ b/src/compile.js @@ -0,0 +1,144 @@ +const path = require('path'); +const fs = require('fs-promise'); +const child_process = require('child-process-promise'); +const antlr = require('antlr4'); + +const config = require('../config'); +const expect_error = require('./utils/expect_error.js'); +const run_antlr = require('./utils/run_antlr.js'); +const tree_matcher = require('./tree_matcher.js'); +const java_func_data_generator = require('./java_func_data_generator.js'); + +const array_diff = (a, b) => a.filter(i => b.indexOf(i) === -1); + +/* +Sequentially invokes each task in `tasks` using the previous tasks's +return value as its argument. `tasks` must be an array of async functions. +The first task is invoked with a null argument. +*/ +const waterfall = async (tasks) => { + let args = null; + for(let i = 0; i < tasks.length; i++) { + args = await tasks[i](args); + } + return args; +}; + +/* +Returns a set of async closure functions with references to the given +lang_compile_config & lang_runtime_config that can be used to build the +parser. +*/ +const build_tasks = (lang_compile_config, lang_runtime_config) => { + const { + tree_matcher_specs, + needs_java_func_data, + } = lang_compile_config; + const { + language, + rules, + } = lang_runtime_config; + + let result; + + const make_lexer_and_parser = async () => { + result = await run_antlr(lang_compile_config, lang_runtime_config, 'JavaScript'); + } + + /* --- Generates java func data, if this lang needs any --- */ + const make_java_func_data = async () => { + if (needs_java_func_data) { + await fs.stat(config.build_path + '/java_func_data') + .catch(expect_error('ENOENT', java_func_data_generator)); + } + } + + /* --- Creates and returns the parser and symbol_name_map --- */ + const make_parser = async () => { + // Create instance of the parser + const parser_classname = language + 'Parser'; + const ParserClass = require(`${result.build_dir}/${parser_classname}.js`)[parser_classname] + const parser = new ParserClass(); + + // Create an array of symbol (terminal) names + const symbol_name_map = ['_EPSILON', '_EOF', '_INVALID'] + .concat(parser.symbolicNames.slice(1)) + .map((val) => val ? '.' + val : undefined); + + // Create the lists of rule names (both terminals and non-terminals) + const parser_rules = parser.ruleNames.concat(symbol_name_map.filter(Boolean)); + const config_rules = Object.keys(rules); + + // Make sure the parser doesn't have extra rules + const config_missing = array_diff(parser_rules, config_rules); + if (config_missing.length) { + throw new Error('Missing rules ' + JSON.stringify(config_missing)); + } + + // Make sure our config doesn't have extra rules + const config_extra = array_diff(config_rules, parser_rules); + if (config_extra.length) { + throw new Error('Extra rules ' + JSON.stringify(config_extra)); + } + + return { symbol_name_map, parser }; + } + + /* --- Dynamically creates runtime config modifier function --- */ + const make_runtime_config_modifier = async ({ symbol_name_map, parser }) => { + // Turns these maps into human-readable JSON for insertion + // into returned function string + const symbol_name_map_str = JSON.stringify(symbol_name_map, null, 2); + const rule_name_map_str = JSON.stringify(parser.ruleNames, null, 2); + + // Add a tree matcher function if appropriate too + let tree_matchers_str = ''; + if (tree_matcher_specs) { + const generator = await tree_matcher.make_generator( + lang_compile_config, + lang_runtime_config + ); + const tree_matchers = tree_matcher_specs.map(generator); + tree_matchers_str = + `lang_runtime_config.tree_matcher = function(root) { + ${tree_matchers.join('\n')} + };`; + } + + // Return the runtime config modifier - a function that + // modifies the lang_runtime_config + return ` + /* + This function is generated by app/compile.js + Do not attempt to make changes. They will be overwritten. + */ + module.exports = function(lang_runtime_config) { + lang_runtime_config.symbol_name_map = ${symbol_name_map_str}; + lang_runtime_config.rule_name_map = ${rule_name_map_str}; + ${tree_matchers_str} + };`; + } + + /* --- Writes the runtime config modifier to file system --- */ + const write_runtime_config_modifier = async (config_modifier) => { + const modifier_path = path.resolve(result.build_dir, 'runtime_config_modifier.js'); + await fs.writeFile(modifier_path, config_modifier); + } + + /* --- Returns the build_dir wrapped in an object --- */ + const final_task = async () => result; + + // Return the set of async closures + return [ + make_lexer_and_parser, + make_java_func_data, + make_parser, + make_runtime_config_modifier, + write_runtime_config_modifier, + final_task, + ]; +} + +module.exports = (lang_compile_config, lang_runtime_config) => { + return waterfall(build_tasks(lang_compile_config, lang_runtime_config)); +}; diff --git a/app/error_listener.js b/src/error_listener.js similarity index 100% rename from app/error_listener.js rename to src/error_listener.js diff --git a/app/java_func_data_generator.js b/src/java_func_data_generator.js similarity index 91% rename from app/java_func_data_generator.js rename to src/java_func_data_generator.js index 23af57f..f2322a9 100644 --- a/app/java_func_data_generator.js +++ b/src/java_func_data_generator.js @@ -3,15 +3,15 @@ let fs = require('fs-promise'); let child_process = require('child-process-promise'); let config = require('../config'); -let expect_error = require('./expect_error.js'); +let expect_error = require('./utils/expect_error.js'); let compile_promise; let compile = async function() { - let cache_dir = path.resolve(config.cache_path, 'java_func_data'); + let cache_dir = path.resolve(config.build_path, 'java_func_data'); // Make sure the cache directory exists - await fs.mkdir(config.cache_path).catch(expect_error('EEXIST', function() {})); + await fs.mkdir(config.build_path).catch(expect_error('EEXIST', function() {})); // Make sure the language cache directory exists await fs.mkdir(cache_dir).catch(expect_error('EEXIST', function() {})); diff --git a/app/java_function_translator/Encoder.java b/src/java_function_translator/Encoder.java similarity index 100% rename from app/java_function_translator/Encoder.java rename to src/java_function_translator/Encoder.java diff --git a/app/java_function_translator/JavaFunctionTranslator.java b/src/java_function_translator/JavaFunctionTranslator.java similarity index 100% rename from app/java_function_translator/JavaFunctionTranslator.java rename to src/java_function_translator/JavaFunctionTranslator.java diff --git a/app/java_function_translator/RangeEncoder.java b/src/java_function_translator/RangeEncoder.java similarity index 100% rename from app/java_function_translator/RangeEncoder.java rename to src/java_function_translator/RangeEncoder.java diff --git a/app/java_function_translator/TranslatedFunction.java b/src/java_function_translator/TranslatedFunction.java similarity index 100% rename from app/java_function_translator/TranslatedFunction.java rename to src/java_function_translator/TranslatedFunction.java diff --git a/app/java_function_translator/functions/Character_isJavaIdentifierPart_int.java b/src/java_function_translator/functions/Character_isJavaIdentifierPart_int.java similarity index 100% rename from app/java_function_translator/functions/Character_isJavaIdentifierPart_int.java rename to src/java_function_translator/functions/Character_isJavaIdentifierPart_int.java diff --git a/app/java_function_translator/functions/Character_isJavaIdentifierStart_int.class b/src/java_function_translator/functions/Character_isJavaIdentifierStart_int.class similarity index 100% rename from app/java_function_translator/functions/Character_isJavaIdentifierStart_int.class rename to src/java_function_translator/functions/Character_isJavaIdentifierStart_int.class diff --git a/app/java_function_translator/functions/Character_isJavaIdentifierStart_int.java b/src/java_function_translator/functions/Character_isJavaIdentifierStart_int.java similarity index 100% rename from app/java_function_translator/functions/Character_isJavaIdentifierStart_int.java rename to src/java_function_translator/functions/Character_isJavaIdentifierStart_int.java diff --git a/app/java_functions/Character_isJavaIdentifierPart.js b/src/java_functions/Character_isJavaIdentifierPart.js similarity index 63% rename from app/java_functions/Character_isJavaIdentifierPart.js rename to src/java_functions/Character_isJavaIdentifierPart.js index a2d8fcb..74836f9 100644 --- a/app/java_functions/Character_isJavaIdentifierPart.js +++ b/src/java_functions/Character_isJavaIdentifierPart.js @@ -1,4 +1,4 @@ -let data = require('Cache/java_func_data/Character_isJavaIdentifierPart_int.json'); +let data = require('Build/java_func_data/Character_isJavaIdentifierPart_int.json'); let range_decoder = require('./range_decoder.js'); module.exports = range_decoder(data); diff --git a/app/java_functions/Character_isJavaIdentifierStart.js b/src/java_functions/Character_isJavaIdentifierStart.js similarity index 63% rename from app/java_functions/Character_isJavaIdentifierStart.js rename to src/java_functions/Character_isJavaIdentifierStart.js index 294193a..5484446 100644 --- a/app/java_functions/Character_isJavaIdentifierStart.js +++ b/src/java_functions/Character_isJavaIdentifierStart.js @@ -1,4 +1,4 @@ -let data = require('Cache/java_func_data/Character_isJavaIdentifierStart_int.json'); +let data = require('Build/java_func_data/Character_isJavaIdentifierStart_int.json'); let range_decoder = require('./range_decoder.js'); module.exports = range_decoder(data); diff --git a/app/java_functions/Character_toCodePoint.js b/src/java_functions/Character_toCodePoint.js similarity index 100% rename from app/java_functions/Character_toCodePoint.js rename to src/java_functions/Character_toCodePoint.js diff --git a/app/java_functions/range_decoder.js b/src/java_functions/range_decoder.js similarity index 100% rename from app/java_functions/range_decoder.js rename to src/java_functions/range_decoder.js diff --git a/app/runtime.js b/src/runtime.js similarity index 79% rename from app/runtime.js rename to src/runtime.js index 8ba677d..816002d 100644 --- a/app/runtime.js +++ b/src/runtime.js @@ -3,14 +3,19 @@ let antlr = require('antlr4'); let lang_runtime_config = require('LangRuntimeConfig'); -require('LangCache/runtime_config_modifier.js')(lang_runtime_config); +// Modify the runtime config to give it the following properties that are +// generated at compile time (in compile.js): +// • symbol_name_map => Array of Strings +// • rule_name_map => Array of Strings +// • tree_matcher => Function (optional) +require('LangBuild/runtime_config_modifier.js')(lang_runtime_config); let lexer_classname = lang_runtime_config.language + 'Lexer'; let parser_classname = lang_runtime_config.language + 'Parser'; // Loads the lexer and parser class generated by the antlr compiler. -let LexerClass = require('LangCache/' + lexer_classname + '.js')[lexer_classname]; -let ParserClass = require('LangCache/' + parser_classname + '.js')[parser_classname]; +let LexerClass = require('LangBuild/' + lexer_classname + '.js')[lexer_classname]; +let ParserClass = require('LangBuild/' + parser_classname + '.js')[parser_classname]; // Loads our custom error listener. let ErrorListener = require('App/error_listener'); @@ -48,6 +53,10 @@ module.exports = function(input, error_callback, options) { // This tree has complicated nodes that need to be simplified by our process_node function. let tree = parser[lang_runtime_config.entry_rule](); + if (options.return_antlr_tree) { + return tree; + } + // Transform the tree and return it return transformers(lang_runtime_config, tree); }; diff --git a/app/transformers.js b/src/transformers.js similarity index 100% rename from app/transformers.js rename to src/transformers.js diff --git a/app/transformers/collapse.js b/src/transformers/collapse.js similarity index 93% rename from app/transformers/collapse.js rename to src/transformers/collapse.js index 3342e38..2893d1b 100644 --- a/app/transformers/collapse.js +++ b/src/transformers/collapse.js @@ -4,7 +4,8 @@ module.exports = function(lang_runtime_config, root) { let collapse = function(node) { let type_opts = lang_runtime_config.rules[node.ast_type]; - // If there is only one child, and it is exactly the same as this node, then eliminate this node. + // If there is only one child, and it is exactly the same as this node, + // then eliminate this node. if (type_opts.collapse && node.children.length === 1 && node.begin === node.children[0].begin diff --git a/app/transformers/run_tree_matchers.js b/src/transformers/run_tree_matchers.js similarity index 100% rename from app/transformers/run_tree_matchers.js rename to src/transformers/run_tree_matchers.js diff --git a/src/transformers/simplify_node.js b/src/transformers/simplify_node.js new file mode 100644 index 0000000..207b81e --- /dev/null +++ b/src/transformers/simplify_node.js @@ -0,0 +1,36 @@ +const { TerminalNodeImpl } = require('antlr4/tree/Tree.js'); + +module.exports = function(lang_runtime_config, root) { + + // Takes an antlr node generated by the antlr parser, and + // outputs our simplified node. + const simplify_node = function(node) { + const { symbol_name_map, rule_name_map } = lang_runtime_config; + if (node instanceof TerminalNodeImpl) { + const { type, start, stop, text } = node.symbol; + let ast_type = symbol_name_map[type + 2]; + return { + 'ast_type': ast_type, + 'tags': [ast_type], + 'begin': start, + 'end': stop + 1, + 'text': text, + 'detail': [], + 'children': [], + }; + } else { + const { ruleIndex, start, stop, children } = node; + let ast_type = rule_name_map[ruleIndex]; + return { + 'ast_type': ast_type, + 'tags': [ast_type], + 'begin': start.start, + 'end': (stop ? stop : start).stop + 1, + 'detail': [], + 'children': children ? children.map(simplify_node) : [], + }; + } + }; + + return simplify_node(root); +}; diff --git a/app/tree_matcher.js b/src/tree_matcher.js similarity index 98% rename from app/tree_matcher.js rename to src/tree_matcher.js index 03453fd..f6069dd 100644 --- a/app/tree_matcher.js +++ b/src/tree_matcher.js @@ -25,8 +25,8 @@ module.exports.make_generator = async function(lang_compile_config, lang_runtime let lexer_classname = tree_matcher_runtime_config.language + 'Lexer'; let parser_classname = tree_matcher_runtime_config.language + 'Parser'; - let LexerClass = require(compile_result.cache_dir + '/' + lexer_classname + '.js')[lexer_classname]; - let ParserClass = require(compile_result.cache_dir + '/' + parser_classname + '.js')[parser_classname]; + let LexerClass = require(compile_result.build_dir + '/' + lexer_classname + '.js')[lexer_classname]; + let ParserClass = require(compile_result.build_dir + '/' + parser_classname + '.js')[parser_classname]; let ErrorListener = require('./error_listener'); // For use later, when the tree matcher is optimized diff --git a/app/tree_matcher_parser/TreeMatcher.g4 b/src/tree_matcher_parser/TreeMatcher.g4 similarity index 100% rename from app/tree_matcher_parser/TreeMatcher.g4 rename to src/tree_matcher_parser/TreeMatcher.g4 diff --git a/app/tree_matcher_parser/generator_listener.js b/src/tree_matcher_parser/generator_listener.js similarity index 100% rename from app/tree_matcher_parser/generator_listener.js rename to src/tree_matcher_parser/generator_listener.js diff --git a/src/tree_matcher_parser/lang_config.compile.js b/src/tree_matcher_parser/lang_config.compile.js new file mode 100644 index 0000000..4c98987 --- /dev/null +++ b/src/tree_matcher_parser/lang_config.compile.js @@ -0,0 +1,8 @@ +let path = require('path'); + +module.exports = { + 'grammar_dir': __dirname, + 'grammar_files': { + 'JavaScript': 'TreeMatcher.g4', + }, +}; diff --git a/app/tree_matcher_parser/lang_config.runtime.js b/src/tree_matcher_parser/lang_config.runtime.js similarity index 100% rename from app/tree_matcher_parser/lang_config.runtime.js rename to src/tree_matcher_parser/lang_config.runtime.js diff --git a/app/type_graph_generator.js b/src/type_graph_generator.js similarity index 100% rename from app/type_graph_generator.js rename to src/type_graph_generator.js diff --git a/app/expect_error.js b/src/utils/expect_error.js similarity index 100% rename from app/expect_error.js rename to src/utils/expect_error.js diff --git a/src/utils/run_antlr.js b/src/utils/run_antlr.js new file mode 100644 index 0000000..cd3668d --- /dev/null +++ b/src/utils/run_antlr.js @@ -0,0 +1,71 @@ +const path = require('path'); +const fs = require('fs-promise'); +const child_process = require('child-process-promise'); + +const config = require('../../config.js'); +const expect_error = require('./expect_error.js'); + +module.exports = async (lang_compile_config, lang_runtime_config, target_language) => { + const { + grammar_dir, + grammar_files, + } = lang_compile_config; + const { + language, + generate_visitor, + generate_listener, + } = lang_runtime_config; + + // Figure out the language key + const language_key = language.toLowerCase(); + + // Figure out the path to the grammar file + const g4_dir = grammar_dir ? grammar_dir : path.resolve(__dirname, '..', '..', 'grammars-v4', language_key); + const g4_path = path.resolve(g4_dir, grammar_files[target_language]); + + const build_dir = config.resolve_build_dir(lang_runtime_config, target_language); + const build_g4_path = path.resolve(build_dir, language + '.g4'); + + /* ============================ Build Tasks: ============================== */ + /* --- Creates build directory and copies grammar files into it. --- */ + const prepare_build_dir = async () => { + const on_eexist_err = expect_error('EEXIST', () => {}); + + // Make sure the build directory exists + await fs.mkdir(config.build_path).catch(on_eexist_err); + + // Make sure the language build directory exists + await fs.mkdir(build_dir).catch(on_eexist_err); + + // Copies the g4 file into the build directory + await fs.copy(g4_path, build_g4_path); + } + + /* --- Invokes the antlr process --- */ + const invoke_antlr = async () => { + // Prepare options to the antlr compiler that generates + // the antlr lexer and antlr parser + const cmd = 'java'; + const args = [ + '-Xmx500M', + '-cp', path.resolve(__dirname, '../../bin/antlr-4.6-complete.jar'), + 'org.antlr.v4.Tool', + '-long-messages', + generate_listener ? '-listener' : '-no-listener', + generate_visitor ? '-visitor' : '-no-visitor', + '-Dlanguage=' + target_language, + language + '.g4', + ]; + const opts = { + 'cwd': build_dir, + 'stdio': ['ignore', process.stdout, process.stderr], + }; + + await child_process.spawn(cmd, args, opts); + } + + await prepare_build_dir(); + await invoke_antlr(); + + return { build_dir }; +} diff --git a/src/utils/run_java.js b/src/utils/run_java.js new file mode 100644 index 0000000..d84b84b --- /dev/null +++ b/src/utils/run_java.js @@ -0,0 +1,31 @@ +const child_process = require('child-process-promise'); +const expect_error = require('./expect_error.js'); + +let javac_cache = {}; + +const run_javac = function(java_sources, opts) { + const key = JSON.stringify(java_sources); + if (typeof javac_cache[key] === 'undefined') { + const cmd = 'javac'; + const cmd_args = [/* put additional arguments in here */].concat(java_sources); + javac_cache[key] = child_process.spawn(cmd, cmd_args, opts); + } + return javac_cache[key]; +} + +const run_java = function(main_class, args, opts, java_stdin) { + const cmd = 'java'; + const cmd_args = [ + main_class, + ].concat(args); + const child = child_process.spawn(cmd, cmd_args, opts); + child.childProcess.stdin.write(java_stdin); + child.childProcess.stdin.end(); + return child; +} + +module.exports = async function(java_sources, main_class, args, opts, java_stdin) { + opts.capture = ['stdout', 'stderr']; + await run_javac(java_sources, opts); + return await run_java(main_class, args, opts, java_stdin); +} diff --git a/src/utils/sample-ast.json b/src/utils/sample-ast.json new file mode 100644 index 0000000..e1f9ece --- /dev/null +++ b/src/utils/sample-ast.json @@ -0,0 +1,277 @@ +{ + "type":"file_input", + "begin":0, + "end":13, + "tags":[ + + ], + "children":[ + { + "type":"simple_stmt", + "begin":0, + "end":13, + "tags":[ + + ], + "children":[ + { + "type":"expr", + "begin":0, + "end":13, + "tags":[ + + ], + "children":[ + { + "type":"trailed_atom", + "begin":0, + "end":13, + "tags":[ + { + "type":"function_call", + "name":"print", + "args":[ + { + "type":"argument", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":"test", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":"or_test", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":"and_test", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":"not_test", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":"comparison", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":"star_expr", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":"expr", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":"atom", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":"str", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":".STRING_LITERAL", + "text":"'test'", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ], + "children":[ + { + "type":"atom", + "begin":0, + "end":5, + "tags":[ + + ], + "children":[ + { + "type":".NAME", + "text":"print", + "begin":0, + "end":5, + "tags":[ + + ], + "children":[ + + ] + } + ] + }, + { + "type":"trailer", + "begin":5, + "end":13, + "tags":[ + + ], + "children":[ + { + "type":".OPEN_PAREN", + "text":"(", + "begin":5, + "end":6, + "tags":[ + + ], + "children":[ + + ] + }, + { + "type":"arglist", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":"expr", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":"atom", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":"str", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":".STRING_LITERAL", + "text":"'test'", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + + ] + } + ] + } + ] + } + ] + } + ] + }, + { + "type":".CLOSE_PAREN", + "text":")", + "begin":12, + "end":13, + "tags":[ + + ], + "children":[ + + ] + } + ] + } + ] + } + ] + } + ] + }, + { + "type":"._EOF", + "text":"", + "begin":13, + "end":13, + "tags":[ + + ], + "children":[ + + ] + } + ] +} diff --git a/src/utils/utils.js b/src/utils/utils.js new file mode 100644 index 0000000..269311d --- /dev/null +++ b/src/utils/utils.js @@ -0,0 +1,60 @@ +const { TerminalNodeImpl } = require('antlr4/tree/Tree.js'); + +/* +Returns an object built from the given arguments. The object is +representative of nodes present in the parser output AST. +*/ +module.exports.makeNode = (ast_type, begin, end, detail, children) => { + return { + ast_type, + tags: [ast_type], + begin, + end, + detail, + children, + } +}; + +/* +Returns an object built from the given arguments. The object is +representative of terminal nodes present in the parser output AST. +*/ +module.exports.makeTerminal = (ast_type, begin, end, text, detail) => { + return { + ast_type, + tags: [ast_type], + begin, + end, + text, + detail, + children: [], + } +}; + +/* +Returns an object built from the given arguments. The object +is a mock of nodes that appear in the ANTLR output AST. Note that these morks are incomplete, and only contain some properties found in real ANTLR nodes. +*/ +module.exports.makeAntlrNode = (ruleIndex, start, stop, children) => { + return { + ruleIndex, + start, + stop, + children, + } +}; + +/* +Returns an object built from the given arguments. The object +is a mock of terminal nodes that appear in ANTLR output ASTs. Note that these mocks are incomplete, and only contain some properties found in real ANTLR nodes. +*/ +module.exports.makeAntlrTerminal = (type, start, stop, text) => { + const terminal = Object.create(TerminalNodeImpl.prototype, {}); + terminal.symbol = { + type, + start, + stop, + text + }; + return terminal; +}; diff --git a/public/index.html b/tools/index.html similarity index 98% rename from public/index.html rename to tools/index.html index 5aa2cba..da342cb 100644 --- a/public/index.html +++ b/tools/index.html @@ -144,7 +144,7 @@ let lang_el = document.getElementById('language'); lang_el.onchange = function() { - let path = 'langs/' + lang_el.value + '.js'; + let path = 'build/parsers/' + lang_el.value + '.js'; // let path = 'https://api.codesplain.io/dev/parsers/' + lang_el.value; var script_el = document.createElement('script'); diff --git a/webpack.config.js b/webpack.config.js index ae4b1d6..07da6b7 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -1,20 +1,20 @@ -let fs = require('fs-promise'); -let path = require('path'); -let webpack = require('webpack'); -let ClosureCompilerPlugin = require('webpack-closure-compiler'); +const fs = require('fs-promise'); +const path = require('path'); +const webpack = require('webpack'); +const ClosureCompilerPlugin = require('webpack-closure-compiler'); - -let config = require('./config.js'); -let compile = require('./app/compile.js'); +const config = require('./config.js'); +const compile = require('./src/compile.js'); let langs; let optimize; let libraryTarget; +let outputPath; // Filter the files that should be compiled -let filter_lang = function(filename) { +const filter_lang = function(filename) { // Get the language name - let lang_name = filename.slice(0, -11); + const lang_name = filename.slice(0, -11); // Make sure the filename ends with ".compile.js" if (filename.slice(-11) !== '.compile.js') {return false;} @@ -29,21 +29,21 @@ let filter_lang = function(filename) { return true; }; -let prepare_lang = async function(filename) { +const prepare_lang = async function(filename) { // Get the language name again (same as above) - let lang_name = filename.slice(0, -11); + const lang_name = filename.slice(0, -11); // Load the language config files - let lang_compile_config_path = path.resolve(config.lang_configs_path, lang_name + '.compile.js'); - let lang_runtime_config_path = path.resolve(config.lang_configs_path, lang_name + '.runtime.js'); - let lang_compile_config = require(lang_compile_config_path); - let lang_runtime_config = require(lang_runtime_config_path); + const lang_compile_config_path = path.resolve(config.lang_configs_path, lang_name + '.compile.js'); + const lang_runtime_config_path = path.resolve(config.lang_configs_path, lang_name + '.runtime.js'); + const lang_compile_config = require(lang_compile_config_path); + const lang_runtime_config = require(lang_runtime_config_path); // Compile it! // This does a few things: // 1. Runs the antlr compiler that generates a javascript lexer and parser // 2. Runs the tree matcher compiler that generates tree matchers. - let compile_result = await compile(lang_compile_config, lang_runtime_config); + const compile_result = await compile(lang_compile_config, lang_runtime_config); // Return an object that will be used by webpack to compile the parser. return { @@ -51,14 +51,14 @@ let prepare_lang = async function(filename) { 'context': __dirname, // The file to compile. All other files are included in this file or in files included from this file. - 'entry': path.resolve(__dirname, 'app', 'runtime.js'), + 'entry': path.resolve(__dirname, 'src', 'runtime.js'), 'resolve': { 'alias': { 'App': config.app_path, - 'Cache': config.cache_path, + 'Build': config.build_path, 'LangRuntimeConfig$': lang_runtime_config_path, - 'LangCache': compile_result.cache_dir, + 'LangBuild': compile_result.build_dir, }, }, @@ -88,7 +88,7 @@ let prepare_lang = async function(filename) { 'filename': lang_name + (optimize ? '.min.js' : '.js'), // The output directory - 'path': path.resolve(__dirname, 'public', 'langs'), + 'path': outputPath, // How to export the parser library // commonjs2 - Use module.exports @@ -104,23 +104,24 @@ let prepare_lang = async function(filename) { module.exports = async function(env) { // Read command line options - langs = env && env.langs ? env.langs.split(',') : undefined; + langs = env && env.langs ? env.langs.toLowerCase().split(',') : undefined; optimize = Boolean(env && parseInt(env.optimize)); libraryTarget = (env && env.libraryTarget) || 'var'; + outputPath = (env && env.outputPath) || path.resolve(__dirname, 'build', 'parsers'); global.enable_debug = Boolean(env && parseInt(env.enable_debug)); // Then, find language configuration files - let files = await fs.readdir(config.lang_configs_path); + const files = await fs.readdir(config.lang_configs_path); // Filter langs and compile them. // Prepare_lang returns a promise, so after this step we have an array of promises. - let lang_configs = files.filter(filter_lang).map(prepare_lang); + const lang_configs = files.filter(filter_lang).map(prepare_lang); if (lang_configs.length === 0) { // If no languages were compiled, warn the user. console.warn('No languages generated...'); } else { // Otherwise, return a promise that resolves when all of the language promises resolve. - return Promise.all(lang_configs); + return await Promise.all(lang_configs); } }; diff --git a/yarn.lock b/yarn.lock index e6eda4e..199b3d0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -789,7 +789,7 @@ emojis-list@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/emojis-list/-/emojis-list-2.1.0.tgz#4daa4d9db00f9819880c79fa457ae5b09a1fd389" -enhanced-resolve@^3.0.0: +enhanced-resolve@^3.0.0, enhanced-resolve@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-3.1.0.tgz#9f4b626f577245edcf4b2ad83d86e17f4f421dec" dependencies: