From a537e6f182addaeb59bfad4e7f1b7fa0b6be376c Mon Sep 17 00:00:00 2001 From: Hopding Date: Wed, 12 Apr 2017 19:18:57 -0500 Subject: [PATCH 01/24] Restructured __tests__ dir; added test for collapse.js transformer --- .../mappings.test.js} | 6 +- .../__snapshots__/utils.test.js.snap | 11 + __tests__/test-utils/utils.test.js | 10 + __tests__/transformers/collapse.test.js | 63 ++++ app/transformers/collapse.js | 3 +- test-utils/sample-ast.json | 277 ++++++++++++++++++ test-utils/utils.js | 9 + 7 files changed, 376 insertions(+), 3 deletions(-) rename __tests__/{mappings.js => mappings/mappings.test.js} (82%) create mode 100644 __tests__/test-utils/__snapshots__/utils.test.js.snap create mode 100644 __tests__/test-utils/utils.test.js create mode 100644 __tests__/transformers/collapse.test.js create mode 100644 test-utils/sample-ast.json create mode 100644 test-utils/utils.js diff --git a/__tests__/mappings.js b/__tests__/mappings/mappings.test.js similarity index 82% rename from __tests__/mappings.js rename to __tests__/mappings/mappings.test.js index b74cc68..844895a 100644 --- a/__tests__/mappings.js +++ b/__tests__/mappings/mappings.test.js @@ -7,7 +7,8 @@ describe('mappings', () => { const csvFileName = `${file.split('.')[0]}.csv`; describe(csvFileName, () => { const encoding = { encoding: 'utf-8' }; - const csv = fs.readFileSync(`./mappings/${csvFileName}`, encoding) + const csvPath = __dirname + `/../../mappings/${csvFileName}`; + const csv = fs.readFileSync(csvPath, encoding) .split('\n') // Make array of lines .slice(1, -1); // Cut off the header and newline it('should have a definition for all rules', () => { @@ -17,7 +18,8 @@ describe('mappings', () => { // Now create sorted array of rules in the config // file, and compare the two - Object.keys(require(`../language_configs/${file}`)) + const languageConfigPath = __dirname + `/../../language_configs/${file}`; + Object.keys(require(languageConfigPath)) .sort((a, b) => a.localeCompare(b)) .forEach((rule, i) => expect(rule).toEqual(tokens[i])); }); diff --git a/__tests__/test-utils/__snapshots__/utils.test.js.snap b/__tests__/test-utils/__snapshots__/utils.test.js.snap new file mode 100644 index 0000000..05688c1 --- /dev/null +++ b/__tests__/test-utils/__snapshots__/utils.test.js.snap @@ -0,0 +1,11 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`test-utils.js makeNode returns the expected object 1`] = ` +Object { + "begin": 0, + "children": Array [], + "end": 5, + "tags": Array [], + "type": "Some Type", +} +`; diff --git a/__tests__/test-utils/utils.test.js b/__tests__/test-utils/utils.test.js new file mode 100644 index 0000000..7fc1ac0 --- /dev/null +++ b/__tests__/test-utils/utils.test.js @@ -0,0 +1,10 @@ +const { makeNode } = require('../../test-utils/utils.js'); + +describe(`test-utils.js`, () => { + describe(`makeNode`, () => { + it(`returns the expected object`, () => { + const node = makeNode('Some Type', 0, 5, [], []); + expect(node).toMatchSnapshot(); + }); + }); +}); diff --git a/__tests__/transformers/collapse.test.js b/__tests__/transformers/collapse.test.js new file mode 100644 index 0000000..513f1a5 --- /dev/null +++ b/__tests__/transformers/collapse.test.js @@ -0,0 +1,63 @@ +describe('collapse.js', () => { + const collapse = require('../../app/transformers/collapse.js'); + const { makeNode } = require('../../test-utils/utils.js'); + + const start = 0; + const end = 5; + const noTags = []; + + const bottomNode = makeNode('bottom_node', start, end, noTags, []); + const middleNode = makeNode('middle_node', start, end, noTags, [ bottomNode ]); + const topNode = makeNode('top_node', start, end, noTags, [ middleNode ]); + + it(`is a function`, () => { + expect(typeof(collapse)).toEqual('function'); + }); + + it(`recursively removes nodes from the tree if those nodes: + • are flagged as collapsible in the runtime config, and + • have single children who occupy the exact same range`, () => { + const langRuntimeConfig1 = { + 'rules': { + 'top_node': { 'collapse': true }, + 'middle_node': { 'collapse': true }, + 'bottom_node': { 'collapse': false }, + }, + }; + expect(collapse(langRuntimeConfig1, topNode)).toBe(bottomNode); + + const langRuntimeConfig2 = { + 'rules': { + 'top_node': { 'collapse': true }, + 'middle_node': { 'collapse': false }, + 'bottom_node': { 'collapse': true } + }, + }; + expect(collapse(langRuntimeConfig2, topNode)).toBe(middleNode); + }); + + it(`does NOT remove nodes from the tree if they are NOT flagged as collapsible`, + () => { + const langRuntimeConfig = { + 'rules': { + 'top_node': {}, + 'middle_node': { 'collapse': true }, + 'bottom_node': { 'collapse': true }, + }, + }; + expect(collapse(langRuntimeConfig, topNode)).toBe(topNode); + }); + + it(`does NOT remove nodes from the tree if they ARE flagged as collapsible, + but whose children do NOT occupt identical ranges`, () => { + const childNode = makeNode('child', 0, 3, noTags, []); + const parentNode = makeNode('parent', 0, 5, noTags, [ childNode ]); + const langRuntimeConfig = { + 'rules': { + 'parent': { 'collapse': true }, + 'child': { 'collapse': true }, + }, + }; + expect(collapse(langRuntimeConfig, parentNode)).toBe(parentNode); + }); +}); diff --git a/app/transformers/collapse.js b/app/transformers/collapse.js index 9dee8b0..6add8b4 100644 --- a/app/transformers/collapse.js +++ b/app/transformers/collapse.js @@ -4,7 +4,8 @@ module.exports = function(lang_runtime_config, root) { let collapse = function(node) { let type_opts = lang_runtime_config.rules[node.type]; - // If there is only one child, and it is exactly the same as this node, then eliminate this node. + // If there is only one child, and it is exactly the same as this node, + // then eliminate this node. if (type_opts.collapse && node.children.length === 1 && node.begin === node.children[0].begin diff --git a/test-utils/sample-ast.json b/test-utils/sample-ast.json new file mode 100644 index 0000000..e1f9ece --- /dev/null +++ b/test-utils/sample-ast.json @@ -0,0 +1,277 @@ +{ + "type":"file_input", + "begin":0, + "end":13, + "tags":[ + + ], + "children":[ + { + "type":"simple_stmt", + "begin":0, + "end":13, + "tags":[ + + ], + "children":[ + { + "type":"expr", + "begin":0, + "end":13, + "tags":[ + + ], + "children":[ + { + "type":"trailed_atom", + "begin":0, + "end":13, + "tags":[ + { + "type":"function_call", + "name":"print", + "args":[ + { + "type":"argument", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":"test", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":"or_test", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":"and_test", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":"not_test", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":"comparison", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":"star_expr", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":"expr", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":"atom", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":"str", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":".STRING_LITERAL", + "text":"'test'", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ], + "children":[ + { + "type":"atom", + "begin":0, + "end":5, + "tags":[ + + ], + "children":[ + { + "type":".NAME", + "text":"print", + "begin":0, + "end":5, + "tags":[ + + ], + "children":[ + + ] + } + ] + }, + { + "type":"trailer", + "begin":5, + "end":13, + "tags":[ + + ], + "children":[ + { + "type":".OPEN_PAREN", + "text":"(", + "begin":5, + "end":6, + "tags":[ + + ], + "children":[ + + ] + }, + { + "type":"arglist", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":"expr", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":"atom", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":"str", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + { + "type":".STRING_LITERAL", + "text":"'test'", + "begin":6, + "end":12, + "tags":[ + + ], + "children":[ + + ] + } + ] + } + ] + } + ] + } + ] + }, + { + "type":".CLOSE_PAREN", + "text":")", + "begin":12, + "end":13, + "tags":[ + + ], + "children":[ + + ] + } + ] + } + ] + } + ] + } + ] + }, + { + "type":"._EOF", + "text":"", + "begin":13, + "end":13, + "tags":[ + + ], + "children":[ + + ] + } + ] +} diff --git a/test-utils/utils.js b/test-utils/utils.js new file mode 100644 index 0000000..81b72f5 --- /dev/null +++ b/test-utils/utils.js @@ -0,0 +1,9 @@ +module.exports.makeNode = (type, begin, end, tags, children) => { + return { + type, + begin, + end, + tags, + children, + } +}; From 113466585e533c558bb11f1296113b16c1b00515 Mon Sep 17 00:00:00 2001 From: Hopding Date: Wed, 12 Apr 2017 21:17:11 -0500 Subject: [PATCH 02/24] Did some significant refactoring to compile.js; WIP --- __tests__/transformers/simplify_node.test.js | 7 + app/compile.js | 298 +++++++++++-------- app/transformers/simplify_node.js | 9 +- 3 files changed, 190 insertions(+), 124 deletions(-) create mode 100644 __tests__/transformers/simplify_node.test.js diff --git a/__tests__/transformers/simplify_node.test.js b/__tests__/transformers/simplify_node.test.js new file mode 100644 index 0000000..46ecee8 --- /dev/null +++ b/__tests__/transformers/simplify_node.test.js @@ -0,0 +1,7 @@ +describe(`simplify_node.js`, () => { + const simplify_node = require('../../app/transformers/simplify_node.js'); + + it(`is a function`, () => { + expect(typeof(simplify_node)).toEqual('function'); + }); +}); diff --git a/app/compile.js b/app/compile.js index 7204dd5..61ab66d 100644 --- a/app/compile.js +++ b/app/compile.js @@ -1,129 +1,187 @@ -let path = require('path'); -let fs = require('fs-promise'); -let child_process = require('child-process-promise'); -let antlr = require('antlr4'); - -let config = require('../config.js'); -let expect_error = require('./expect_error.js'); -let tree_matcher = require('./tree_matcher.js'); -let java_func_data_generator = require('./java_func_data_generator.js'); - -let array_diff = function(a, b) { - return a.filter(function(i) {return b.indexOf(i) === -1;}); -}; - -module.exports = function(lang_compile_config, lang_runtime_config) { +const path = require('path'); +const fs = require('fs-promise'); +const child_process = require('child-process-promise'); +const antlr = require('antlr4'); + +const config = require('../config.js'); +const expect_error = require('./expect_error.js'); +const tree_matcher = require('./tree_matcher.js'); +const java_func_data_generator = require('./java_func_data_generator.js'); + +const array_diff = (a, b) => a.filter(i => b.indexOf(i) === -1); + +const resolve_grammar_path = (lang_key, grammar_file) => + path.resolve(__dirname, '..', 'grammars-v4', lang_key, grammar_file); + +// Throws an error if the generated parser does not have same +// rules as the specified config file. +const ensure_rules_match = (lang_runtime_config) => { + const { + language, + rules, + } = lang_runtime_config; + + // Create instance of the parser + const parser_classname = language + 'Parser'; + const ParserClass = require(`${cache_dir}/${parser_classname}.js`)[parser_classname] + const parser = new ParserClass(); + + // Create an array of symbol (terminal) names + const symbol_name_map = ['_EPSILON', '_EOF', '_INVALID'] + .concat(parser.symbolicNames.slice(1)) + .map((val) => val ? '.' + val : undefined); + + // Create the lists of rule names (both terminals and non-terminals) + const parser_rules = parser.ruleNames.concat(symbol_name_map.filter(Boolean)); + const config_rules = Object.keys(rules); + + // Make sure the parser doesn't have extra rules + const config_missing = array_diff(parser_rules, config_rules); + if (config_missing.length) { + throw new Error('Missing rules ' + JSON.stringify(config_missing)); + } + + // Make sure our config doesn't have extra rules + const config_extra = array_diff(config_rules, parser_rules); + if (config_extra.length) { + throw new Error('Extra rules ' + JSON.stringify(config_extra)); + } + + return { symbol_name_map, parser }; +} + +const generate_runtime_config_modifier = ( + lang_compile_config, + lang_runtime_config, + symbol_name_map, + parser +) => { + // Turns these maps into human-readable JSON for insertion + // into returned function string + const symbol_name_map = JSON.stringify(symbol_name_map, null, 2); + const rule_name_map = JSON.stringify(parser.ruleNames, null, 2); + + const { tree_matcher_specs } = lang_compile_config; + let tree_matchers; + if (tree_matcher_specs) { + const generator = await tree_matcher.make_generator( + lang_compile_config, + lang_runtime_config + ); + tree_matchers = tree_matcher_specs.map(generator); + } + + // Return the runtime config modifier - a function that + // modifies the lang_runtime_config + return ` + /* + This function is generated by app/compile.js + Do not attempt to make changes. They will be overwritten. + */ + module.exports = function(lang_runtime_config) { + lang_runtime_config.symbol_name_map = ${symbol_name_map}; + lang_runtime_config.rule_name_map = ${rule_name_map}; + ${ + // Add a tree matcher function if appropriate too + tree_matcher_specs ? + `lang_runtime_config.tree_matcher = function(root) { + ${tree_matchers.join('\n')} + };` + : + ''; + } + }; + `; +} + +// Invokes the antlr process. Returns a Promise. +const invokeAntlr = (lang_runtime_config, cache_dir) => { + const { + language, + generate_visitor, + generate_listener, + } = lang_runtime_config; + + // Prepare options to the antlr compiler that generates + // the antlr lexer and antlr parser + const cmd = 'java'; + const args = [ + '-Xmx500M', + '-cp', '../../bin/antlr-4.6-complete.jar', + 'org.antlr.v4.Tool', + '-long-messages', + generate_listener ? '-listener' : '-no-listener', + generate_visitor ? '-visitor' : '-no-visitor', + '-Dlanguage=JavaScript', + language + '.g4', + ]; + const opts = { + 'cwd': cache_dir, + 'stdio': ['ignore', process.stdout, process.stderr], + }; + + return child_process.spawn(cmd, args, opts); +} + +module.exports = (lang_compile_config, lang_runtime_config) => { // Figure out the language key - let language_key = lang_runtime_config.language.toLowerCase(); + const language_key = lang_runtime_config.language.toLowerCase(); // Figure out the path to the grammar file - let g4_path = lang_compile_config.grammar_path; - if (!g4_path) { - g4_path = path.resolve(__dirname, '..', 'grammars-v4', language_key, lang_compile_config.grammar_file); - } + const { + grammar_path, + grammar_file, + needs_java_func_data, + } = lang_compile_config; + + const g4_path = grammar_path ? grammar_path : + resolve_grammar_path(language_key, grammar_file); // Figure out the path to the cache directory - let cache_dir = config.resolve_cache_dir(lang_runtime_config); - let cache_g4_path = path.resolve(cache_dir, lang_runtime_config.language + '.g4'); - - - let compile_promise = async function() { - // Make sure the cache directory exists - await fs.mkdir(config.cache_path).catch(expect_error('EEXIST', function() {})); - - // Make sure the language cache directory exists - await fs.mkdir(cache_dir).catch(expect_error('EEXIST', function() {})); - - // Copies the g4 file into the cache directory - await fs.copy(g4_path, cache_g4_path); - - // Prepare options to the antlr compiler that generates the antlr lexer and antlr parser - let cmd = 'java'; - let args = [ - '-Xmx500M', - '-cp', '../../bin/antlr-4.6-complete.jar', - 'org.antlr.v4.Tool', - '-long-messages', - lang_compile_config.generate_listener ? '-listener' : '-no-listener', - lang_compile_config.generate_visitor ? '-visitor' : '-no-visitor', - '-Dlanguage=JavaScript', - lang_runtime_config.language + '.g4', - ]; - let opts = { - 'cwd': cache_dir, - 'stdio': ['ignore', process.stdout, process.stderr], - }; - - // Call antlr - await child_process.spawn(cmd, args, opts); - - if (lang_compile_config.needs_java_func_data) { - await fs.stat(config.cache_path + '/java_func_data') - .catch(expect_error('ENOENT', java_func_data_generator)); - } - - // Make sure the generated parser has the same rules as our config file. - let parser_classname = lang_runtime_config.language + 'Parser'; - let ParserClass = require(cache_dir + '/' + parser_classname + '.js')[parser_classname]; - let parser = new ParserClass(); - - // Create an array of symbol (terminal) names - let symbol_name_map = ['_EPSILON', '_EOF', '_INVALID'] - .concat(parser.symbolicNames.slice(1)) - .map(function(val) {return val ? '.' + val : undefined;}); - - // Create the list of rule names (both terminals and non-terminals) - let parser_rules = parser.ruleNames.concat(symbol_name_map.filter(Boolean)); - let config_rules = Object.keys(lang_runtime_config.rules); - - // Make sure the parser doesn't have extra rules - let config_missing = array_diff(parser_rules, config_rules); - if (config_missing.length) { - throw new Error('Missing rules ' + JSON.stringify(config_missing)); - } - - // Make sure our config doesn't have extra rules - let config_extra = array_diff(config_rules, parser_rules); - if (config_extra.length) { - throw new Error('Extra rules ' + JSON.stringify(config_extra)); - } - - // Generate the runtime config modifier - let code = ''; - code += '// This function is generated by app/compile.js.\n'; - code += '// Do not attempt to make changes. They will be over-written.\n\n'; - - // This is a function that modifies the lang_runtime_config - code += 'module.exports = function(lang_runtime_config) {\n'; - - // It adds a symbol_name_map array - code += 'lang_runtime_config.symbol_name_map = ' + JSON.stringify(symbol_name_map, null, 2) + ';\n'; - - // And a rule_name_map array - code += 'lang_runtime_config.rule_name_map = ' + JSON.stringify(parser.ruleNames, null, 2) + ';\n'; - - // And a tree_matcher function - if (lang_compile_config.tree_matcher_specs) { - let generator = await tree_matcher.make_generator(lang_compile_config, lang_runtime_config); - let tree_matchers = lang_compile_config.tree_matcher_specs.map(generator); - code += 'lang_runtime_config.tree_matcher = function(root) {\n' + tree_matchers.join('\n') + '\n};\n'; - } - - code += '};'; - - // Write the runtime config modifier - let modifier_path = path.resolve(cache_dir, 'runtime_config_modifier.js'); - await fs.writeFile(modifier_path, code); + const cache_dir = config.resolve_cache_dir(lang_runtime_config); + const cache_g4_path = path.resolve(cache_dir, language + '.g4'); + + const compile_promise = async () => { + const on_eexist_err = expect_error('EEXIST', () => {}); + + // Make sure the cache directory exists + await fs.mkdir(config.cache_path).catch(on_eexist_err); + + // Make sure the language cache directory exists + await fs.mkdir(cache_dir).catch(on_eexist_err); + + // Copies the g4 file into the cache directory + await fs.copy(g4_path, cache_g4_path); + + await invokeAntlr(lang_runtime_config, cache_dir); + + if (needs_java_func_data) { + await fs.stat(config.cache_path + '/java_func_data') + .catch(expect_error('ENOENT', java_func_data_generator)); + } + + const { + symbol_name_map, + parser + } = ensure_rules_match(lang_runtime_config); + + const runtime_config_modifier = generate_runtime_config_modifier( + lang_compile_config, + lang_runtime_config, + symbol_name_map, + parser + ); + + // Write the runtime config modifier + const modifier_path = path.resolve(cache_dir, 'runtime_config_modifier.js'); + await fs.writeFile(modifier_path, runtime_config_modifier); }; - // Stat the cache directory, which is the standard way if checking if it exists. + // Check if the cache directory exists using 'stat' return fs.stat(cache_dir) - .catch(expect_error('ENOENT', compile_promise)) - .then(function() { - // In either case, return an object describing the results. - return { - // Currently, this description is just where the compiled files are stored. - 'cache_dir': cache_dir - }; - }); + // If it does not exist, then build it up using compile_promise(). + .catch(expect_error('ENOENT', compile_promise)) + // In either case, return an object describing the results. + // Currently, this description is just where the compiled files are stored. + .then(() => ({ cache_dir })); }; diff --git a/app/transformers/simplify_node.js b/app/transformers/simplify_node.js index fe3d79d..ceb9f27 100644 --- a/app/transformers/simplify_node.js +++ b/app/transformers/simplify_node.js @@ -1,17 +1,18 @@ -let TerminalNodeImpl = require('antlr4/tree/Tree.js').TerminalNodeImpl; +const TerminalNodeImpl = require('antlr4/tree/Tree.js').TerminalNodeImpl; module.exports = function(lang_runtime_config, root) { - // Takes an antlr node generated by the antlr parser, and outputs our simplified node. - let simplify_node = function(node) { + // Takes an antlr node generated by the antlr parser, and + // outputs our simplified node. + const simplify_node = function(node) { if (node instanceof TerminalNodeImpl) { return { 'type': lang_runtime_config.symbol_name_map[node.symbol.type + 2], - 'text': node.symbol.text, 'begin': node.symbol.start, 'end': node.symbol.stop + 1, 'tags': [], 'children': [], + 'text': node.symbol.text, }; } else { return { From 59ff56140b4c8cb7670d1f5a945f64559d16210d Mon Sep 17 00:00:00 2001 From: Hopding Date: Thu, 13 Apr 2017 09:53:59 -0500 Subject: [PATCH 03/24] Working version of compile.js refactor --- app/compile.js | 272 ++++++++++++++++++++++++++----------------------- package.json | 9 +- yarn.lock | 4 + 3 files changed, 157 insertions(+), 128 deletions(-) diff --git a/app/compile.js b/app/compile.js index 61ab66d..1afd062 100644 --- a/app/compile.js +++ b/app/compile.js @@ -13,161 +13,178 @@ const array_diff = (a, b) => a.filter(i => b.indexOf(i) === -1); const resolve_grammar_path = (lang_key, grammar_file) => path.resolve(__dirname, '..', 'grammars-v4', lang_key, grammar_file); -// Throws an error if the generated parser does not have same -// rules as the specified config file. -const ensure_rules_match = (lang_runtime_config) => { +/* +Returns a set of async closure functions with references to the given +lang_compile_config & lang_runtime_config that can be used to build the +parser. +*/ +const make_config_closures = (lang_compile_config, lang_runtime_config) => { const { language, rules, + generate_visitor, + generate_listener, } = lang_runtime_config; + const { + tree_matcher_specs, + needs_java_func_data, + grammar_path, + grammar_file, + } = lang_compile_config; + + // Figure out the language key + const language_key = language.toLowerCase(); + + // Figure out the path to the grammar file + const g4_path = grammar_path ? grammar_path : + resolve_grammar_path(language_key, grammar_file); - // Create instance of the parser - const parser_classname = language + 'Parser'; - const ParserClass = require(`${cache_dir}/${parser_classname}.js`)[parser_classname] - const parser = new ParserClass(); + // Figure out the path to the cache directory + const cache_dir = config.resolve_cache_dir(lang_runtime_config); + const cache_g4_path = path.resolve(cache_dir, language + '.g4'); - // Create an array of symbol (terminal) names - const symbol_name_map = ['_EPSILON', '_EOF', '_INVALID'] - .concat(parser.symbolicNames.slice(1)) - .map((val) => val ? '.' + val : undefined); + /* --- Creates cache directory and copies grammar files into it. --- */ + const prepare_cache_dir = async () => { + const on_eexist_err = expect_error('EEXIST', () => {}); - // Create the lists of rule names (both terminals and non-terminals) - const parser_rules = parser.ruleNames.concat(symbol_name_map.filter(Boolean)); - const config_rules = Object.keys(rules); + // Make sure the cache directory exists + await fs.mkdir(config.cache_path).catch(on_eexist_err); - // Make sure the parser doesn't have extra rules - const config_missing = array_diff(parser_rules, config_rules); - if (config_missing.length) { - throw new Error('Missing rules ' + JSON.stringify(config_missing)); + // Make sure the language cache directory exists + await fs.mkdir(cache_dir).catch(on_eexist_err); + + // Copies the g4 file into the cache directory + await fs.copy(g4_path, cache_g4_path); } - // Make sure our config doesn't have extra rules - const config_extra = array_diff(config_rules, parser_rules); - if (config_extra.length) { - throw new Error('Extra rules ' + JSON.stringify(config_extra)); + /* --- Invokes the antlr process, returns a Promise --- */ + const invokeAntlr = async () => { + // Prepare options to the antlr compiler that generates + // the antlr lexer and antlr parser + const cmd = 'java'; + const args = [ + '-Xmx500M', + '-cp', '../../bin/antlr-4.6-complete.jar', + 'org.antlr.v4.Tool', + '-long-messages', + generate_listener ? '-listener' : '-no-listener', + generate_visitor ? '-visitor' : '-no-visitor', + '-Dlanguage=JavaScript', + language + '.g4', + ]; + const opts = { + 'cwd': cache_dir, + 'stdio': ['ignore', process.stdout, process.stderr], + }; + + return child_process.spawn(cmd, args, opts); } - return { symbol_name_map, parser }; -} + /* --- Generates java func data, if this lang needs any --- */ + const make_java_func_data = async () => { + if (needs_java_func_data) { + await fs.stat(config.cache_path + '/java_func_data') + .catch(expect_error('ENOENT', java_func_data_generator)); + } + } + + /* --- Creates and returns the parser and symbol_name_map --- */ + const make_parser = async () => { + // Create instance of the parser + const parser_classname = language + 'Parser'; + const ParserClass = require(`${cache_dir}/${parser_classname}.js`)[parser_classname] + const parser = new ParserClass(); + + // Create an array of symbol (terminal) names + const symbol_name_map = ['_EPSILON', '_EOF', '_INVALID'] + .concat(parser.symbolicNames.slice(1)) + .map((val) => val ? '.' + val : undefined); + + // Create the lists of rule names (both terminals and non-terminals) + const parser_rules = parser.ruleNames.concat(symbol_name_map.filter(Boolean)); + const config_rules = Object.keys(rules); + + // Make sure the parser doesn't have extra rules + const config_missing = array_diff(parser_rules, config_rules); + if (config_missing.length) { + throw new Error('Missing rules ' + JSON.stringify(config_missing)); + } + + // Make sure our config doesn't have extra rules + const config_extra = array_diff(config_rules, parser_rules); + if (config_extra.length) { + throw new Error('Extra rules ' + JSON.stringify(config_extra)); + } -const generate_runtime_config_modifier = ( - lang_compile_config, - lang_runtime_config, - symbol_name_map, - parser -) => { - // Turns these maps into human-readable JSON for insertion - // into returned function string - const symbol_name_map = JSON.stringify(symbol_name_map, null, 2); - const rule_name_map = JSON.stringify(parser.ruleNames, null, 2); - - const { tree_matcher_specs } = lang_compile_config; - let tree_matchers; - if (tree_matcher_specs) { - const generator = await tree_matcher.make_generator( - lang_compile_config, - lang_runtime_config - ); - tree_matchers = tree_matcher_specs.map(generator); + return { symbol_name_map, parser }; } - // Return the runtime config modifier - a function that - // modifies the lang_runtime_config - return ` - /* - This function is generated by app/compile.js - Do not attempt to make changes. They will be overwritten. - */ - module.exports = function(lang_runtime_config) { - lang_runtime_config.symbol_name_map = ${symbol_name_map}; - lang_runtime_config.rule_name_map = ${rule_name_map}; - ${ - // Add a tree matcher function if appropriate too - tree_matcher_specs ? + /* --- Dynamically creates runtime config modifier function --- */ + const make_runtime_config_modifier = async (symbol_name_map, parser) => { + // Turns these maps into human-readable JSON for insertion + // into returned function string + const symbol_name_map_str = JSON.stringify(symbol_name_map, null, 2); + const rule_name_map_str = JSON.stringify(parser.ruleNames, null, 2); + + // Add a tree matcher function if appropriate too + let tree_matchers_str = ''; + if (tree_matcher_specs) { + const generator = await tree_matcher.make_generator( + lang_compile_config, + lang_runtime_config + ); + const tree_matchers = tree_matcher_specs.map(generator); + tree_matchers_str = `lang_runtime_config.tree_matcher = function(root) { ${tree_matchers.join('\n')} - };` - : - ''; + };`; } - }; - `; -} -// Invokes the antlr process. Returns a Promise. -const invokeAntlr = (lang_runtime_config, cache_dir) => { - const { - language, - generate_visitor, - generate_listener, - } = lang_runtime_config; + // Return the runtime config modifier - a function that + // modifies the lang_runtime_config + return ` + /* + This function is generated by app/compile.js + Do not attempt to make changes. They will be overwritten. + */ + module.exports = function(lang_runtime_config) { + lang_runtime_config.symbol_name_map = ${symbol_name_map_str}; + lang_runtime_config.rule_name_map = ${rule_name_map_str}; + ${tree_matchers_str} + }; + `; + } - // Prepare options to the antlr compiler that generates - // the antlr lexer and antlr parser - const cmd = 'java'; - const args = [ - '-Xmx500M', - '-cp', '../../bin/antlr-4.6-complete.jar', - 'org.antlr.v4.Tool', - '-long-messages', - generate_listener ? '-listener' : '-no-listener', - generate_visitor ? '-visitor' : '-no-visitor', - '-Dlanguage=JavaScript', - language + '.g4', - ]; - const opts = { - 'cwd': cache_dir, - 'stdio': ['ignore', process.stdout, process.stderr], - }; - - return child_process.spawn(cmd, args, opts); + // Return the set of async closures + return { + prepare_cache_dir, + invokeAntlr, + make_java_func_data, + make_parser, + make_runtime_config_modifier, + } } -module.exports = (lang_compile_config, lang_runtime_config) => { - // Figure out the language key - const language_key = lang_runtime_config.language.toLowerCase(); - - // Figure out the path to the grammar file - const { - grammar_path, - grammar_file, - needs_java_func_data, - } = lang_compile_config; - const g4_path = grammar_path ? grammar_path : - resolve_grammar_path(language_key, grammar_file); - - // Figure out the path to the cache directory - const cache_dir = config.resolve_cache_dir(lang_runtime_config); - const cache_g4_path = path.resolve(cache_dir, language + '.g4'); +module.exports = (lang_compile_config, lang_runtime_config) => { const compile_promise = async () => { - const on_eexist_err = expect_error('EEXIST', () => {}); - - // Make sure the cache directory exists - await fs.mkdir(config.cache_path).catch(on_eexist_err); - - // Make sure the language cache directory exists - await fs.mkdir(cache_dir).catch(on_eexist_err); - - // Copies the g4 file into the cache directory - await fs.copy(g4_path, cache_g4_path); - - await invokeAntlr(lang_runtime_config, cache_dir); - - if (needs_java_func_data) { - await fs.stat(config.cache_path + '/java_func_data') - .catch(expect_error('ENOENT', java_func_data_generator)); - } - + const { + prepare_cache_dir, + invokeAntlr, + make_java_func_data, + make_parser, + make_runtime_config_modifier, + } = make_config_closures(lang_compile_config, lang_runtime_config); + + await prepare_cache_dir(); + await invokeAntlr(); + await make_java_func_data(); const { symbol_name_map, parser - } = ensure_rules_match(lang_runtime_config); - - const runtime_config_modifier = generate_runtime_config_modifier( - lang_compile_config, - lang_runtime_config, + } = await make_parser(lang_runtime_config); + const runtime_config_modifier = await make_runtime_config_modifier( symbol_name_map, parser ); @@ -177,6 +194,7 @@ module.exports = (lang_compile_config, lang_runtime_config) => { await fs.writeFile(modifier_path, runtime_config_modifier); }; + const cache_dir = config.resolve_cache_dir(lang_runtime_config); // Check if the cache directory exists using 'stat' return fs.stat(cache_dir) // If it does not exist, then build it up using compile_promise(). diff --git a/package.json b/package.json index 6815205..b4a1b26 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "author": "Maryville Techonologies", "license": "ISC", "dependencies": { + "async-waterfall": "^0.1.5", "child-process-promise": "^2.2.0", "fs-promise": "^2.0.0", "webpack": "^2.2.1", @@ -21,6 +22,12 @@ "jest": "^19.0.2" }, "jest": { - "moduleFileExtensions": ["js", "json", "jsx", "node", "csv"] + "moduleFileExtensions": [ + "js", + "json", + "jsx", + "node", + "csv" + ] } } diff --git a/yarn.lock b/yarn.lock index e6eda4e..7f31c52 100644 --- a/yarn.lock +++ b/yarn.lock @@ -161,6 +161,10 @@ async-each@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/async-each/-/async-each-1.0.1.tgz#19d386a1d9edc6e7c1c85d388aedbcc56d33602d" +async-waterfall@^0.1.5: + version "0.1.5" + resolved "https://registry.yarnpkg.com/async-waterfall/-/async-waterfall-0.1.5.tgz#398bd48b0eac5d40ffbe400fe9e37a53ba966dae" + async@^1.4.0, async@^1.5.0: version "1.5.2" resolved "https://registry.yarnpkg.com/async/-/async-1.5.2.tgz#ec6a61ae56480c0c3cb241c95618e20892f9672a" From 73cf366ae531e25a7383b0ee1c773627b897f98b Mon Sep 17 00:00:00 2001 From: Hopding Date: Thu, 13 Apr 2017 10:34:17 -0500 Subject: [PATCH 04/24] Cleaner version of refactored compile.js --- app/compile.js | 51 +++++++++++++++++++++----------------------------- package.json | 1 - yarn.lock | 4 ---- 3 files changed, 21 insertions(+), 35 deletions(-) diff --git a/app/compile.js b/app/compile.js index 1afd062..81ba31a 100644 --- a/app/compile.js +++ b/app/compile.js @@ -13,12 +13,18 @@ const array_diff = (a, b) => a.filter(i => b.indexOf(i) === -1); const resolve_grammar_path = (lang_key, grammar_file) => path.resolve(__dirname, '..', 'grammars-v4', lang_key, grammar_file); +// Calls each function in 'tasks' with the results of the previous task's +// invocation. The first task is called with no args. +const waterfall = async (tasks) => { + tasks.reduce((acc, task) => task(acc), tasks[0]()); +}; + /* Returns a set of async closure functions with references to the given lang_compile_config & lang_runtime_config that can be used to build the parser. */ -const make_config_closures = (lang_compile_config, lang_runtime_config) => { +const make_build_tasks = (lang_compile_config, lang_runtime_config) => { const { language, rules, @@ -77,7 +83,7 @@ const make_config_closures = (lang_compile_config, lang_runtime_config) => { 'stdio': ['ignore', process.stdout, process.stderr], }; - return child_process.spawn(cmd, args, opts); + await child_process.spawn(cmd, args, opts); } /* --- Generates java func data, if this lang needs any --- */ @@ -120,7 +126,7 @@ const make_config_closures = (lang_compile_config, lang_runtime_config) => { } /* --- Dynamically creates runtime config modifier function --- */ - const make_runtime_config_modifier = async (symbol_name_map, parser) => { + const make_runtime_config_modifier = async ({ symbol_name_map, parser }) => { // Turns these maps into human-readable JSON for insertion // into returned function string const symbol_name_map_str = JSON.stringify(symbol_name_map, null, 2); @@ -155,44 +161,29 @@ const make_config_closures = (lang_compile_config, lang_runtime_config) => { `; } + /* --- Writes the runtime config modifier to file system --- */ + const write_runtime_config_modifier = async ({ config_modifier }) => { + const modifier_path = path.resolve(cache_dir, 'runtime_config_modifier.js'); + await fs.writeFile(modifier_path, config_modifier); + } + // Return the set of async closures - return { + return [ prepare_cache_dir, invokeAntlr, make_java_func_data, make_parser, make_runtime_config_modifier, - } + write_runtime_config_modifier, + ]; } module.exports = (lang_compile_config, lang_runtime_config) => { - const compile_promise = async () => { - const { - prepare_cache_dir, - invokeAntlr, - make_java_func_data, - make_parser, - make_runtime_config_modifier, - } = make_config_closures(lang_compile_config, lang_runtime_config); - - await prepare_cache_dir(); - await invokeAntlr(); - await make_java_func_data(); - const { - symbol_name_map, - parser - } = await make_parser(lang_runtime_config); - const runtime_config_modifier = await make_runtime_config_modifier( - symbol_name_map, - parser - ); - - // Write the runtime config modifier - const modifier_path = path.resolve(cache_dir, 'runtime_config_modifier.js'); - await fs.writeFile(modifier_path, runtime_config_modifier); - }; + const compile_promise = async () => { + await waterfall(make_build_tasks(lang_compile_config, lang_runtime_config)); + }; const cache_dir = config.resolve_cache_dir(lang_runtime_config); // Check if the cache directory exists using 'stat' diff --git a/package.json b/package.json index b4a1b26..e51daea 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,6 @@ "author": "Maryville Techonologies", "license": "ISC", "dependencies": { - "async-waterfall": "^0.1.5", "child-process-promise": "^2.2.0", "fs-promise": "^2.0.0", "webpack": "^2.2.1", diff --git a/yarn.lock b/yarn.lock index 7f31c52..e6eda4e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -161,10 +161,6 @@ async-each@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/async-each/-/async-each-1.0.1.tgz#19d386a1d9edc6e7c1c85d388aedbcc56d33602d" -async-waterfall@^0.1.5: - version "0.1.5" - resolved "https://registry.yarnpkg.com/async-waterfall/-/async-waterfall-0.1.5.tgz#398bd48b0eac5d40ffbe400fe9e37a53ba966dae" - async@^1.4.0, async@^1.5.0: version "1.5.2" resolved "https://registry.yarnpkg.com/async/-/async-1.5.2.tgz#ec6a61ae56480c0c3cb241c95618e20892f9672a" From 9f9979e19f51304956593042fb76d8ab1f784854 Mon Sep 17 00:00:00 2001 From: Hopding Date: Thu, 13 Apr 2017 13:21:07 -0500 Subject: [PATCH 05/24] Fixed bugs in compile.js refactor --- app/compile.js | 42 +++++++++++++++++++++--------------------- make | 1 + 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/app/compile.js b/app/compile.js index 81ba31a..2597515 100644 --- a/app/compile.js +++ b/app/compile.js @@ -13,10 +13,16 @@ const array_diff = (a, b) => a.filter(i => b.indexOf(i) === -1); const resolve_grammar_path = (lang_key, grammar_file) => path.resolve(__dirname, '..', 'grammars-v4', lang_key, grammar_file); -// Calls each function in 'tasks' with the results of the previous task's -// invocation. The first task is called with no args. +/* +Sequentially invokes each task in `tasks` using the previous tasks's +return value as its argument. `tasks` must be an array of async functions. +*/ const waterfall = async (tasks) => { - tasks.reduce((acc, task) => task(acc), tasks[0]()); + let args = null; + for(let i = 0; i < tasks.length; i++) { + args = await tasks[i](args); + } + return args; }; /* @@ -24,7 +30,7 @@ Returns a set of async closure functions with references to the given lang_compile_config & lang_runtime_config that can be used to build the parser. */ -const make_build_tasks = (lang_compile_config, lang_runtime_config) => { +const build_tasks = (lang_compile_config, lang_runtime_config) => { const { language, rules, @@ -49,6 +55,7 @@ const make_build_tasks = (lang_compile_config, lang_runtime_config) => { const cache_dir = config.resolve_cache_dir(lang_runtime_config); const cache_g4_path = path.resolve(cache_dir, language + '.g4'); + /* ============================ Build Tasks: ============================== */ /* --- Creates cache directory and copies grammar files into it. --- */ const prepare_cache_dir = async () => { const on_eexist_err = expect_error('EEXIST', () => {}); @@ -64,7 +71,7 @@ const make_build_tasks = (lang_compile_config, lang_runtime_config) => { } /* --- Invokes the antlr process, returns a Promise --- */ - const invokeAntlr = async () => { + const invoke_antlr = async () => { // Prepare options to the antlr compiler that generates // the antlr lexer and antlr parser const cmd = 'java'; @@ -162,35 +169,28 @@ const make_build_tasks = (lang_compile_config, lang_runtime_config) => { } /* --- Writes the runtime config modifier to file system --- */ - const write_runtime_config_modifier = async ({ config_modifier }) => { + const write_runtime_config_modifier = async (config_modifier) => { const modifier_path = path.resolve(cache_dir, 'runtime_config_modifier.js'); await fs.writeFile(modifier_path, config_modifier); } + /* --- Returns the cache_dir wrapped in an object --- */ + const final_task = async () => { + return { cache_dir }; + } + // Return the set of async closures return [ prepare_cache_dir, - invokeAntlr, + invoke_antlr, make_java_func_data, make_parser, make_runtime_config_modifier, write_runtime_config_modifier, + final_task, ]; } - - module.exports = (lang_compile_config, lang_runtime_config) => { - const compile_promise = async () => { - await waterfall(make_build_tasks(lang_compile_config, lang_runtime_config)); - }; - - const cache_dir = config.resolve_cache_dir(lang_runtime_config); - // Check if the cache directory exists using 'stat' - return fs.stat(cache_dir) - // If it does not exist, then build it up using compile_promise(). - .catch(expect_error('ENOENT', compile_promise)) - // In either case, return an object describing the results. - // Currently, this description is just where the compiled files are stored. - .then(() => ({ cache_dir })); + return waterfall(build_tasks(lang_compile_config, lang_runtime_config)); }; diff --git a/make b/make index 56284d5..f15f695 100755 --- a/make +++ b/make @@ -48,5 +48,6 @@ while (( "$#" )); do esac done +rm -r ./_cache $NPM run build -- --env.langs="$LANG" --env.optimize=$MINIFY --env.enable_debug=$DEBUG exit From 5e8a391b33d5d21ec44ebea5cd3332d9c6a75fb0 Mon Sep 17 00:00:00 2001 From: Hopding Date: Thu, 13 Apr 2017 13:23:12 -0500 Subject: [PATCH 06/24] Renamed app -> src --- src/compile.js | 196 ++++++++++++++++++ {app => src}/error_listener.js | 0 {app => src}/expect_error.js | 0 {app => src}/java_func_data_generator.js | 0 .../java_function_translator/Encoder.java | 0 .../JavaFunctionTranslator.java | 0 .../RangeEncoder.java | 0 .../TranslatedFunction.java | 0 .../Character_isJavaIdentifierPart_int.java | 0 .../Character_isJavaIdentifierStart_int.class | Bin .../Character_isJavaIdentifierStart_int.java | 0 .../Character_isJavaIdentifierPart.js | 0 .../Character_isJavaIdentifierStart.js | 0 .../java_functions/Character_toCodePoint.js | 0 {app => src}/java_functions/range_decoder.js | 0 {app => src}/runtime.js | 0 {app => src}/transformers.js | 0 {app => src}/transformers/collapse.js | 0 .../transformers/run_tree_matchers.js | 0 src/transformers/simplify_node.js | 29 +++ {app => src}/tree_matcher.js | 0 .../tree_matcher_parser/TreeMatcher.g4 | 0 .../tree_matcher_parser/generator_listener.js | 0 .../lang_config.compile.js | 0 .../lang_config.runtime.js | 0 {app => src}/type_graph_generator.js | 0 26 files changed, 225 insertions(+) create mode 100644 src/compile.js rename {app => src}/error_listener.js (100%) rename {app => src}/expect_error.js (100%) rename {app => src}/java_func_data_generator.js (100%) rename {app => src}/java_function_translator/Encoder.java (100%) rename {app => src}/java_function_translator/JavaFunctionTranslator.java (100%) rename {app => src}/java_function_translator/RangeEncoder.java (100%) rename {app => src}/java_function_translator/TranslatedFunction.java (100%) rename {app => src}/java_function_translator/functions/Character_isJavaIdentifierPart_int.java (100%) rename {app => src}/java_function_translator/functions/Character_isJavaIdentifierStart_int.class (100%) rename {app => src}/java_function_translator/functions/Character_isJavaIdentifierStart_int.java (100%) rename {app => src}/java_functions/Character_isJavaIdentifierPart.js (100%) rename {app => src}/java_functions/Character_isJavaIdentifierStart.js (100%) rename {app => src}/java_functions/Character_toCodePoint.js (100%) rename {app => src}/java_functions/range_decoder.js (100%) rename {app => src}/runtime.js (100%) rename {app => src}/transformers.js (100%) rename {app => src}/transformers/collapse.js (100%) rename {app => src}/transformers/run_tree_matchers.js (100%) create mode 100644 src/transformers/simplify_node.js rename {app => src}/tree_matcher.js (100%) rename {app => src}/tree_matcher_parser/TreeMatcher.g4 (100%) rename {app => src}/tree_matcher_parser/generator_listener.js (100%) rename {app => src}/tree_matcher_parser/lang_config.compile.js (100%) rename {app => src}/tree_matcher_parser/lang_config.runtime.js (100%) rename {app => src}/type_graph_generator.js (100%) diff --git a/src/compile.js b/src/compile.js new file mode 100644 index 0000000..2597515 --- /dev/null +++ b/src/compile.js @@ -0,0 +1,196 @@ +const path = require('path'); +const fs = require('fs-promise'); +const child_process = require('child-process-promise'); +const antlr = require('antlr4'); + +const config = require('../config.js'); +const expect_error = require('./expect_error.js'); +const tree_matcher = require('./tree_matcher.js'); +const java_func_data_generator = require('./java_func_data_generator.js'); + +const array_diff = (a, b) => a.filter(i => b.indexOf(i) === -1); + +const resolve_grammar_path = (lang_key, grammar_file) => + path.resolve(__dirname, '..', 'grammars-v4', lang_key, grammar_file); + +/* +Sequentially invokes each task in `tasks` using the previous tasks's +return value as its argument. `tasks` must be an array of async functions. +*/ +const waterfall = async (tasks) => { + let args = null; + for(let i = 0; i < tasks.length; i++) { + args = await tasks[i](args); + } + return args; +}; + +/* +Returns a set of async closure functions with references to the given +lang_compile_config & lang_runtime_config that can be used to build the +parser. +*/ +const build_tasks = (lang_compile_config, lang_runtime_config) => { + const { + language, + rules, + generate_visitor, + generate_listener, + } = lang_runtime_config; + const { + tree_matcher_specs, + needs_java_func_data, + grammar_path, + grammar_file, + } = lang_compile_config; + + // Figure out the language key + const language_key = language.toLowerCase(); + + // Figure out the path to the grammar file + const g4_path = grammar_path ? grammar_path : + resolve_grammar_path(language_key, grammar_file); + + // Figure out the path to the cache directory + const cache_dir = config.resolve_cache_dir(lang_runtime_config); + const cache_g4_path = path.resolve(cache_dir, language + '.g4'); + + /* ============================ Build Tasks: ============================== */ + /* --- Creates cache directory and copies grammar files into it. --- */ + const prepare_cache_dir = async () => { + const on_eexist_err = expect_error('EEXIST', () => {}); + + // Make sure the cache directory exists + await fs.mkdir(config.cache_path).catch(on_eexist_err); + + // Make sure the language cache directory exists + await fs.mkdir(cache_dir).catch(on_eexist_err); + + // Copies the g4 file into the cache directory + await fs.copy(g4_path, cache_g4_path); + } + + /* --- Invokes the antlr process, returns a Promise --- */ + const invoke_antlr = async () => { + // Prepare options to the antlr compiler that generates + // the antlr lexer and antlr parser + const cmd = 'java'; + const args = [ + '-Xmx500M', + '-cp', '../../bin/antlr-4.6-complete.jar', + 'org.antlr.v4.Tool', + '-long-messages', + generate_listener ? '-listener' : '-no-listener', + generate_visitor ? '-visitor' : '-no-visitor', + '-Dlanguage=JavaScript', + language + '.g4', + ]; + const opts = { + 'cwd': cache_dir, + 'stdio': ['ignore', process.stdout, process.stderr], + }; + + await child_process.spawn(cmd, args, opts); + } + + /* --- Generates java func data, if this lang needs any --- */ + const make_java_func_data = async () => { + if (needs_java_func_data) { + await fs.stat(config.cache_path + '/java_func_data') + .catch(expect_error('ENOENT', java_func_data_generator)); + } + } + + /* --- Creates and returns the parser and symbol_name_map --- */ + const make_parser = async () => { + // Create instance of the parser + const parser_classname = language + 'Parser'; + const ParserClass = require(`${cache_dir}/${parser_classname}.js`)[parser_classname] + const parser = new ParserClass(); + + // Create an array of symbol (terminal) names + const symbol_name_map = ['_EPSILON', '_EOF', '_INVALID'] + .concat(parser.symbolicNames.slice(1)) + .map((val) => val ? '.' + val : undefined); + + // Create the lists of rule names (both terminals and non-terminals) + const parser_rules = parser.ruleNames.concat(symbol_name_map.filter(Boolean)); + const config_rules = Object.keys(rules); + + // Make sure the parser doesn't have extra rules + const config_missing = array_diff(parser_rules, config_rules); + if (config_missing.length) { + throw new Error('Missing rules ' + JSON.stringify(config_missing)); + } + + // Make sure our config doesn't have extra rules + const config_extra = array_diff(config_rules, parser_rules); + if (config_extra.length) { + throw new Error('Extra rules ' + JSON.stringify(config_extra)); + } + + return { symbol_name_map, parser }; + } + + /* --- Dynamically creates runtime config modifier function --- */ + const make_runtime_config_modifier = async ({ symbol_name_map, parser }) => { + // Turns these maps into human-readable JSON for insertion + // into returned function string + const symbol_name_map_str = JSON.stringify(symbol_name_map, null, 2); + const rule_name_map_str = JSON.stringify(parser.ruleNames, null, 2); + + // Add a tree matcher function if appropriate too + let tree_matchers_str = ''; + if (tree_matcher_specs) { + const generator = await tree_matcher.make_generator( + lang_compile_config, + lang_runtime_config + ); + const tree_matchers = tree_matcher_specs.map(generator); + tree_matchers_str = + `lang_runtime_config.tree_matcher = function(root) { + ${tree_matchers.join('\n')} + };`; + } + + // Return the runtime config modifier - a function that + // modifies the lang_runtime_config + return ` + /* + This function is generated by app/compile.js + Do not attempt to make changes. They will be overwritten. + */ + module.exports = function(lang_runtime_config) { + lang_runtime_config.symbol_name_map = ${symbol_name_map_str}; + lang_runtime_config.rule_name_map = ${rule_name_map_str}; + ${tree_matchers_str} + }; + `; + } + + /* --- Writes the runtime config modifier to file system --- */ + const write_runtime_config_modifier = async (config_modifier) => { + const modifier_path = path.resolve(cache_dir, 'runtime_config_modifier.js'); + await fs.writeFile(modifier_path, config_modifier); + } + + /* --- Returns the cache_dir wrapped in an object --- */ + const final_task = async () => { + return { cache_dir }; + } + + // Return the set of async closures + return [ + prepare_cache_dir, + invoke_antlr, + make_java_func_data, + make_parser, + make_runtime_config_modifier, + write_runtime_config_modifier, + final_task, + ]; +} + +module.exports = (lang_compile_config, lang_runtime_config) => { + return waterfall(build_tasks(lang_compile_config, lang_runtime_config)); +}; diff --git a/app/error_listener.js b/src/error_listener.js similarity index 100% rename from app/error_listener.js rename to src/error_listener.js diff --git a/app/expect_error.js b/src/expect_error.js similarity index 100% rename from app/expect_error.js rename to src/expect_error.js diff --git a/app/java_func_data_generator.js b/src/java_func_data_generator.js similarity index 100% rename from app/java_func_data_generator.js rename to src/java_func_data_generator.js diff --git a/app/java_function_translator/Encoder.java b/src/java_function_translator/Encoder.java similarity index 100% rename from app/java_function_translator/Encoder.java rename to src/java_function_translator/Encoder.java diff --git a/app/java_function_translator/JavaFunctionTranslator.java b/src/java_function_translator/JavaFunctionTranslator.java similarity index 100% rename from app/java_function_translator/JavaFunctionTranslator.java rename to src/java_function_translator/JavaFunctionTranslator.java diff --git a/app/java_function_translator/RangeEncoder.java b/src/java_function_translator/RangeEncoder.java similarity index 100% rename from app/java_function_translator/RangeEncoder.java rename to src/java_function_translator/RangeEncoder.java diff --git a/app/java_function_translator/TranslatedFunction.java b/src/java_function_translator/TranslatedFunction.java similarity index 100% rename from app/java_function_translator/TranslatedFunction.java rename to src/java_function_translator/TranslatedFunction.java diff --git a/app/java_function_translator/functions/Character_isJavaIdentifierPart_int.java b/src/java_function_translator/functions/Character_isJavaIdentifierPart_int.java similarity index 100% rename from app/java_function_translator/functions/Character_isJavaIdentifierPart_int.java rename to src/java_function_translator/functions/Character_isJavaIdentifierPart_int.java diff --git a/app/java_function_translator/functions/Character_isJavaIdentifierStart_int.class b/src/java_function_translator/functions/Character_isJavaIdentifierStart_int.class similarity index 100% rename from app/java_function_translator/functions/Character_isJavaIdentifierStart_int.class rename to src/java_function_translator/functions/Character_isJavaIdentifierStart_int.class diff --git a/app/java_function_translator/functions/Character_isJavaIdentifierStart_int.java b/src/java_function_translator/functions/Character_isJavaIdentifierStart_int.java similarity index 100% rename from app/java_function_translator/functions/Character_isJavaIdentifierStart_int.java rename to src/java_function_translator/functions/Character_isJavaIdentifierStart_int.java diff --git a/app/java_functions/Character_isJavaIdentifierPart.js b/src/java_functions/Character_isJavaIdentifierPart.js similarity index 100% rename from app/java_functions/Character_isJavaIdentifierPart.js rename to src/java_functions/Character_isJavaIdentifierPart.js diff --git a/app/java_functions/Character_isJavaIdentifierStart.js b/src/java_functions/Character_isJavaIdentifierStart.js similarity index 100% rename from app/java_functions/Character_isJavaIdentifierStart.js rename to src/java_functions/Character_isJavaIdentifierStart.js diff --git a/app/java_functions/Character_toCodePoint.js b/src/java_functions/Character_toCodePoint.js similarity index 100% rename from app/java_functions/Character_toCodePoint.js rename to src/java_functions/Character_toCodePoint.js diff --git a/app/java_functions/range_decoder.js b/src/java_functions/range_decoder.js similarity index 100% rename from app/java_functions/range_decoder.js rename to src/java_functions/range_decoder.js diff --git a/app/runtime.js b/src/runtime.js similarity index 100% rename from app/runtime.js rename to src/runtime.js diff --git a/app/transformers.js b/src/transformers.js similarity index 100% rename from app/transformers.js rename to src/transformers.js diff --git a/app/transformers/collapse.js b/src/transformers/collapse.js similarity index 100% rename from app/transformers/collapse.js rename to src/transformers/collapse.js diff --git a/app/transformers/run_tree_matchers.js b/src/transformers/run_tree_matchers.js similarity index 100% rename from app/transformers/run_tree_matchers.js rename to src/transformers/run_tree_matchers.js diff --git a/src/transformers/simplify_node.js b/src/transformers/simplify_node.js new file mode 100644 index 0000000..ceb9f27 --- /dev/null +++ b/src/transformers/simplify_node.js @@ -0,0 +1,29 @@ +const TerminalNodeImpl = require('antlr4/tree/Tree.js').TerminalNodeImpl; + +module.exports = function(lang_runtime_config, root) { + + // Takes an antlr node generated by the antlr parser, and + // outputs our simplified node. + const simplify_node = function(node) { + if (node instanceof TerminalNodeImpl) { + return { + 'type': lang_runtime_config.symbol_name_map[node.symbol.type + 2], + 'begin': node.symbol.start, + 'end': node.symbol.stop + 1, + 'tags': [], + 'children': [], + 'text': node.symbol.text, + }; + } else { + return { + 'type': lang_runtime_config.rule_name_map[node.ruleIndex], + 'begin': node.start.start, + 'end': (node.stop ? node.stop : node.start).stop + 1, + 'tags': [], + 'children': node.children ? node.children.map(simplify_node) : [], + }; + } + }; + + return simplify_node(root); +}; diff --git a/app/tree_matcher.js b/src/tree_matcher.js similarity index 100% rename from app/tree_matcher.js rename to src/tree_matcher.js diff --git a/app/tree_matcher_parser/TreeMatcher.g4 b/src/tree_matcher_parser/TreeMatcher.g4 similarity index 100% rename from app/tree_matcher_parser/TreeMatcher.g4 rename to src/tree_matcher_parser/TreeMatcher.g4 diff --git a/app/tree_matcher_parser/generator_listener.js b/src/tree_matcher_parser/generator_listener.js similarity index 100% rename from app/tree_matcher_parser/generator_listener.js rename to src/tree_matcher_parser/generator_listener.js diff --git a/app/tree_matcher_parser/lang_config.compile.js b/src/tree_matcher_parser/lang_config.compile.js similarity index 100% rename from app/tree_matcher_parser/lang_config.compile.js rename to src/tree_matcher_parser/lang_config.compile.js diff --git a/app/tree_matcher_parser/lang_config.runtime.js b/src/tree_matcher_parser/lang_config.runtime.js similarity index 100% rename from app/tree_matcher_parser/lang_config.runtime.js rename to src/tree_matcher_parser/lang_config.runtime.js diff --git a/app/type_graph_generator.js b/src/type_graph_generator.js similarity index 100% rename from app/type_graph_generator.js rename to src/type_graph_generator.js From e706a98ba6d1b34f1808693ed091be3ed01263bf Mon Sep 17 00:00:00 2001 From: Hopding Date: Thu, 13 Apr 2017 13:31:00 -0500 Subject: [PATCH 07/24] Fixed imports after renaming --- config.js | 2 +- make | 9 ++++++--- webpack.config.js | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/config.js b/config.js index c2fceec..b780120 100644 --- a/config.js +++ b/config.js @@ -1,7 +1,7 @@ let path = require('path'); module.exports = { - 'app_path': path.resolve(__dirname, 'app'), + 'app_path': path.resolve(__dirname, 'src'), 'lang_configs_path': path.resolve(__dirname, 'language_configs'), 'cache_path': path.resolve(__dirname, '_cache'), 'resolve_cache_dir': function(lang_runtime_config) { diff --git a/make b/make index f15f695..539fa3b 100755 --- a/make +++ b/make @@ -1,5 +1,5 @@ #!/bin/bash -#Operate in current direcory +# Operate in current direcory cd $(dirname $0) # Variables @@ -17,7 +17,7 @@ function help { echo " -h|--help: Prints this help text" } -#Ensure npm installed +# Ensure npm installed NPM="$(which npm)" if [ "$NPM" == "" ]; then echo "npm not installed" @@ -48,6 +48,9 @@ while (( "$#" )); do esac done -rm -r ./_cache +if [[ -e "./_cache" ]]; then + rm -r ./_cache +fi + $NPM run build -- --env.langs="$LANG" --env.optimize=$MINIFY --env.enable_debug=$DEBUG exit diff --git a/webpack.config.js b/webpack.config.js index ae4b1d6..eb860d9 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -5,7 +5,7 @@ let ClosureCompilerPlugin = require('webpack-closure-compiler'); let config = require('./config.js'); -let compile = require('./app/compile.js'); +let compile = require('./src/compile.js'); let langs; let optimize; @@ -51,7 +51,7 @@ let prepare_lang = async function(filename) { 'context': __dirname, // The file to compile. All other files are included in this file or in files included from this file. - 'entry': path.resolve(__dirname, 'app', 'runtime.js'), + 'entry': path.resolve(__dirname, 'src', 'runtime.js'), 'resolve': { 'alias': { From 1b2bc49445b27a1e2caacf83937b75171a091d12 Mon Sep 17 00:00:00 2001 From: Hopding Date: Thu, 13 Apr 2017 13:36:44 -0500 Subject: [PATCH 08/24] Fixed imports after renaming --- config.js | 11 +++++------ src/compile.js | 36 ++++++++++++++++++------------------ 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/config.js b/config.js index b780120..e9d7e17 100644 --- a/config.js +++ b/config.js @@ -1,11 +1,10 @@ let path = require('path'); module.exports = { - 'app_path': path.resolve(__dirname, 'src'), + 'app_path': path.resolve(__dirname, 'src'), 'lang_configs_path': path.resolve(__dirname, 'language_configs'), - 'cache_path': path.resolve(__dirname, '_cache'), - 'resolve_cache_dir': function(lang_runtime_config) { - let language_key = lang_runtime_config.language.toLowerCase(); - return path.resolve(this.cache_path, language_key); - }, + 'build_path': path.resolve(__dirname, 'build'), + 'resolve_build_dir': ({ language }) => ( + path.resolve(this.build_path, language.toLowerCase()) + ), }; diff --git a/src/compile.js b/src/compile.js index 2597515..c5e2b39 100644 --- a/src/compile.js +++ b/src/compile.js @@ -51,23 +51,23 @@ const build_tasks = (lang_compile_config, lang_runtime_config) => { const g4_path = grammar_path ? grammar_path : resolve_grammar_path(language_key, grammar_file); - // Figure out the path to the cache directory - const cache_dir = config.resolve_cache_dir(lang_runtime_config); - const cache_g4_path = path.resolve(cache_dir, language + '.g4'); + // Figure out the path to the build directory + const build = config.resolve_build_dir(lang_runtime_config); + const build_g4_path = path.resolve(build_dir, language + '.g4'); /* ============================ Build Tasks: ============================== */ - /* --- Creates cache directory and copies grammar files into it. --- */ - const prepare_cache_dir = async () => { + /* --- Creates build directory and copies grammar files into it. --- */ + const prepare_build_dir = async () => { const on_eexist_err = expect_error('EEXIST', () => {}); - // Make sure the cache directory exists - await fs.mkdir(config.cache_path).catch(on_eexist_err); + // Make sure the build directory exists + await fs.mkdir(config.build_path).catch(on_eexist_err); - // Make sure the language cache directory exists - await fs.mkdir(cache_dir).catch(on_eexist_err); + // Make sure the language build directory exists + await fs.mkdir(build_dir).catch(on_eexist_err); - // Copies the g4 file into the cache directory - await fs.copy(g4_path, cache_g4_path); + // Copies the g4 file into the build directory + await fs.copy(g4_path, build_g4_path); } /* --- Invokes the antlr process, returns a Promise --- */ @@ -86,7 +86,7 @@ const build_tasks = (lang_compile_config, lang_runtime_config) => { language + '.g4', ]; const opts = { - 'cwd': cache_dir, + 'cwd': build_dir, 'stdio': ['ignore', process.stdout, process.stderr], }; @@ -96,7 +96,7 @@ const build_tasks = (lang_compile_config, lang_runtime_config) => { /* --- Generates java func data, if this lang needs any --- */ const make_java_func_data = async () => { if (needs_java_func_data) { - await fs.stat(config.cache_path + '/java_func_data') + await fs.stat(config.build_path + '/java_func_data') .catch(expect_error('ENOENT', java_func_data_generator)); } } @@ -105,7 +105,7 @@ const build_tasks = (lang_compile_config, lang_runtime_config) => { const make_parser = async () => { // Create instance of the parser const parser_classname = language + 'Parser'; - const ParserClass = require(`${cache_dir}/${parser_classname}.js`)[parser_classname] + const ParserClass = require(`${build_dir}/${parser_classname}.js`)[parser_classname] const parser = new ParserClass(); // Create an array of symbol (terminal) names @@ -170,18 +170,18 @@ const build_tasks = (lang_compile_config, lang_runtime_config) => { /* --- Writes the runtime config modifier to file system --- */ const write_runtime_config_modifier = async (config_modifier) => { - const modifier_path = path.resolve(cache_dir, 'runtime_config_modifier.js'); + const modifier_path = path.resolve(build_dir, 'runtime_config_modifier.js'); await fs.writeFile(modifier_path, config_modifier); } - /* --- Returns the cache_dir wrapped in an object --- */ + /* --- Returns the build_dir wrapped in an object --- */ const final_task = async () => { - return { cache_dir }; + return { build_dir }; } // Return the set of async closures return [ - prepare_cache_dir, + prepare_build_dir, invoke_antlr, make_java_func_data, make_parser, From 2869c94bcb81d8a5ce70cefe1e2b9d2facb7d3de Mon Sep 17 00:00:00 2001 From: Hopding Date: Thu, 13 Apr 2017 14:38:54 -0500 Subject: [PATCH 09/24] Renamed _cache dir to build --- .gitignore | 1 + config.js | 7 ++++--- make | 4 ++-- src/compile.js | 3 +-- src/runtime.js | 6 +++--- src/tree_matcher.js | 4 ++-- webpack.config.js | 4 ++-- 7 files changed, 15 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index 8047755..7228e88 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ _cache/ +build/ bin/ node_modules/ .DS_Store diff --git a/config.js b/config.js index e9d7e17..5b9eb1c 100644 --- a/config.js +++ b/config.js @@ -4,7 +4,8 @@ module.exports = { 'app_path': path.resolve(__dirname, 'src'), 'lang_configs_path': path.resolve(__dirname, 'language_configs'), 'build_path': path.resolve(__dirname, 'build'), - 'resolve_build_dir': ({ language }) => ( - path.resolve(this.build_path, language.toLowerCase()) - ), + 'resolve_build_dir': function(lang_runtime_config) { + const language_key = lang_runtime_config.language.toLowerCase(); + return path.resolve(this.build_path, language_key); + }, }; diff --git a/make b/make index 539fa3b..da77dd9 100755 --- a/make +++ b/make @@ -48,8 +48,8 @@ while (( "$#" )); do esac done -if [[ -e "./_cache" ]]; then - rm -r ./_cache +if [[ -e "./build" ]]; then + rm -r ./build fi $NPM run build -- --env.langs="$LANG" --env.optimize=$MINIFY --env.enable_debug=$DEBUG diff --git a/src/compile.js b/src/compile.js index c5e2b39..f7d91bb 100644 --- a/src/compile.js +++ b/src/compile.js @@ -51,8 +51,7 @@ const build_tasks = (lang_compile_config, lang_runtime_config) => { const g4_path = grammar_path ? grammar_path : resolve_grammar_path(language_key, grammar_file); - // Figure out the path to the build directory - const build = config.resolve_build_dir(lang_runtime_config); + const build_dir = config.resolve_build_dir(lang_runtime_config); const build_g4_path = path.resolve(build_dir, language + '.g4'); /* ============================ Build Tasks: ============================== */ diff --git a/src/runtime.js b/src/runtime.js index 8ba677d..2538a1a 100644 --- a/src/runtime.js +++ b/src/runtime.js @@ -3,14 +3,14 @@ let antlr = require('antlr4'); let lang_runtime_config = require('LangRuntimeConfig'); -require('LangCache/runtime_config_modifier.js')(lang_runtime_config); +require('LangBuild/runtime_config_modifier.js')(lang_runtime_config); let lexer_classname = lang_runtime_config.language + 'Lexer'; let parser_classname = lang_runtime_config.language + 'Parser'; // Loads the lexer and parser class generated by the antlr compiler. -let LexerClass = require('LangCache/' + lexer_classname + '.js')[lexer_classname]; -let ParserClass = require('LangCache/' + parser_classname + '.js')[parser_classname]; +let LexerClass = require('LangBuild/' + lexer_classname + '.js')[lexer_classname]; +let ParserClass = require('LangBuild/' + parser_classname + '.js')[parser_classname]; // Loads our custom error listener. let ErrorListener = require('App/error_listener'); diff --git a/src/tree_matcher.js b/src/tree_matcher.js index 451e01b..a34085b 100644 --- a/src/tree_matcher.js +++ b/src/tree_matcher.js @@ -25,8 +25,8 @@ module.exports.make_generator = async function(lang_compile_config, lang_runtime let lexer_classname = tree_matcher_runtime_config.language + 'Lexer'; let parser_classname = tree_matcher_runtime_config.language + 'Parser'; - let LexerClass = require(compile_result.cache_dir + '/' + lexer_classname + '.js')[lexer_classname]; - let ParserClass = require(compile_result.cache_dir + '/' + parser_classname + '.js')[parser_classname]; + let LexerClass = require(compile_result.build_dir + '/' + lexer_classname + '.js')[lexer_classname]; + let ParserClass = require(compile_result.build_dir + '/' + parser_classname + '.js')[parser_classname]; let ErrorListener = require('./error_listener'); // For use later, when the tree matcher is optimized diff --git a/webpack.config.js b/webpack.config.js index eb860d9..7626770 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -56,9 +56,9 @@ let prepare_lang = async function(filename) { 'resolve': { 'alias': { 'App': config.app_path, - 'Cache': config.cache_path, + 'Build': config.build_path, 'LangRuntimeConfig$': lang_runtime_config_path, - 'LangCache': compile_result.cache_dir, + 'LangBuild': compile_result.build_dir, }, }, From 33c871801c6f3b9cd54e9ab64052411bd41df8af Mon Sep 17 00:00:00 2001 From: Hopding Date: Thu, 13 Apr 2017 15:58:43 -0500 Subject: [PATCH 10/24] Relocated parser build output to /build dir; Updated publish script --- make | 2 +- public/index.html | 2 +- publish | 68 ++++++++++++++++++++++++++++++----------------- webpack.config.js | 2 +- 4 files changed, 47 insertions(+), 27 deletions(-) diff --git a/make b/make index da77dd9..c851eab 100755 --- a/make +++ b/make @@ -10,7 +10,7 @@ DEBUG=0 # Help function help { echo "Usage: $0 [-l|--lang lang] [-m|--minify] [-d|--debug] [-h|--help]" - echo " Makes the parser with the current configuration and places in './public/lang'" + echo " Makes the parser with the current configuration and places in './build/parsers'" echo " -l|--lang [lang]: Which language the parser is for" echo " -m|--minify: Optimizes the parser, will not be human-readable" echo " -d|--debug: Enables debugging info" diff --git a/public/index.html b/public/index.html index 2fc0910..6e5981c 100644 --- a/public/index.html +++ b/public/index.html @@ -141,7 +141,7 @@ let lang_el = document.getElementById('language'); lang_el.onchange = function() { - let path = 'langs/' + lang_el.value + '.js'; + let path = '../build/parsers/' + lang_el.value + '.js'; var script_el = document.createElement('script'); script_el.setAttribute('type', 'text/javascript'); diff --git a/publish b/publish index 03f8705..4b84ecb 100755 --- a/publish +++ b/publish @@ -1,48 +1,68 @@ #!/usr/bin/env bash +################################################################################ +# This script builds the parsers and pushes them to their S3 bucket. It can be # +# used in dev mode, to push to the dev bucket: # +# ./publish dev # +# or release mode, to push to a prod bucket with the specified release version:# +# ./publish release v0.1 # +################################################################################ + # Config begin -LANG_CONFIGS_DIR=language_configs/ -CACHE_DIR=_cache/ -OUTPUT_DIR=public/langs +LANG_CONFIGS_DIR="language_configs/" +BUILD_DIR="build/" MAPPINGS_DIR="mappings" + PUSH_ENV="$1" VERSION_TAG="$2" - # Config end +### Helper functions: +# Pushes parsers for specified language and their mappings to S3 +function pushToS3 { + printf "Pushing $1.js and $1.min.js to S3... " + aws s3 cp \ + $BUILD_DIR/parsers/$1.js \ + s3://codesplain-parsers/$1/$VERSION_TAG/$1.js + aws s3 cp \ + $BUILD_DIR/parsers/$1.min.js \ + s3://codesplain-parsers/$1/$VERSION_TAG/$1.min.js + aws s3 cp \ + $MAPPINGS_DIR/$1.csv \ + s3://codesplain-parsers/$1/$VERSION_TAG/$1.csv + printf "Done.\n" +} -if [ "$PUSH_ENV" == "release" ] -then - echo "Releasing Codesplain Parsers" -elif [ "$PUSH_ENV" == "dev" ] -then - echo "Updating Codesplain Parsers in dev environment" +# Builds the minified and non-minified parsers for the given language +function buildParsers { + printf "Building parsers for $1... " + ./make --lang $1 --debug >/dev/null + ./make --lang $1 --minify >/dev/null + printf "Done.\n" +} + +### Process command line args +if [[ "$PUSH_ENV" == "release" ]]; then + echo "### Releasing Codesplain Parsers ###" +elif [[ "$PUSH_ENV" == "dev" ]]; then + echo "### Updating Codesplain Parsers in dev environment ###" VERSION_TAG="dev" else echo "Error! Invalid PUSH_ENV" 1>&2 exit 1 fi - -rm -rf $CACHE_DIR/treematcher/ - # In the future we want to iterate over all language configs: # for lang_config in $LANG_CONFIGS_DIR/*.compile.js; do # But for now just do python3: -lang_config="$LANG_CONFIGS_DIR/python3.js" - -PARSELANG="$(basename $lang_config .js)" +lang_config="$BUILD_DIR/parsers/python3.js" -rm -rf "$CACHE_DIR/$PARSELANG/" -./make -d -./make -m +parser_lang="$(basename $lang_config .js)" - -aws s3 cp $OUTPUT_DIR/$PARSELANG.min.js s3://codesplain-parsers/$PARSELANG/$VERSION_TAG/$PARSELANG.min.js -aws s3 cp $OUTPUT_DIR/$PARSELANG.js s3://codesplain-parsers/$PARSELANG/$VERSION_TAG/$PARSELANG.js -aws s3 cp $MAPPINGS_DIR/$PARSELANG.csv s3://codesplain-parsers/$PARSELANG/$VERSION_TAG/$PARSELANG.csv +# Build parser and push to S3 +buildParsers $parser_lang +pushToS3 $parser_lang exit - # done diff --git a/webpack.config.js b/webpack.config.js index 7626770..1a4baa2 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -88,7 +88,7 @@ let prepare_lang = async function(filename) { 'filename': lang_name + (optimize ? '.min.js' : '.js'), // The output directory - 'path': path.resolve(__dirname, 'public', 'langs'), + 'path': path.resolve(__dirname, 'build', 'parsers'), // How to export the parser library // commonjs2 - Use module.exports From a4c39abb658e6821fd7e9c9f4fd476217bc907e2 Mon Sep 17 00:00:00 2001 From: Hopding Date: Thu, 13 Apr 2017 16:13:24 -0500 Subject: [PATCH 11/24] Reorganized some stuff --- __tests__/transformers/collapse.test.js | 4 +- __tests__/transformers/simplify_node.test.js | 2 +- .../__snapshots__/utils.test.js.snap | 0 __tests__/{test-utils => utils}/utils.test.js | 2 +- app/compile.js | 196 ------------------ app/transformers/simplify_node.js | 29 --- src/compile.js | 1 + {public => tools}/index.html | 0 {test-utils => utils}/sample-ast.json | 0 {test-utils => utils}/utils.js | 0 10 files changed, 5 insertions(+), 229 deletions(-) rename __tests__/{test-utils => utils}/__snapshots__/utils.test.js.snap (100%) rename __tests__/{test-utils => utils}/utils.test.js (79%) delete mode 100644 app/compile.js delete mode 100644 app/transformers/simplify_node.js rename {public => tools}/index.html (100%) rename {test-utils => utils}/sample-ast.json (100%) rename {test-utils => utils}/utils.js (100%) diff --git a/__tests__/transformers/collapse.test.js b/__tests__/transformers/collapse.test.js index 513f1a5..7b42881 100644 --- a/__tests__/transformers/collapse.test.js +++ b/__tests__/transformers/collapse.test.js @@ -1,6 +1,6 @@ describe('collapse.js', () => { - const collapse = require('../../app/transformers/collapse.js'); - const { makeNode } = require('../../test-utils/utils.js'); + const collapse = require('../../src/transformers/collapse.js'); + const { makeNode } = require('../../utils/utils.js'); const start = 0; const end = 5; diff --git a/__tests__/transformers/simplify_node.test.js b/__tests__/transformers/simplify_node.test.js index 46ecee8..7e6d6a0 100644 --- a/__tests__/transformers/simplify_node.test.js +++ b/__tests__/transformers/simplify_node.test.js @@ -1,5 +1,5 @@ describe(`simplify_node.js`, () => { - const simplify_node = require('../../app/transformers/simplify_node.js'); + const simplify_node = require('../../src/transformers/simplify_node.js'); it(`is a function`, () => { expect(typeof(simplify_node)).toEqual('function'); diff --git a/__tests__/test-utils/__snapshots__/utils.test.js.snap b/__tests__/utils/__snapshots__/utils.test.js.snap similarity index 100% rename from __tests__/test-utils/__snapshots__/utils.test.js.snap rename to __tests__/utils/__snapshots__/utils.test.js.snap diff --git a/__tests__/test-utils/utils.test.js b/__tests__/utils/utils.test.js similarity index 79% rename from __tests__/test-utils/utils.test.js rename to __tests__/utils/utils.test.js index 7fc1ac0..0a855ec 100644 --- a/__tests__/test-utils/utils.test.js +++ b/__tests__/utils/utils.test.js @@ -1,4 +1,4 @@ -const { makeNode } = require('../../test-utils/utils.js'); +const { makeNode } = require('../../utils/utils.js'); describe(`test-utils.js`, () => { describe(`makeNode`, () => { diff --git a/app/compile.js b/app/compile.js deleted file mode 100644 index 2597515..0000000 --- a/app/compile.js +++ /dev/null @@ -1,196 +0,0 @@ -const path = require('path'); -const fs = require('fs-promise'); -const child_process = require('child-process-promise'); -const antlr = require('antlr4'); - -const config = require('../config.js'); -const expect_error = require('./expect_error.js'); -const tree_matcher = require('./tree_matcher.js'); -const java_func_data_generator = require('./java_func_data_generator.js'); - -const array_diff = (a, b) => a.filter(i => b.indexOf(i) === -1); - -const resolve_grammar_path = (lang_key, grammar_file) => - path.resolve(__dirname, '..', 'grammars-v4', lang_key, grammar_file); - -/* -Sequentially invokes each task in `tasks` using the previous tasks's -return value as its argument. `tasks` must be an array of async functions. -*/ -const waterfall = async (tasks) => { - let args = null; - for(let i = 0; i < tasks.length; i++) { - args = await tasks[i](args); - } - return args; -}; - -/* -Returns a set of async closure functions with references to the given -lang_compile_config & lang_runtime_config that can be used to build the -parser. -*/ -const build_tasks = (lang_compile_config, lang_runtime_config) => { - const { - language, - rules, - generate_visitor, - generate_listener, - } = lang_runtime_config; - const { - tree_matcher_specs, - needs_java_func_data, - grammar_path, - grammar_file, - } = lang_compile_config; - - // Figure out the language key - const language_key = language.toLowerCase(); - - // Figure out the path to the grammar file - const g4_path = grammar_path ? grammar_path : - resolve_grammar_path(language_key, grammar_file); - - // Figure out the path to the cache directory - const cache_dir = config.resolve_cache_dir(lang_runtime_config); - const cache_g4_path = path.resolve(cache_dir, language + '.g4'); - - /* ============================ Build Tasks: ============================== */ - /* --- Creates cache directory and copies grammar files into it. --- */ - const prepare_cache_dir = async () => { - const on_eexist_err = expect_error('EEXIST', () => {}); - - // Make sure the cache directory exists - await fs.mkdir(config.cache_path).catch(on_eexist_err); - - // Make sure the language cache directory exists - await fs.mkdir(cache_dir).catch(on_eexist_err); - - // Copies the g4 file into the cache directory - await fs.copy(g4_path, cache_g4_path); - } - - /* --- Invokes the antlr process, returns a Promise --- */ - const invoke_antlr = async () => { - // Prepare options to the antlr compiler that generates - // the antlr lexer and antlr parser - const cmd = 'java'; - const args = [ - '-Xmx500M', - '-cp', '../../bin/antlr-4.6-complete.jar', - 'org.antlr.v4.Tool', - '-long-messages', - generate_listener ? '-listener' : '-no-listener', - generate_visitor ? '-visitor' : '-no-visitor', - '-Dlanguage=JavaScript', - language + '.g4', - ]; - const opts = { - 'cwd': cache_dir, - 'stdio': ['ignore', process.stdout, process.stderr], - }; - - await child_process.spawn(cmd, args, opts); - } - - /* --- Generates java func data, if this lang needs any --- */ - const make_java_func_data = async () => { - if (needs_java_func_data) { - await fs.stat(config.cache_path + '/java_func_data') - .catch(expect_error('ENOENT', java_func_data_generator)); - } - } - - /* --- Creates and returns the parser and symbol_name_map --- */ - const make_parser = async () => { - // Create instance of the parser - const parser_classname = language + 'Parser'; - const ParserClass = require(`${cache_dir}/${parser_classname}.js`)[parser_classname] - const parser = new ParserClass(); - - // Create an array of symbol (terminal) names - const symbol_name_map = ['_EPSILON', '_EOF', '_INVALID'] - .concat(parser.symbolicNames.slice(1)) - .map((val) => val ? '.' + val : undefined); - - // Create the lists of rule names (both terminals and non-terminals) - const parser_rules = parser.ruleNames.concat(symbol_name_map.filter(Boolean)); - const config_rules = Object.keys(rules); - - // Make sure the parser doesn't have extra rules - const config_missing = array_diff(parser_rules, config_rules); - if (config_missing.length) { - throw new Error('Missing rules ' + JSON.stringify(config_missing)); - } - - // Make sure our config doesn't have extra rules - const config_extra = array_diff(config_rules, parser_rules); - if (config_extra.length) { - throw new Error('Extra rules ' + JSON.stringify(config_extra)); - } - - return { symbol_name_map, parser }; - } - - /* --- Dynamically creates runtime config modifier function --- */ - const make_runtime_config_modifier = async ({ symbol_name_map, parser }) => { - // Turns these maps into human-readable JSON for insertion - // into returned function string - const symbol_name_map_str = JSON.stringify(symbol_name_map, null, 2); - const rule_name_map_str = JSON.stringify(parser.ruleNames, null, 2); - - // Add a tree matcher function if appropriate too - let tree_matchers_str = ''; - if (tree_matcher_specs) { - const generator = await tree_matcher.make_generator( - lang_compile_config, - lang_runtime_config - ); - const tree_matchers = tree_matcher_specs.map(generator); - tree_matchers_str = - `lang_runtime_config.tree_matcher = function(root) { - ${tree_matchers.join('\n')} - };`; - } - - // Return the runtime config modifier - a function that - // modifies the lang_runtime_config - return ` - /* - This function is generated by app/compile.js - Do not attempt to make changes. They will be overwritten. - */ - module.exports = function(lang_runtime_config) { - lang_runtime_config.symbol_name_map = ${symbol_name_map_str}; - lang_runtime_config.rule_name_map = ${rule_name_map_str}; - ${tree_matchers_str} - }; - `; - } - - /* --- Writes the runtime config modifier to file system --- */ - const write_runtime_config_modifier = async (config_modifier) => { - const modifier_path = path.resolve(cache_dir, 'runtime_config_modifier.js'); - await fs.writeFile(modifier_path, config_modifier); - } - - /* --- Returns the cache_dir wrapped in an object --- */ - const final_task = async () => { - return { cache_dir }; - } - - // Return the set of async closures - return [ - prepare_cache_dir, - invoke_antlr, - make_java_func_data, - make_parser, - make_runtime_config_modifier, - write_runtime_config_modifier, - final_task, - ]; -} - -module.exports = (lang_compile_config, lang_runtime_config) => { - return waterfall(build_tasks(lang_compile_config, lang_runtime_config)); -}; diff --git a/app/transformers/simplify_node.js b/app/transformers/simplify_node.js deleted file mode 100644 index ceb9f27..0000000 --- a/app/transformers/simplify_node.js +++ /dev/null @@ -1,29 +0,0 @@ -const TerminalNodeImpl = require('antlr4/tree/Tree.js').TerminalNodeImpl; - -module.exports = function(lang_runtime_config, root) { - - // Takes an antlr node generated by the antlr parser, and - // outputs our simplified node. - const simplify_node = function(node) { - if (node instanceof TerminalNodeImpl) { - return { - 'type': lang_runtime_config.symbol_name_map[node.symbol.type + 2], - 'begin': node.symbol.start, - 'end': node.symbol.stop + 1, - 'tags': [], - 'children': [], - 'text': node.symbol.text, - }; - } else { - return { - 'type': lang_runtime_config.rule_name_map[node.ruleIndex], - 'begin': node.start.start, - 'end': (node.stop ? node.stop : node.start).stop + 1, - 'tags': [], - 'children': node.children ? node.children.map(simplify_node) : [], - }; - } - }; - - return simplify_node(root); -}; diff --git a/src/compile.js b/src/compile.js index f7d91bb..6958a21 100644 --- a/src/compile.js +++ b/src/compile.js @@ -16,6 +16,7 @@ const resolve_grammar_path = (lang_key, grammar_file) => /* Sequentially invokes each task in `tasks` using the previous tasks's return value as its argument. `tasks` must be an array of async functions. +The first task is invoked with a null argument. */ const waterfall = async (tasks) => { let args = null; diff --git a/public/index.html b/tools/index.html similarity index 100% rename from public/index.html rename to tools/index.html diff --git a/test-utils/sample-ast.json b/utils/sample-ast.json similarity index 100% rename from test-utils/sample-ast.json rename to utils/sample-ast.json diff --git a/test-utils/utils.js b/utils/utils.js similarity index 100% rename from test-utils/utils.js rename to utils/utils.js From 2fde23a45f5bbd18a0af86cbf27dceeb80746cee Mon Sep 17 00:00:00 2001 From: Hopding Date: Thu, 13 Apr 2017 20:39:10 -0500 Subject: [PATCH 12/24] Added unit test for simplify_node transformer --- __tests__/transformers/simplify_node.test.js | 56 ++++++++++++++++++++ src/compile.js | 5 +- src/runtime.js | 5 ++ src/transformers/simplify_node.js | 51 +++++++++--------- utils/utils.js | 49 +++++++++++++++++ 5 files changed, 139 insertions(+), 27 deletions(-) diff --git a/__tests__/transformers/simplify_node.test.js b/__tests__/transformers/simplify_node.test.js index 7e6d6a0..336ed57 100644 --- a/__tests__/transformers/simplify_node.test.js +++ b/__tests__/transformers/simplify_node.test.js @@ -1,7 +1,63 @@ describe(`simplify_node.js`, () => { const simplify_node = require('../../src/transformers/simplify_node.js'); + const { + makeAntlrNode, + makeAntlrTerminal, + makeNode, + makeTerminal, + } = require('../../utils/utils'); + const lang_runtime_config = { + symbol_name_map: [ + '.BLAH', + '.FOOBAR', + '.QUXBAZ', + ], + rule_name_map: [ + 'floop_type', + 'groop_loop', + ], + }; it(`is a function`, () => { expect(typeof(simplify_node)).toEqual('function'); }); + + it(`recursively extracts the following properties from non-terminal + input nodes: + • type + • begin + • end + • tags + • children`, () => { + // Different forms for starts and stops are intentionally + // used here, as both can be found in real ANTLR nodes. + const input_node = ( + makeAntlrNode(0, { start: 0 }, { stop: 5 }, [ + makeAntlrNode(1, { start: 0, stop: 4 }, undefined, undefined), + ])); + + const { rule_name_map } = lang_runtime_config; + const expected = ( + makeNode(rule_name_map[0], 0, 5 + 1, [], [ + makeNode(rule_name_map[1], 0, 4 + 1, [], []), + ])); + expect(simplify_node(lang_runtime_config, input_node)).toEqual(expected); + }); + + it(`extracts the following properties from terminal input nodes: + • type + • begin + • end + • tags + • children + • text`, () => { + const input_terminal = ( + makeAntlrTerminal(0, 0, 5, 'foobar') + ); + const { symbol_name_map } = lang_runtime_config; + const expected = ( + makeTerminal(symbol_name_map[0 + 2], 0, 6, [], [], 'foobar') + ); + expect(simplify_node(lang_runtime_config, input_terminal)).toEqual(expected); + }); }); diff --git a/src/compile.js b/src/compile.js index 6958a21..c018e7b 100644 --- a/src/compile.js +++ b/src/compile.js @@ -70,7 +70,7 @@ const build_tasks = (lang_compile_config, lang_runtime_config) => { await fs.copy(g4_path, build_g4_path); } - /* --- Invokes the antlr process, returns a Promise --- */ + /* --- Invokes the antlr process --- */ const invoke_antlr = async () => { // Prepare options to the antlr compiler that generates // the antlr lexer and antlr parser @@ -164,8 +164,7 @@ const build_tasks = (lang_compile_config, lang_runtime_config) => { lang_runtime_config.symbol_name_map = ${symbol_name_map_str}; lang_runtime_config.rule_name_map = ${rule_name_map_str}; ${tree_matchers_str} - }; - `; + };`; } /* --- Writes the runtime config modifier to file system --- */ diff --git a/src/runtime.js b/src/runtime.js index 2538a1a..909bd02 100644 --- a/src/runtime.js +++ b/src/runtime.js @@ -3,6 +3,11 @@ let antlr = require('antlr4'); let lang_runtime_config = require('LangRuntimeConfig'); +// Modify the runtime config to give it the following properties that are +// generated at compile time (in compile.js): +// • symbol_name_map => Array of Strings +// • rule_name_map => Array of Strings +// • tree_matcher => Function (optional) require('LangBuild/runtime_config_modifier.js')(lang_runtime_config); let lexer_classname = lang_runtime_config.language + 'Lexer'; diff --git a/src/transformers/simplify_node.js b/src/transformers/simplify_node.js index ceb9f27..3138c7c 100644 --- a/src/transformers/simplify_node.js +++ b/src/transformers/simplify_node.js @@ -1,29 +1,32 @@ -const TerminalNodeImpl = require('antlr4/tree/Tree.js').TerminalNodeImpl; +const { TerminalNodeImpl } = require('antlr4/tree/Tree.js'); module.exports = function(lang_runtime_config, root) { - // Takes an antlr node generated by the antlr parser, and - // outputs our simplified node. - const simplify_node = function(node) { - if (node instanceof TerminalNodeImpl) { - return { - 'type': lang_runtime_config.symbol_name_map[node.symbol.type + 2], - 'begin': node.symbol.start, - 'end': node.symbol.stop + 1, - 'tags': [], - 'children': [], - 'text': node.symbol.text, - }; - } else { - return { - 'type': lang_runtime_config.rule_name_map[node.ruleIndex], - 'begin': node.start.start, - 'end': (node.stop ? node.stop : node.start).stop + 1, - 'tags': [], - 'children': node.children ? node.children.map(simplify_node) : [], - }; - } - }; + // Takes an antlr node generated by the antlr parser, and + // outputs our simplified node. + const simplify_node = function(node) { + const { symbol_name_map, rule_name_map } = lang_runtime_config; + if (node instanceof TerminalNodeImpl) { + const { type, start, stop, text } = node.symbol; + return { + 'type': symbol_name_map[type + 2], + 'begin': start, + 'end': stop + 1, + 'tags': [], + 'children': [], + 'text': text, + }; + } else { + const { ruleIndex, start, stop, children } = node; + return { + 'type': rule_name_map[ruleIndex], + 'begin': start.start, + 'end': (stop ? stop : start).stop + 1, + 'tags': [], + 'children': children ? children.map(simplify_node) : [], + }; + } + }; - return simplify_node(root); + return simplify_node(root); }; diff --git a/utils/utils.js b/utils/utils.js index 81b72f5..4ed44bd 100644 --- a/utils/utils.js +++ b/utils/utils.js @@ -1,3 +1,9 @@ +const { TerminalNodeImpl } = require('antlr4/tree/Tree.js'); + +/* +Returns an object built from the given arguments. The object is +representative of nodes present in the parser output AST. +*/ module.exports.makeNode = (type, begin, end, tags, children) => { return { type, @@ -7,3 +13,46 @@ module.exports.makeNode = (type, begin, end, tags, children) => { children, } }; + +/* +Returns an object built from the given arguments. The object is +representative of terminal nodes present in the parser output AST. +*/ +module.exports.makeTerminal = (type, begin, end, tags, children, text) => { + return { + type, + begin, + end, + tags, + children, + text, + } +}; + +/* +Returns an object built from the given arguments. The object +is a mock of nodes that appear in the ANTLR output AST. Note that these morks are incomplete, and only contain some properties found in real ANTLR nodes. +*/ +module.exports.makeAntlrNode = (ruleIndex, start, stop, children) => { + return { + ruleIndex, + start, + stop, + children, + } +}; + +/* +Returns an object built from the given arguments. The object +is a mock of terminal nodes that appear in ANTLR output ASTs. Note that these mocks are incomplete, and only contain some properties found in real ANTLR nodes. +*/ +module.exports.makeAntlrTerminal = (type, start, stop, text) => { + const terminal = Object.create(TerminalNodeImpl.prototype, {}); + terminal.symbol = { + type, + start, + stop, + text + }; + return terminal; +}; From ebd506cd69af85c6ff608a6c0775e1540d36d652 Mon Sep 17 00:00:00 2001 From: Joel Walker Date: Fri, 21 Apr 2017 09:25:09 -0500 Subject: [PATCH 13/24] Added unit tests comparing java tree to javascript tree --- __tests__/tree/compare.test.js | 108 ++++++++++ config.js | 2 +- grammars-v4 | 2 +- language_configs/java8.compile.js | 5 +- language_configs/python3.compile.js | 5 +- package.json | 1 + src/compile.js | 61 +----- src/compile.js.orig | 200 ++++++++++++++++++ .../lang_config.compile.js | 5 +- utils/run_antlr.js | 72 +++++++ utils/run_java.js | 31 +++ 11 files changed, 434 insertions(+), 58 deletions(-) create mode 100644 __tests__/tree/compare.test.js create mode 100644 src/compile.js.orig create mode 100644 utils/run_antlr.js create mode 100644 utils/run_java.js diff --git a/__tests__/tree/compare.test.js b/__tests__/tree/compare.test.js new file mode 100644 index 0000000..ca29338 --- /dev/null +++ b/__tests__/tree/compare.test.js @@ -0,0 +1,108 @@ +const path = require('path'); +const fs = require('fs-promise'); +const antlr = require('antlr4'); +const expect_error = require('../../utils/expect_error.js'); +const run_antlr = require('../../utils/run_antlr.js'); +const run_java = require('../../utils/run_java.js'); + +const cache_dir = '/tmp/codesplain/'; + +const genTreeViaJava = async function(lang_compile_config, lang_runtime_config, code) { + const { + build_dir + } = await run_antlr(lang_compile_config, lang_runtime_config, 'Java'); + + const java_sources = [ + lang_runtime_config.language + 'Lexer.java', + lang_runtime_config.language + 'Parser.java', + ]; + + // Add CLASSPATH to environment + const environment = Object.create(process.env); + let classpath = environment.CLASSPATH ? environment.CLASSPATH.split(':') : []; + classpath.unshift(path.resolve(__dirname, '../../bin/antlr-4.6-complete.jar')); + classpath.unshift('.'); + environment.CLASSPATH = classpath.join(':'); + + let result = await run_java( + java_sources, + 'org.antlr.v4.gui.TestRig', + [lang_runtime_config.language, lang_runtime_config.entry_rule, '-tree'], + { + 'cwd': build_dir, + 'env': environment, + //'stdio': ['pipe', 'pipe', process.stderr], + }, + code + ); + + console.log(result.stderr.toString()); + return result.stdout.toString(); +}; + +const genTreeViaJs = async function(lang_compile_config, lang_runtime_config, code) { + const { + build_dir + } = await run_antlr(lang_compile_config, lang_runtime_config, 'JavaScript'); + + let lexer_classname = lang_runtime_config.language + 'Lexer'; + let parser_classname = lang_runtime_config.language + 'Parser'; + + // Loads the lexer and parser class generated by the antlr compiler. + let LexerClass = require(build_dir + '/' + lexer_classname + '.js')[lexer_classname]; + let ParserClass = require(build_dir + '/' + parser_classname + '.js')[parser_classname]; + + // Take the string of code, and generate a stream of tokens using the antlr lexer. + // Example: ['if', '(', 'var', '==', '123', ')', '{', ...] + let chars = new antlr.InputStream(code); + let lexer = new LexerClass(chars); + let tokens = new antlr.CommonTokenStream(lexer); + + // Take the stream of tokens, and create a parser class using the antlr parser. + // Doesn't execute it yet. + let parser = new ParserClass(tokens); + + // I don't know what this does + parser.buildParseTrees = true; + + // Execute the parser and generate a tree. + // This tree has complicated nodes that need to be simplified by our process_node function. + let tree = parser[lang_runtime_config.entry_rule](); + + return tree.toStringTree(parser); +}; + +const compare = async function(language_name, code_file) { + const lang_compile_config = require(`../../language_configs/${language_name}.compile.js`); + const lang_runtime_config = require(`../../language_configs/${language_name}.runtime.js`); + const code = await fs.readFile(code_file); + const treeViaJava = await genTreeViaJava(lang_compile_config, lang_runtime_config, code); + console.log('abc'); + const treeViaJs = await genTreeViaJs(lang_compile_config, lang_runtime_config, code); + console.log('def'); + return treeViaJava === treeViaJs; +}; + + +const prev_timeout = jasmine.getEnv().DEFAULT_TIMEOUT_INTERVAL; +jasmine.getEnv().DEFAULT_TIMEOUT_INTERVAL = 20000; + +describe('grammars-v4/', () => { + describe('grammars-v4/java8/', () => { + it(`handles basic hello worlds`, () => { + return compare('java8', __dirname + '/code/snippet.0.java').then(data => { + expect(data).toBeTruthy(); + }); + }); + }); + + describe('grammars-v4/java8/', () => { + it(`tags for range loops`, () => { + return compare('python3', __dirname + '/code/snippet.1.py').then(data => { + expect(data).toBeTruthy(); + }); + }); + }); +}); + +jasmine.getEnv().DEFAULT_TIMEOUT_INTERVAL = prev_timeout; diff --git a/config.js b/config.js index 5b9eb1c..cd232de 100644 --- a/config.js +++ b/config.js @@ -3,7 +3,7 @@ let path = require('path'); module.exports = { 'app_path': path.resolve(__dirname, 'src'), 'lang_configs_path': path.resolve(__dirname, 'language_configs'), - 'build_path': path.resolve(__dirname, 'build'), + 'build_path': path.resolve(__dirname, '/tmp/codesplain_build'), 'resolve_build_dir': function(lang_runtime_config) { const language_key = lang_runtime_config.language.toLowerCase(); return path.resolve(this.build_path, language_key); diff --git a/grammars-v4 b/grammars-v4 index 9788310..727ec61 160000 --- a/grammars-v4 +++ b/grammars-v4 @@ -1 +1 @@ -Subproject commit 9788310ac2cf4666bc11695f4b40c7462fa54f70 +Subproject commit 727ec61bfa476363400a7162f6c681a783ddc573 diff --git a/language_configs/java8.compile.js b/language_configs/java8.compile.js index 0615069..051210c 100644 --- a/language_configs/java8.compile.js +++ b/language_configs/java8.compile.js @@ -1,7 +1,10 @@ // Configuration that can only be read by the compiler. module.exports = { - 'grammar_file': 'Java8.JavaScriptTarget.g4', + 'grammar_files': { + 'Java': 'Java8.g4', + 'JavaScript': 'Java8.JavaScriptTarget.g4', + }, 'needs_java_func_data': true, 'tree_matcher_specs': require('./java8.tree_matcher_specs.js'), }; diff --git a/language_configs/python3.compile.js b/language_configs/python3.compile.js index 72ea244..c3b92b9 100644 --- a/language_configs/python3.compile.js +++ b/language_configs/python3.compile.js @@ -1,6 +1,9 @@ // Configuration that can only be read by the compiler. module.exports = { - 'grammar_file': 'Python3.JavaScriptTarget.g4', + 'grammar_files': { + 'Java': 'Python3.g4', + 'JavaScript': 'Python3.JavaScriptTarget.g4', + }, 'tree_matcher_specs': require('./python3.tree_matcher_specs.js'), }; diff --git a/package.json b/package.json index e51daea..fadc487 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "webpack-closure-compiler": "^2.1.4" }, "devDependencies": { + "enhanced-resolve": "^3.1.0", "jest": "^19.0.2" }, "jest": { diff --git a/src/compile.js b/src/compile.js index c018e7b..e933ccf 100644 --- a/src/compile.js +++ b/src/compile.js @@ -4,7 +4,8 @@ const child_process = require('child-process-promise'); const antlr = require('antlr4'); const config = require('../config.js'); -const expect_error = require('./expect_error.js'); +const expect_error = require('../utils/expect_error.js'); +const run_antlr = require('./run_antlr.js'); const tree_matcher = require('./tree_matcher.js'); const java_func_data_generator = require('./java_func_data_generator.js'); @@ -32,11 +33,13 @@ lang_compile_config & lang_runtime_config that can be used to build the parser. */ const build_tasks = (lang_compile_config, lang_runtime_config) => { + const { + tree_matcher_specs, + needs_java_func_data, + } = lang_compile_config; const { language, rules, - generate_visitor, - generate_listener, } = lang_runtime_config; const { tree_matcher_specs, @@ -45,53 +48,7 @@ const build_tasks = (lang_compile_config, lang_runtime_config) => { grammar_file, } = lang_compile_config; - // Figure out the language key - const language_key = language.toLowerCase(); - - // Figure out the path to the grammar file - const g4_path = grammar_path ? grammar_path : - resolve_grammar_path(language_key, grammar_file); - - const build_dir = config.resolve_build_dir(lang_runtime_config); - const build_g4_path = path.resolve(build_dir, language + '.g4'); - - /* ============================ Build Tasks: ============================== */ - /* --- Creates build directory and copies grammar files into it. --- */ - const prepare_build_dir = async () => { - const on_eexist_err = expect_error('EEXIST', () => {}); - - // Make sure the build directory exists - await fs.mkdir(config.build_path).catch(on_eexist_err); - - // Make sure the language build directory exists - await fs.mkdir(build_dir).catch(on_eexist_err); - - // Copies the g4 file into the build directory - await fs.copy(g4_path, build_g4_path); - } - - /* --- Invokes the antlr process --- */ - const invoke_antlr = async () => { - // Prepare options to the antlr compiler that generates - // the antlr lexer and antlr parser - const cmd = 'java'; - const args = [ - '-Xmx500M', - '-cp', '../../bin/antlr-4.6-complete.jar', - 'org.antlr.v4.Tool', - '-long-messages', - generate_listener ? '-listener' : '-no-listener', - generate_visitor ? '-visitor' : '-no-visitor', - '-Dlanguage=JavaScript', - language + '.g4', - ]; - const opts = { - 'cwd': build_dir, - 'stdio': ['ignore', process.stdout, process.stderr], - }; - - await child_process.spawn(cmd, args, opts); - } + const result = await run_antlr(lang_compile_config, lang_runtime_config, 'JavaScript'); /* --- Generates java func data, if this lang needs any --- */ const make_java_func_data = async () => { @@ -174,9 +131,7 @@ const build_tasks = (lang_compile_config, lang_runtime_config) => { } /* --- Returns the build_dir wrapped in an object --- */ - const final_task = async () => { - return { build_dir }; - } + const final_task = async () => result // Return the set of async closures return [ diff --git a/src/compile.js.orig b/src/compile.js.orig new file mode 100644 index 0000000..24acc81 --- /dev/null +++ b/src/compile.js.orig @@ -0,0 +1,200 @@ +const path = require('path'); +const fs = require('fs-promise'); +const child_process = require('child-process-promise'); +const antlr = require('antlr4'); + +const config = require('../config.js'); +const expect_error = require('./expect_error.js'); +const tree_matcher = require('./tree_matcher.js'); +const java_func_data_generator = require('./java_func_data_generator.js'); + +const array_diff = (a, b) => a.filter(i => b.indexOf(i) === -1); + +const resolve_grammar_path = (lang_key, grammar_file) => + path.resolve(__dirname, '..', 'grammars-v4', lang_key, grammar_file); + +/* +Sequentially invokes each task in `tasks` using the previous tasks's +return value as its argument. `tasks` must be an array of async functions. +The first task is invoked with a null argument. +*/ +const waterfall = async (tasks) => { + let args = null; + for(let i = 0; i < tasks.length; i++) { + args = await tasks[i](args); + } + return args; +}; + +/* +Returns a set of async closure functions with references to the given +lang_compile_config & lang_runtime_config that can be used to build the +parser. +*/ +const build_tasks = (lang_compile_config, lang_runtime_config) => { + const { +<<<<<<< Updated upstream +======= + tree_matcher_specs, + needs_java_func_data, + } = lang_compile_config; + const { +>>>>>>> Stashed changes + language, + rules, + } = lang_runtime_config; + const { + tree_matcher_specs, + needs_java_func_data, + grammar_path, + grammar_file, + } = lang_compile_config; + + // Figure out the language key + const language_key = language.toLowerCase(); + + // Figure out the path to the grammar file + const g4_path = grammar_path ? grammar_path : + resolve_grammar_path(language_key, grammar_file); + + const build_dir = config.resolve_build_dir(lang_runtime_config); + const build_g4_path = path.resolve(build_dir, language + '.g4'); + + /* ============================ Build Tasks: ============================== */ + /* --- Creates build directory and copies grammar files into it. --- */ + const prepare_build_dir = async () => { + const on_eexist_err = expect_error('EEXIST', () => {}); + + // Make sure the build directory exists + await fs.mkdir(config.build_path).catch(on_eexist_err); + + // Make sure the language build directory exists + await fs.mkdir(build_dir).catch(on_eexist_err); + + // Copies the g4 file into the build directory + await fs.copy(g4_path, build_g4_path); + } + + /* --- Invokes the antlr process --- */ + const invoke_antlr = async () => { + // Prepare options to the antlr compiler that generates + // the antlr lexer and antlr parser + const cmd = 'java'; + const args = [ + '-Xmx500M', + '-cp', '../../bin/antlr-4.6-complete.jar', + 'org.antlr.v4.Tool', + '-long-messages', + generate_listener ? '-listener' : '-no-listener', + generate_visitor ? '-visitor' : '-no-visitor', + '-Dlanguage=JavaScript', + language + '.g4', + ]; + const opts = { + 'cwd': build_dir, + 'stdio': ['ignore', process.stdout, process.stderr], + }; + + await child_process.spawn(cmd, args, opts); + } + + /* --- Generates java func data, if this lang needs any --- */ + const make_java_func_data = async () => { + if (needs_java_func_data) { + await fs.stat(config.build_path + '/java_func_data') + .catch(expect_error('ENOENT', java_func_data_generator)); + } + } + + /* --- Creates and returns the parser and symbol_name_map --- */ + const make_parser = async () => { + // Create instance of the parser + const parser_classname = language + 'Parser'; + const ParserClass = require(`${build_dir}/${parser_classname}.js`)[parser_classname] + const parser = new ParserClass(); + + // Create an array of symbol (terminal) names + const symbol_name_map = ['_EPSILON', '_EOF', '_INVALID'] + .concat(parser.symbolicNames.slice(1)) + .map((val) => val ? '.' + val : undefined); + + // Create the lists of rule names (both terminals and non-terminals) + const parser_rules = parser.ruleNames.concat(symbol_name_map.filter(Boolean)); + const config_rules = Object.keys(rules); + + // Make sure the parser doesn't have extra rules + const config_missing = array_diff(parser_rules, config_rules); + if (config_missing.length) { + throw new Error('Missing rules ' + JSON.stringify(config_missing)); + } + + // Make sure our config doesn't have extra rules + const config_extra = array_diff(config_rules, parser_rules); + if (config_extra.length) { + throw new Error('Extra rules ' + JSON.stringify(config_extra)); + } + + return { symbol_name_map, parser }; + } + + /* --- Dynamically creates runtime config modifier function --- */ + const make_runtime_config_modifier = async ({ symbol_name_map, parser }) => { + // Turns these maps into human-readable JSON for insertion + // into returned function string + const symbol_name_map_str = JSON.stringify(symbol_name_map, null, 2); + const rule_name_map_str = JSON.stringify(parser.ruleNames, null, 2); + + // Add a tree matcher function if appropriate too + let tree_matchers_str = ''; + if (tree_matcher_specs) { + const generator = await tree_matcher.make_generator( + lang_compile_config, + lang_runtime_config + ); + const tree_matchers = tree_matcher_specs.map(generator); + tree_matchers_str = + `lang_runtime_config.tree_matcher = function(root) { + ${tree_matchers.join('\n')} + };`; + } + + // Return the runtime config modifier - a function that + // modifies the lang_runtime_config + return ` + /* + This function is generated by app/compile.js + Do not attempt to make changes. They will be overwritten. + */ + module.exports = function(lang_runtime_config) { + lang_runtime_config.symbol_name_map = ${symbol_name_map_str}; + lang_runtime_config.rule_name_map = ${rule_name_map_str}; + ${tree_matchers_str} + };`; + } + + /* --- Writes the runtime config modifier to file system --- */ + const write_runtime_config_modifier = async (config_modifier) => { + const modifier_path = path.resolve(build_dir, 'runtime_config_modifier.js'); + await fs.writeFile(modifier_path, config_modifier); + } + + /* --- Returns the build_dir wrapped in an object --- */ + const final_task = async () => { + return { build_dir }; + } + + // Return the set of async closures + return [ + prepare_build_dir, + invoke_antlr, + make_java_func_data, + make_parser, + make_runtime_config_modifier, + write_runtime_config_modifier, + final_task, + ]; +} + +module.exports = (lang_compile_config, lang_runtime_config) => { + return waterfall(build_tasks(lang_compile_config, lang_runtime_config)); +}; diff --git a/src/tree_matcher_parser/lang_config.compile.js b/src/tree_matcher_parser/lang_config.compile.js index e305562..fe77b5b 100644 --- a/src/tree_matcher_parser/lang_config.compile.js +++ b/src/tree_matcher_parser/lang_config.compile.js @@ -1,5 +1,8 @@ let path = require('path'); module.exports = { - 'grammar_path': path.resolve(__dirname, 'TreeMatcher.g4'), + 'grammar_path': __dirname, + 'grammar_files': { + 'JavaScript': 'TreeMatcher.g4', + }, }; diff --git a/utils/run_antlr.js b/utils/run_antlr.js new file mode 100644 index 0000000..ac8b683 --- /dev/null +++ b/utils/run_antlr.js @@ -0,0 +1,72 @@ +const path = require('path'); +const fs = require('fs-promise'); +const child_process = require('child-process-promise'); + +const config = require('../config.js'); +const expect_error = require('./expect_error.js'); + + +module.exports = async (lang_compile_config, lang_runtime_config, target_language) => { + const { + grammar_dir, + grammar_files, + } = lang_compile_config; + const { + language, + generate_visitor, + generate_listener, + } = lang_runtime_config; + + // Figure out the language key + const language_key = language.toLowerCase(); + + // Figure out the path to the grammar file + const g4_dir = grammar_dir ? grammar_dir : path.resolve(__dirname, '..', 'grammars-v4', language_key); + const g4_path = path.resolve(g4_dir, grammar_files[target_language]); + + const build_dir = config.resolve_build_dir(lang_runtime_config); + const build_g4_path = path.resolve(build_dir, language + '.g4'); + + /* ============================ Build Tasks: ============================== */ + /* --- Creates build directory and copies grammar files into it. --- */ + const prepare_build_dir = async () => { + const on_eexist_err = expect_error('EEXIST', () => {}); + + // Make sure the build directory exists + await fs.mkdir(config.build_path).catch(on_eexist_err); + + // Make sure the language build directory exists + await fs.mkdir(build_dir).catch(on_eexist_err); + + // Copies the g4 file into the build directory + await fs.copy(g4_path, build_g4_path); + } + + /* --- Invokes the antlr process --- */ + const invoke_antlr = async () => { + // Prepare options to the antlr compiler that generates + // the antlr lexer and antlr parser + const cmd = 'java'; + const args = [ + '-Xmx500M', + '-cp', path.resolve(__dirname, '../bin/antlr-4.6-complete.jar'), + 'org.antlr.v4.Tool', + '-long-messages', + generate_listener ? '-listener' : '-no-listener', + generate_visitor ? '-visitor' : '-no-visitor', + '-Dlanguage=' + target_language, + language + '.g4', + ]; + const opts = { + 'cwd': build_dir, + 'stdio': ['ignore', process.stdout, process.stderr], + }; + + await child_process.spawn(cmd, args, opts); + } + + await prepare_build_dir(); + await invoke_antlr(); + + return { build_dir }; +} diff --git a/utils/run_java.js b/utils/run_java.js new file mode 100644 index 0000000..d84b84b --- /dev/null +++ b/utils/run_java.js @@ -0,0 +1,31 @@ +const child_process = require('child-process-promise'); +const expect_error = require('./expect_error.js'); + +let javac_cache = {}; + +const run_javac = function(java_sources, opts) { + const key = JSON.stringify(java_sources); + if (typeof javac_cache[key] === 'undefined') { + const cmd = 'javac'; + const cmd_args = [/* put additional arguments in here */].concat(java_sources); + javac_cache[key] = child_process.spawn(cmd, cmd_args, opts); + } + return javac_cache[key]; +} + +const run_java = function(main_class, args, opts, java_stdin) { + const cmd = 'java'; + const cmd_args = [ + main_class, + ].concat(args); + const child = child_process.spawn(cmd, cmd_args, opts); + child.childProcess.stdin.write(java_stdin); + child.childProcess.stdin.end(); + return child; +} + +module.exports = async function(java_sources, main_class, args, opts, java_stdin) { + opts.capture = ['stdout', 'stderr']; + await run_javac(java_sources, opts); + return await run_java(main_class, args, opts, java_stdin); +} From c730fe1c51f2a3963b3f31ce89f7de17ad053eb5 Mon Sep 17 00:00:00 2001 From: Hopding Date: Fri, 21 Apr 2017 10:11:01 -0500 Subject: [PATCH 14/24] Fix problems with building parsers --- config.js | 2 +- src/compile.js | 21 +++++++++---------- src/java_func_data_generator.js | 2 +- .../lang_config.compile.js | 2 +- src/{ => utils}/expect_error.js | 0 {utils => src/utils}/run_antlr.js | 7 +++---- {utils => src/utils}/run_java.js | 0 {utils => src/utils}/sample-ast.json | 0 {utils => src/utils}/utils.js | 0 webpack.config.js | 1 - yarn.lock | 2 +- 11 files changed, 17 insertions(+), 20 deletions(-) rename src/{ => utils}/expect_error.js (100%) rename {utils => src/utils}/run_antlr.js (92%) rename {utils => src/utils}/run_java.js (100%) rename {utils => src/utils}/sample-ast.json (100%) rename {utils => src/utils}/utils.js (100%) diff --git a/config.js b/config.js index cd232de..5b9eb1c 100644 --- a/config.js +++ b/config.js @@ -3,7 +3,7 @@ let path = require('path'); module.exports = { 'app_path': path.resolve(__dirname, 'src'), 'lang_configs_path': path.resolve(__dirname, 'language_configs'), - 'build_path': path.resolve(__dirname, '/tmp/codesplain_build'), + 'build_path': path.resolve(__dirname, 'build'), 'resolve_build_dir': function(lang_runtime_config) { const language_key = lang_runtime_config.language.toLowerCase(); return path.resolve(this.build_path, language_key); diff --git a/src/compile.js b/src/compile.js index e933ccf..7213324 100644 --- a/src/compile.js +++ b/src/compile.js @@ -3,9 +3,9 @@ const fs = require('fs-promise'); const child_process = require('child-process-promise'); const antlr = require('antlr4'); -const config = require('../config.js'); -const expect_error = require('../utils/expect_error.js'); -const run_antlr = require('./run_antlr.js'); +const config = require('../config'); +const expect_error = require('./utils/expect_error.js'); +const run_antlr = require('./utils/run_antlr.js'); const tree_matcher = require('./tree_matcher.js'); const java_func_data_generator = require('./java_func_data_generator.js'); @@ -33,10 +33,6 @@ lang_compile_config & lang_runtime_config that can be used to build the parser. */ const build_tasks = (lang_compile_config, lang_runtime_config) => { - const { - tree_matcher_specs, - needs_java_func_data, - } = lang_compile_config; const { language, rules, @@ -48,7 +44,11 @@ const build_tasks = (lang_compile_config, lang_runtime_config) => { grammar_file, } = lang_compile_config; - const result = await run_antlr(lang_compile_config, lang_runtime_config, 'JavaScript'); + const build_dir = config.resolve_build_dir(lang_runtime_config); + + const make_lexer_and_parser = async () => { + await run_antlr(lang_compile_config, lang_runtime_config, 'JavaScript'); + } /* --- Generates java func data, if this lang needs any --- */ const make_java_func_data = async () => { @@ -131,12 +131,11 @@ const build_tasks = (lang_compile_config, lang_runtime_config) => { } /* --- Returns the build_dir wrapped in an object --- */ - const final_task = async () => result + const final_task = async () => ({ build_dir }); // Return the set of async closures return [ - prepare_build_dir, - invoke_antlr, + make_lexer_and_parser, make_java_func_data, make_parser, make_runtime_config_modifier, diff --git a/src/java_func_data_generator.js b/src/java_func_data_generator.js index 23af57f..eed2f6c 100644 --- a/src/java_func_data_generator.js +++ b/src/java_func_data_generator.js @@ -3,7 +3,7 @@ let fs = require('fs-promise'); let child_process = require('child-process-promise'); let config = require('../config'); -let expect_error = require('./expect_error.js'); +let expect_error = require('./utils/expect_error.js'); let compile_promise; diff --git a/src/tree_matcher_parser/lang_config.compile.js b/src/tree_matcher_parser/lang_config.compile.js index fe77b5b..4c98987 100644 --- a/src/tree_matcher_parser/lang_config.compile.js +++ b/src/tree_matcher_parser/lang_config.compile.js @@ -1,7 +1,7 @@ let path = require('path'); module.exports = { - 'grammar_path': __dirname, + 'grammar_dir': __dirname, 'grammar_files': { 'JavaScript': 'TreeMatcher.g4', }, diff --git a/src/expect_error.js b/src/utils/expect_error.js similarity index 100% rename from src/expect_error.js rename to src/utils/expect_error.js diff --git a/utils/run_antlr.js b/src/utils/run_antlr.js similarity index 92% rename from utils/run_antlr.js rename to src/utils/run_antlr.js index ac8b683..b896045 100644 --- a/utils/run_antlr.js +++ b/src/utils/run_antlr.js @@ -2,10 +2,9 @@ const path = require('path'); const fs = require('fs-promise'); const child_process = require('child-process-promise'); -const config = require('../config.js'); +const config = require('../../config.js'); const expect_error = require('./expect_error.js'); - module.exports = async (lang_compile_config, lang_runtime_config, target_language) => { const { grammar_dir, @@ -21,7 +20,7 @@ module.exports = async (lang_compile_config, lang_runtime_config, target_languag const language_key = language.toLowerCase(); // Figure out the path to the grammar file - const g4_dir = grammar_dir ? grammar_dir : path.resolve(__dirname, '..', 'grammars-v4', language_key); + const g4_dir = grammar_dir ? grammar_dir : path.resolve(__dirname, '..', '..', 'grammars-v4', language_key); const g4_path = path.resolve(g4_dir, grammar_files[target_language]); const build_dir = config.resolve_build_dir(lang_runtime_config); @@ -49,7 +48,7 @@ module.exports = async (lang_compile_config, lang_runtime_config, target_languag const cmd = 'java'; const args = [ '-Xmx500M', - '-cp', path.resolve(__dirname, '../bin/antlr-4.6-complete.jar'), + '-cp', path.resolve(__dirname, '../../bin/antlr-4.6-complete.jar'), 'org.antlr.v4.Tool', '-long-messages', generate_listener ? '-listener' : '-no-listener', diff --git a/utils/run_java.js b/src/utils/run_java.js similarity index 100% rename from utils/run_java.js rename to src/utils/run_java.js diff --git a/utils/sample-ast.json b/src/utils/sample-ast.json similarity index 100% rename from utils/sample-ast.json rename to src/utils/sample-ast.json diff --git a/utils/utils.js b/src/utils/utils.js similarity index 100% rename from utils/utils.js rename to src/utils/utils.js diff --git a/webpack.config.js b/webpack.config.js index 1a4baa2..c84573e 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -3,7 +3,6 @@ let path = require('path'); let webpack = require('webpack'); let ClosureCompilerPlugin = require('webpack-closure-compiler'); - let config = require('./config.js'); let compile = require('./src/compile.js'); diff --git a/yarn.lock b/yarn.lock index e6eda4e..199b3d0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -789,7 +789,7 @@ emojis-list@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/emojis-list/-/emojis-list-2.1.0.tgz#4daa4d9db00f9819880c79fa457ae5b09a1fd389" -enhanced-resolve@^3.0.0: +enhanced-resolve@^3.0.0, enhanced-resolve@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-3.1.0.tgz#9f4b626f577245edcf4b2ad83d86e17f4f421dec" dependencies: From 1549b62a1389f4e2f41582e787b36465a88f68f2 Mon Sep 17 00:00:00 2001 From: Joel Walker Date: Fri, 21 Apr 2017 10:21:57 -0500 Subject: [PATCH 15/24] Fixed merge conflicts --- __tests__/tree/code/snippet.0.java | 5 + __tests__/tree/code/snippet.1.py | 1 + __tests__/tree/compare.test.js | 18 ++- src/compile.js | 23 ++-- src/compile.js.orig | 200 ----------------------------- src/java_func_data_generator.js | 4 +- src/runtime.js | 4 + webpack.config.js | 2 +- 8 files changed, 39 insertions(+), 218 deletions(-) create mode 100644 __tests__/tree/code/snippet.0.java create mode 100644 __tests__/tree/code/snippet.1.py delete mode 100644 src/compile.js.orig diff --git a/__tests__/tree/code/snippet.0.java b/__tests__/tree/code/snippet.0.java new file mode 100644 index 0000000..2c72395 --- /dev/null +++ b/__tests__/tree/code/snippet.0.java @@ -0,0 +1,5 @@ +class HelloWorldApp { + public static void main(String[] args) { + System.out.println("Hello World!"); + } +} diff --git a/__tests__/tree/code/snippet.1.py b/__tests__/tree/code/snippet.1.py new file mode 100644 index 0000000..f2da5dc --- /dev/null +++ b/__tests__/tree/code/snippet.1.py @@ -0,0 +1 @@ +print 'Hello, world!' diff --git a/__tests__/tree/compare.test.js b/__tests__/tree/compare.test.js index ca29338..93c13af 100644 --- a/__tests__/tree/compare.test.js +++ b/__tests__/tree/compare.test.js @@ -4,10 +4,12 @@ const antlr = require('antlr4'); const expect_error = require('../../utils/expect_error.js'); const run_antlr = require('../../utils/run_antlr.js'); const run_java = require('../../utils/run_java.js'); +const webpack = require('webpack'); const cache_dir = '/tmp/codesplain/'; const genTreeViaJava = async function(lang_compile_config, lang_runtime_config, code) { + return 'abc'; const { build_dir } = await run_antlr(lang_compile_config, lang_runtime_config, 'Java'); @@ -41,6 +43,19 @@ const genTreeViaJava = async function(lang_compile_config, lang_runtime_config, }; const genTreeViaJs = async function(lang_compile_config, lang_runtime_config, code) { + let config = await require('../../webpack.config.js')({ + 'langs': lang_runtime_config.language, + 'optimize': 0, + 'libraryTarget': 'commonjs', + 'enable_debug': true, + }); + let compiler = webpack(config); + compiler.run(function(err, stats) { + if (err) throw err; + console.log(arguments); + }); + +/* const { build_dir } = await run_antlr(lang_compile_config, lang_runtime_config, 'JavaScript'); @@ -70,6 +85,7 @@ const genTreeViaJs = async function(lang_compile_config, lang_runtime_config, co let tree = parser[lang_runtime_config.entry_rule](); return tree.toStringTree(parser); +*/ }; const compare = async function(language_name, code_file) { @@ -97,7 +113,7 @@ describe('grammars-v4/', () => { }); describe('grammars-v4/java8/', () => { - it(`tags for range loops`, () => { + it(`handles basic hello worlds`, () => { return compare('python3', __dirname + '/code/snippet.1.py').then(data => { expect(data).toBeTruthy(); }); diff --git a/src/compile.js b/src/compile.js index 7213324..055696d 100644 --- a/src/compile.js +++ b/src/compile.js @@ -11,9 +11,6 @@ const java_func_data_generator = require('./java_func_data_generator.js'); const array_diff = (a, b) => a.filter(i => b.indexOf(i) === -1); -const resolve_grammar_path = (lang_key, grammar_file) => - path.resolve(__dirname, '..', 'grammars-v4', lang_key, grammar_file); - /* Sequentially invokes each task in `tasks` using the previous tasks's return value as its argument. `tasks` must be an array of async functions. @@ -33,21 +30,19 @@ lang_compile_config & lang_runtime_config that can be used to build the parser. */ const build_tasks = (lang_compile_config, lang_runtime_config) => { - const { - language, - rules, - } = lang_runtime_config; const { tree_matcher_specs, needs_java_func_data, - grammar_path, - grammar_file, } = lang_compile_config; + const { + language, + rules, + } = lang_runtime_config; - const build_dir = config.resolve_build_dir(lang_runtime_config); + let result; const make_lexer_and_parser = async () => { - await run_antlr(lang_compile_config, lang_runtime_config, 'JavaScript'); + result = await run_antlr(lang_compile_config, lang_runtime_config, 'JavaScript'); } /* --- Generates java func data, if this lang needs any --- */ @@ -62,7 +57,7 @@ const build_tasks = (lang_compile_config, lang_runtime_config) => { const make_parser = async () => { // Create instance of the parser const parser_classname = language + 'Parser'; - const ParserClass = require(`${build_dir}/${parser_classname}.js`)[parser_classname] + const ParserClass = require(`${result.build_dir}/${parser_classname}.js`)[parser_classname] const parser = new ParserClass(); // Create an array of symbol (terminal) names @@ -126,12 +121,12 @@ const build_tasks = (lang_compile_config, lang_runtime_config) => { /* --- Writes the runtime config modifier to file system --- */ const write_runtime_config_modifier = async (config_modifier) => { - const modifier_path = path.resolve(build_dir, 'runtime_config_modifier.js'); + const modifier_path = path.resolve(result.build_dir, 'runtime_config_modifier.js'); await fs.writeFile(modifier_path, config_modifier); } /* --- Returns the build_dir wrapped in an object --- */ - const final_task = async () => ({ build_dir }); + const final_task = async () => result; // Return the set of async closures return [ diff --git a/src/compile.js.orig b/src/compile.js.orig deleted file mode 100644 index 24acc81..0000000 --- a/src/compile.js.orig +++ /dev/null @@ -1,200 +0,0 @@ -const path = require('path'); -const fs = require('fs-promise'); -const child_process = require('child-process-promise'); -const antlr = require('antlr4'); - -const config = require('../config.js'); -const expect_error = require('./expect_error.js'); -const tree_matcher = require('./tree_matcher.js'); -const java_func_data_generator = require('./java_func_data_generator.js'); - -const array_diff = (a, b) => a.filter(i => b.indexOf(i) === -1); - -const resolve_grammar_path = (lang_key, grammar_file) => - path.resolve(__dirname, '..', 'grammars-v4', lang_key, grammar_file); - -/* -Sequentially invokes each task in `tasks` using the previous tasks's -return value as its argument. `tasks` must be an array of async functions. -The first task is invoked with a null argument. -*/ -const waterfall = async (tasks) => { - let args = null; - for(let i = 0; i < tasks.length; i++) { - args = await tasks[i](args); - } - return args; -}; - -/* -Returns a set of async closure functions with references to the given -lang_compile_config & lang_runtime_config that can be used to build the -parser. -*/ -const build_tasks = (lang_compile_config, lang_runtime_config) => { - const { -<<<<<<< Updated upstream -======= - tree_matcher_specs, - needs_java_func_data, - } = lang_compile_config; - const { ->>>>>>> Stashed changes - language, - rules, - } = lang_runtime_config; - const { - tree_matcher_specs, - needs_java_func_data, - grammar_path, - grammar_file, - } = lang_compile_config; - - // Figure out the language key - const language_key = language.toLowerCase(); - - // Figure out the path to the grammar file - const g4_path = grammar_path ? grammar_path : - resolve_grammar_path(language_key, grammar_file); - - const build_dir = config.resolve_build_dir(lang_runtime_config); - const build_g4_path = path.resolve(build_dir, language + '.g4'); - - /* ============================ Build Tasks: ============================== */ - /* --- Creates build directory and copies grammar files into it. --- */ - const prepare_build_dir = async () => { - const on_eexist_err = expect_error('EEXIST', () => {}); - - // Make sure the build directory exists - await fs.mkdir(config.build_path).catch(on_eexist_err); - - // Make sure the language build directory exists - await fs.mkdir(build_dir).catch(on_eexist_err); - - // Copies the g4 file into the build directory - await fs.copy(g4_path, build_g4_path); - } - - /* --- Invokes the antlr process --- */ - const invoke_antlr = async () => { - // Prepare options to the antlr compiler that generates - // the antlr lexer and antlr parser - const cmd = 'java'; - const args = [ - '-Xmx500M', - '-cp', '../../bin/antlr-4.6-complete.jar', - 'org.antlr.v4.Tool', - '-long-messages', - generate_listener ? '-listener' : '-no-listener', - generate_visitor ? '-visitor' : '-no-visitor', - '-Dlanguage=JavaScript', - language + '.g4', - ]; - const opts = { - 'cwd': build_dir, - 'stdio': ['ignore', process.stdout, process.stderr], - }; - - await child_process.spawn(cmd, args, opts); - } - - /* --- Generates java func data, if this lang needs any --- */ - const make_java_func_data = async () => { - if (needs_java_func_data) { - await fs.stat(config.build_path + '/java_func_data') - .catch(expect_error('ENOENT', java_func_data_generator)); - } - } - - /* --- Creates and returns the parser and symbol_name_map --- */ - const make_parser = async () => { - // Create instance of the parser - const parser_classname = language + 'Parser'; - const ParserClass = require(`${build_dir}/${parser_classname}.js`)[parser_classname] - const parser = new ParserClass(); - - // Create an array of symbol (terminal) names - const symbol_name_map = ['_EPSILON', '_EOF', '_INVALID'] - .concat(parser.symbolicNames.slice(1)) - .map((val) => val ? '.' + val : undefined); - - // Create the lists of rule names (both terminals and non-terminals) - const parser_rules = parser.ruleNames.concat(symbol_name_map.filter(Boolean)); - const config_rules = Object.keys(rules); - - // Make sure the parser doesn't have extra rules - const config_missing = array_diff(parser_rules, config_rules); - if (config_missing.length) { - throw new Error('Missing rules ' + JSON.stringify(config_missing)); - } - - // Make sure our config doesn't have extra rules - const config_extra = array_diff(config_rules, parser_rules); - if (config_extra.length) { - throw new Error('Extra rules ' + JSON.stringify(config_extra)); - } - - return { symbol_name_map, parser }; - } - - /* --- Dynamically creates runtime config modifier function --- */ - const make_runtime_config_modifier = async ({ symbol_name_map, parser }) => { - // Turns these maps into human-readable JSON for insertion - // into returned function string - const symbol_name_map_str = JSON.stringify(symbol_name_map, null, 2); - const rule_name_map_str = JSON.stringify(parser.ruleNames, null, 2); - - // Add a tree matcher function if appropriate too - let tree_matchers_str = ''; - if (tree_matcher_specs) { - const generator = await tree_matcher.make_generator( - lang_compile_config, - lang_runtime_config - ); - const tree_matchers = tree_matcher_specs.map(generator); - tree_matchers_str = - `lang_runtime_config.tree_matcher = function(root) { - ${tree_matchers.join('\n')} - };`; - } - - // Return the runtime config modifier - a function that - // modifies the lang_runtime_config - return ` - /* - This function is generated by app/compile.js - Do not attempt to make changes. They will be overwritten. - */ - module.exports = function(lang_runtime_config) { - lang_runtime_config.symbol_name_map = ${symbol_name_map_str}; - lang_runtime_config.rule_name_map = ${rule_name_map_str}; - ${tree_matchers_str} - };`; - } - - /* --- Writes the runtime config modifier to file system --- */ - const write_runtime_config_modifier = async (config_modifier) => { - const modifier_path = path.resolve(build_dir, 'runtime_config_modifier.js'); - await fs.writeFile(modifier_path, config_modifier); - } - - /* --- Returns the build_dir wrapped in an object --- */ - const final_task = async () => { - return { build_dir }; - } - - // Return the set of async closures - return [ - prepare_build_dir, - invoke_antlr, - make_java_func_data, - make_parser, - make_runtime_config_modifier, - write_runtime_config_modifier, - final_task, - ]; -} - -module.exports = (lang_compile_config, lang_runtime_config) => { - return waterfall(build_tasks(lang_compile_config, lang_runtime_config)); -}; diff --git a/src/java_func_data_generator.js b/src/java_func_data_generator.js index eed2f6c..f2322a9 100644 --- a/src/java_func_data_generator.js +++ b/src/java_func_data_generator.js @@ -8,10 +8,10 @@ let expect_error = require('./utils/expect_error.js'); let compile_promise; let compile = async function() { - let cache_dir = path.resolve(config.cache_path, 'java_func_data'); + let cache_dir = path.resolve(config.build_path, 'java_func_data'); // Make sure the cache directory exists - await fs.mkdir(config.cache_path).catch(expect_error('EEXIST', function() {})); + await fs.mkdir(config.build_path).catch(expect_error('EEXIST', function() {})); // Make sure the language cache directory exists await fs.mkdir(cache_dir).catch(expect_error('EEXIST', function() {})); diff --git a/src/runtime.js b/src/runtime.js index 909bd02..8320be4 100644 --- a/src/runtime.js +++ b/src/runtime.js @@ -53,6 +53,10 @@ module.exports = function(input, error_callback, options) { // This tree has complicated nodes that need to be simplified by our process_node function. let tree = parser[lang_runtime_config.entry_rule](); + if (options.return_toStringTree) { + return tree.toStringTree(); + } + // Transform the tree and return it return transformers(lang_runtime_config, tree); }; diff --git a/webpack.config.js b/webpack.config.js index c84573e..491dd82 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -103,7 +103,7 @@ let prepare_lang = async function(filename) { module.exports = async function(env) { // Read command line options - langs = env && env.langs ? env.langs.split(',') : undefined; + langs = env && env.langs ? env.langs.toLowerCase().split(',') : undefined; optimize = Boolean(env && parseInt(env.optimize)); libraryTarget = (env && env.libraryTarget) || 'var'; global.enable_debug = Boolean(env && parseInt(env.enable_debug)); From 073bb0c4f54e8d88e9a5e4a930b135b106513df5 Mon Sep 17 00:00:00 2001 From: Joel Walker Date: Thu, 27 Apr 2017 13:47:40 -0500 Subject: [PATCH 16/24] Working on tests --- __tests__/transformers/collapse.test.js | 2 +- __tests__/transformers/simplify_node.test.js | 2 +- __tests__/tree/compare.test.js | 59 ++++++++++++++----- __tests__/utils/utils.test.js | 2 +- config.js | 4 +- package.json | 1 - src/error_listener.js | 2 +- .../Character_isJavaIdentifierPart.js | 2 +- .../Character_isJavaIdentifierStart.js | 2 +- webpack.config.js | 38 ++++++------ 10 files changed, 72 insertions(+), 42 deletions(-) diff --git a/__tests__/transformers/collapse.test.js b/__tests__/transformers/collapse.test.js index 7b42881..09afb15 100644 --- a/__tests__/transformers/collapse.test.js +++ b/__tests__/transformers/collapse.test.js @@ -1,6 +1,6 @@ describe('collapse.js', () => { const collapse = require('../../src/transformers/collapse.js'); - const { makeNode } = require('../../utils/utils.js'); + const { makeNode } = require('../../src/utils/utils.js'); const start = 0; const end = 5; diff --git a/__tests__/transformers/simplify_node.test.js b/__tests__/transformers/simplify_node.test.js index 336ed57..020da96 100644 --- a/__tests__/transformers/simplify_node.test.js +++ b/__tests__/transformers/simplify_node.test.js @@ -5,7 +5,7 @@ describe(`simplify_node.js`, () => { makeAntlrTerminal, makeNode, makeTerminal, - } = require('../../utils/utils'); + } = require('../../src/utils/utils.js'); const lang_runtime_config = { symbol_name_map: [ '.BLAH', diff --git a/__tests__/tree/compare.test.js b/__tests__/tree/compare.test.js index 93c13af..3ac95a5 100644 --- a/__tests__/tree/compare.test.js +++ b/__tests__/tree/compare.test.js @@ -1,12 +1,12 @@ const path = require('path'); const fs = require('fs-promise'); const antlr = require('antlr4'); -const expect_error = require('../../utils/expect_error.js'); -const run_antlr = require('../../utils/run_antlr.js'); -const run_java = require('../../utils/run_java.js'); +const expect_error = require('../../src/utils/expect_error.js'); +const run_antlr = require('../../src/utils/run_antlr.js'); +const run_java = require('../../src/utils/run_java.js'); const webpack = require('webpack'); -const cache_dir = '/tmp/codesplain/'; +const config = require('../../config'); const genTreeViaJava = async function(lang_compile_config, lang_runtime_config, code) { return 'abc'; @@ -21,12 +21,12 @@ const genTreeViaJava = async function(lang_compile_config, lang_runtime_config, // Add CLASSPATH to environment const environment = Object.create(process.env); - let classpath = environment.CLASSPATH ? environment.CLASSPATH.split(':') : []; + const classpath = environment.CLASSPATH ? environment.CLASSPATH.split(':') : []; classpath.unshift(path.resolve(__dirname, '../../bin/antlr-4.6-complete.jar')); classpath.unshift('.'); environment.CLASSPATH = classpath.join(':'); - let result = await run_java( + const result = await run_java( java_sources, 'org.antlr.v4.gui.TestRig', [lang_runtime_config.language, lang_runtime_config.entry_rule, '-tree'], @@ -43,18 +43,39 @@ const genTreeViaJava = async function(lang_compile_config, lang_runtime_config, }; const genTreeViaJs = async function(lang_compile_config, lang_runtime_config, code) { - let config = await require('../../webpack.config.js')({ + const output_path = path.resolve(config.build_path, 'output'); + + const webpack_config = await require('../../webpack.config.js')({ 'langs': lang_runtime_config.language, 'optimize': 0, - 'libraryTarget': 'commonjs', + 'libraryTarget': 'commonjs2', + 'outputPath': output_path, 'enable_debug': true, }); - let compiler = webpack(config); - compiler.run(function(err, stats) { - if (err) throw err; - console.log(arguments); + + const compiler = webpack(webpack_config); + await new Promise(function(resolve, reject) { + compiler.run(function(err, stats) { + if (err) { + reject(err); + } else { + resolve(stats); + } + }); }); + const parser = require(path.resolve(output_path, lang_runtime_config.language)); + +console.log('code start'); + const string_tree = parser(code, function(err) { + throw err; + }, { + 'return_toStringTree': true, + }); + console.log('code end'); + + return string_tree; + /* const { build_dir @@ -91,19 +112,24 @@ const genTreeViaJs = async function(lang_compile_config, lang_runtime_config, co const compare = async function(language_name, code_file) { const lang_compile_config = require(`../../language_configs/${language_name}.compile.js`); const lang_runtime_config = require(`../../language_configs/${language_name}.runtime.js`); - const code = await fs.readFile(code_file); + const code = await fs.readFile(code_file, {'encoding': 'utf8'}); const treeViaJava = await genTreeViaJava(lang_compile_config, lang_runtime_config, code); - console.log('abc'); + console.log(treeViaJava); const treeViaJs = await genTreeViaJs(lang_compile_config, lang_runtime_config, code); - console.log('def'); + console.log(treeViaJs); return treeViaJava === treeViaJs; }; + const prev_timeout = jasmine.getEnv().DEFAULT_TIMEOUT_INTERVAL; jasmine.getEnv().DEFAULT_TIMEOUT_INTERVAL = 20000; describe('grammars-v4/', () => { + afterAll(() => { + fs.unlink(config.build_path); + }); + describe('grammars-v4/java8/', () => { it(`handles basic hello worlds`, () => { return compare('java8', __dirname + '/code/snippet.0.java').then(data => { @@ -111,7 +137,7 @@ describe('grammars-v4/', () => { }); }); }); - +/* describe('grammars-v4/java8/', () => { it(`handles basic hello worlds`, () => { return compare('python3', __dirname + '/code/snippet.1.py').then(data => { @@ -119,6 +145,7 @@ describe('grammars-v4/', () => { }); }); }); +*/ }); jasmine.getEnv().DEFAULT_TIMEOUT_INTERVAL = prev_timeout; diff --git a/__tests__/utils/utils.test.js b/__tests__/utils/utils.test.js index 0a855ec..5e2bd5c 100644 --- a/__tests__/utils/utils.test.js +++ b/__tests__/utils/utils.test.js @@ -1,4 +1,4 @@ -const { makeNode } = require('../../utils/utils.js'); +const { makeNode } = require('../../src/utils/utils.js'); describe(`test-utils.js`, () => { describe(`makeNode`, () => { diff --git a/config.js b/config.js index 5b9eb1c..ba9696f 100644 --- a/config.js +++ b/config.js @@ -1,8 +1,10 @@ -let path = require('path'); +const path = require('path'); +const os = require('os'); module.exports = { 'app_path': path.resolve(__dirname, 'src'), 'lang_configs_path': path.resolve(__dirname, 'language_configs'), + //'build_path': path.resolve(os.tmpdir(), 'codesplain_build'), 'build_path': path.resolve(__dirname, 'build'), 'resolve_build_dir': function(lang_runtime_config) { const language_key = lang_runtime_config.language.toLowerCase(); diff --git a/package.json b/package.json index fadc487..e51daea 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,6 @@ "webpack-closure-compiler": "^2.1.4" }, "devDependencies": { - "enhanced-resolve": "^3.1.0", "jest": "^19.0.2" }, "jest": { diff --git a/src/error_listener.js b/src/error_listener.js index d7f6a23..28732f4 100644 --- a/src/error_listener.js +++ b/src/error_listener.js @@ -13,7 +13,7 @@ ErrorListener.prototype.syntaxError = function(recognizer, offendingSymbol, line this.callback({ 'type': 'syntaxError', 'begin': offendingSymbol.start, - 'end': offendingSymbol.end + 1, + 'end': offendingSymbol.stop + 1, 'msg': msg, }); }; diff --git a/src/java_functions/Character_isJavaIdentifierPart.js b/src/java_functions/Character_isJavaIdentifierPart.js index a2d8fcb..74836f9 100644 --- a/src/java_functions/Character_isJavaIdentifierPart.js +++ b/src/java_functions/Character_isJavaIdentifierPart.js @@ -1,4 +1,4 @@ -let data = require('Cache/java_func_data/Character_isJavaIdentifierPart_int.json'); +let data = require('Build/java_func_data/Character_isJavaIdentifierPart_int.json'); let range_decoder = require('./range_decoder.js'); module.exports = range_decoder(data); diff --git a/src/java_functions/Character_isJavaIdentifierStart.js b/src/java_functions/Character_isJavaIdentifierStart.js index 294193a..5484446 100644 --- a/src/java_functions/Character_isJavaIdentifierStart.js +++ b/src/java_functions/Character_isJavaIdentifierStart.js @@ -1,4 +1,4 @@ -let data = require('Cache/java_func_data/Character_isJavaIdentifierStart_int.json'); +let data = require('Build/java_func_data/Character_isJavaIdentifierStart_int.json'); let range_decoder = require('./range_decoder.js'); module.exports = range_decoder(data); diff --git a/webpack.config.js b/webpack.config.js index 491dd82..335f7bf 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -1,19 +1,20 @@ -let fs = require('fs-promise'); -let path = require('path'); -let webpack = require('webpack'); -let ClosureCompilerPlugin = require('webpack-closure-compiler'); +const fs = require('fs-promise'); +const path = require('path'); +const webpack = require('webpack'); +const ClosureCompilerPlugin = require('webpack-closure-compiler'); -let config = require('./config.js'); -let compile = require('./src/compile.js'); +const config = require('./config.js'); +const compile = require('./src/compile.js'); let langs; let optimize; let libraryTarget; +let outputPath; // Filter the files that should be compiled -let filter_lang = function(filename) { +const filter_lang = function(filename) { // Get the language name - let lang_name = filename.slice(0, -11); + const lang_name = filename.slice(0, -11); // Make sure the filename ends with ".compile.js" if (filename.slice(-11) !== '.compile.js') {return false;} @@ -28,21 +29,21 @@ let filter_lang = function(filename) { return true; }; -let prepare_lang = async function(filename) { +const prepare_lang = async function(filename) { // Get the language name again (same as above) - let lang_name = filename.slice(0, -11); + const lang_name = filename.slice(0, -11); // Load the language config files - let lang_compile_config_path = path.resolve(config.lang_configs_path, lang_name + '.compile.js'); - let lang_runtime_config_path = path.resolve(config.lang_configs_path, lang_name + '.runtime.js'); - let lang_compile_config = require(lang_compile_config_path); - let lang_runtime_config = require(lang_runtime_config_path); + const lang_compile_config_path = path.resolve(config.lang_configs_path, lang_name + '.compile.js'); + const lang_runtime_config_path = path.resolve(config.lang_configs_path, lang_name + '.runtime.js'); + const lang_compile_config = require(lang_compile_config_path); + const lang_runtime_config = require(lang_runtime_config_path); // Compile it! // This does a few things: // 1. Runs the antlr compiler that generates a javascript lexer and parser // 2. Runs the tree matcher compiler that generates tree matchers. - let compile_result = await compile(lang_compile_config, lang_runtime_config); + const compile_result = await compile(lang_compile_config, lang_runtime_config); // Return an object that will be used by webpack to compile the parser. return { @@ -87,7 +88,7 @@ let prepare_lang = async function(filename) { 'filename': lang_name + (optimize ? '.min.js' : '.js'), // The output directory - 'path': path.resolve(__dirname, 'build', 'parsers'), + 'path': outputPath, // How to export the parser library // commonjs2 - Use module.exports @@ -106,14 +107,15 @@ module.exports = async function(env) { langs = env && env.langs ? env.langs.toLowerCase().split(',') : undefined; optimize = Boolean(env && parseInt(env.optimize)); libraryTarget = (env && env.libraryTarget) || 'var'; + outputPath = (env && env.outputPath) || path.resolve(__dirname, 'build', 'parsers'); global.enable_debug = Boolean(env && parseInt(env.enable_debug)); // Then, find language configuration files - let files = await fs.readdir(config.lang_configs_path); + const files = await fs.readdir(config.lang_configs_path); // Filter langs and compile them. // Prepare_lang returns a promise, so after this step we have an array of promises. - let lang_configs = files.filter(filter_lang).map(prepare_lang); + const lang_configs = files.filter(filter_lang).map(prepare_lang); if (lang_configs.length === 0) { // If no languages were compiled, warn the user. From bafe60f596cfa647a7ac8bc8058ca7d8bcaa1f9b Mon Sep 17 00:00:00 2001 From: Joel Walker Date: Thu, 4 May 2017 08:47:47 -0500 Subject: [PATCH 17/24] Tree comparison tests work --- __tests__/mappings/mappings.test.js | 2 +- __tests__/mappings/mappings.test.js.orig | 49 ------- __tests__/tree/code/rename_random.sh | 3 + __tests__/tree/code/snippet.1.py | 1 - .../{snippet.0.java => snippet.k17eu4f2.java} | 2 +- __tests__/tree/code/snippet.voc84cjo.py | 1 + __tests__/tree/compare.test.js | 131 +++++++++--------- antlr4 | 2 +- config.js | 4 +- grammars-v4 | 2 +- language_configs/python3.rules.js | 1 - .../python3.tree_matcher_specs.js | 4 +- mappings/python3.csv | 1 - src/runtime.js | 4 +- src/utils/run_antlr.js | 2 +- webpack.config.js | 2 +- 16 files changed, 84 insertions(+), 127 deletions(-) delete mode 100644 __tests__/mappings/mappings.test.js.orig create mode 100755 __tests__/tree/code/rename_random.sh delete mode 100644 __tests__/tree/code/snippet.1.py rename __tests__/tree/code/{snippet.0.java => snippet.k17eu4f2.java} (76%) create mode 100644 __tests__/tree/code/snippet.voc84cjo.py diff --git a/__tests__/mappings/mappings.test.js b/__tests__/mappings/mappings.test.js index c422598..4e370b2 100644 --- a/__tests__/mappings/mappings.test.js +++ b/__tests__/mappings/mappings.test.js @@ -24,7 +24,7 @@ describe('mappings', () => { // Now create a sorted array of rules in the config files, and // the tree matcher specs file, then compare the two - Object.keys(require(`../language_configs/${lang}.rules.js`)) + Object.keys(require(`../../language_configs/${lang}.rules.js`)) .concat(treeMatcherTypes) .sort((a, b) => a.localeCompare(b)) .forEach((rule, i) => expect(rule).toEqual(tokens[i])); diff --git a/__tests__/mappings/mappings.test.js.orig b/__tests__/mappings/mappings.test.js.orig deleted file mode 100644 index e1862bc..0000000 --- a/__tests__/mappings/mappings.test.js.orig +++ /dev/null @@ -1,49 +0,0 @@ -const fs = require('fs'); - -describe('mappings', () => { - fs.readdirSync('./language_configs/') - // Gather set of all langs in "language_configs" - .reduce((langs, name) => langs.add(name.split('.')[0]), new Set()) - .forEach(lang => { - const csvFileName = `${lang}.csv` - describe(csvFileName, () => { - const encoding = { encoding: 'utf-8' }; - const csvPath = __dirname + `/../../mappings/${csvFileName}`; - const csv = fs.readFileSync(csvPath, encoding) - .split('\n') // Make array of lines - .slice(1, -1); // Cut off the header and newline - it(`should have a definition for all rules`, () => { - // Create a sorted array of tokens in the csv - const tokens = csv.map(line => line.split(',')[0]) - .sort((a, b) => a.localeCompare(b)); - -<<<<<<< HEAD:__tests__/mappings/mappings.test.js - // Now create sorted array of rules in the config - // file, and compare the two - const languageConfigPath = __dirname + `/../../language_configs/${file}`; - Object.keys(require(languageConfigPath)) -======= - // Extract types from tree_matcher_specs.js file for this lang - const treeMatcherTypes = require( - `../language_configs/${lang}.tree_matcher_specs.js` - ).map(spec => spec.type); - - // Now create a sorted array of rules in the config files, and - // the tree matcher specs file, then compare the two - Object.keys(require(`../language_configs/${lang}.rules.js`)) - .concat(treeMatcherTypes) ->>>>>>> master:__tests__/mappings.js - .sort((a, b) => a.localeCompare(b)) - .forEach((rule, i) => expect(rule).toEqual(tokens[i])); - }); - it('should have a pretty name and color for enabled rules', () => { - csv.filter((line) => line.split(',')[1] != 0) - .forEach((line) => { - const cols = line.split(','); - expect(cols[2]).toBeTruthy(); - expect(cols[3]).toMatch(/#[a-fA-F0-9]{6}/); - }); - }); - }); - }); -}); diff --git a/__tests__/tree/code/rename_random.sh b/__tests__/tree/code/rename_random.sh new file mode 100755 index 0000000..f521155 --- /dev/null +++ b/__tests__/tree/code/rename_random.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +mv "$1" "snippet.`node --print "Math.random().toString(36).substr(2, 8)"`.${1##*.}" diff --git a/__tests__/tree/code/snippet.1.py b/__tests__/tree/code/snippet.1.py deleted file mode 100644 index f2da5dc..0000000 --- a/__tests__/tree/code/snippet.1.py +++ /dev/null @@ -1 +0,0 @@ -print 'Hello, world!' diff --git a/__tests__/tree/code/snippet.0.java b/__tests__/tree/code/snippet.k17eu4f2.java similarity index 76% rename from __tests__/tree/code/snippet.0.java rename to __tests__/tree/code/snippet.k17eu4f2.java index 2c72395..5dedd6b 100644 --- a/__tests__/tree/code/snippet.0.java +++ b/__tests__/tree/code/snippet.k17eu4f2.java @@ -1,4 +1,4 @@ -class HelloWorldApp { +public class HelloWorldApp { public static void main(String[] args) { System.out.println("Hello World!"); } diff --git a/__tests__/tree/code/snippet.voc84cjo.py b/__tests__/tree/code/snippet.voc84cjo.py new file mode 100644 index 0000000..ec7780c --- /dev/null +++ b/__tests__/tree/code/snippet.voc84cjo.py @@ -0,0 +1 @@ +print('Hello, world!') diff --git a/__tests__/tree/compare.test.js b/__tests__/tree/compare.test.js index 3ac95a5..2bec6bd 100644 --- a/__tests__/tree/compare.test.js +++ b/__tests__/tree/compare.test.js @@ -8,11 +8,14 @@ const webpack = require('webpack'); const config = require('../../config'); -const genTreeViaJava = async function(lang_compile_config, lang_runtime_config, code) { - return 'abc'; +const prepareJava = async function(lang_compile_config, lang_runtime_config) { + return await run_antlr(lang_compile_config, lang_runtime_config, 'Java'); +} + +const genTreeViaJava = async function(lang_runtime_config, antlr_result, code) { const { build_dir - } = await run_antlr(lang_compile_config, lang_runtime_config, 'Java'); + } = antlr_result; const java_sources = [ lang_runtime_config.language + 'Lexer.java', @@ -38,11 +41,14 @@ const genTreeViaJava = async function(lang_compile_config, lang_runtime_config, code ); - console.log(result.stderr.toString()); - return result.stdout.toString(); + if (result.code) { + throw result.stderr.toString(); + } else { + return result.stdout.toString(); + } }; -const genTreeViaJs = async function(lang_compile_config, lang_runtime_config, code) { +const prepareJs = async function(lang_compile_config, lang_runtime_config) { const output_path = path.resolve(config.build_path, 'output'); const webpack_config = await require('../../webpack.config.js')({ @@ -50,10 +56,11 @@ const genTreeViaJs = async function(lang_compile_config, lang_runtime_config, co 'optimize': 0, 'libraryTarget': 'commonjs2', 'outputPath': output_path, - 'enable_debug': true, + 'enable_debug': false, }); const compiler = webpack(webpack_config); + await new Promise(function(resolve, reject) { compiler.run(function(err, stats) { if (err) { @@ -64,88 +71,86 @@ const genTreeViaJs = async function(lang_compile_config, lang_runtime_config, co }); }); - const parser = require(path.resolve(output_path, lang_runtime_config.language)); + return require(path.resolve(output_path, lang_runtime_config.language)); +} -console.log('code start'); - const string_tree = parser(code, function(err) { +const genTreeViaJs = async function(lang_runtime_config, js_parser, code) { + const tree = js_parser(code, function(err) { throw err; }, { - 'return_toStringTree': true, + 'return_antlr_tree': true, }); - console.log('code end'); - return string_tree; + return tree.toStringTree(tree.parser.ruleNames); +}; -/* - const { - build_dir - } = await run_antlr(lang_compile_config, lang_runtime_config, 'JavaScript'); - let lexer_classname = lang_runtime_config.language + 'Lexer'; - let parser_classname = lang_runtime_config.language + 'Parser'; +let prepare_cache = {}; - // Loads the lexer and parser class generated by the antlr compiler. - let LexerClass = require(build_dir + '/' + lexer_classname + '.js')[lexer_classname]; - let ParserClass = require(build_dir + '/' + parser_classname + '.js')[parser_classname]; +const compare = async function(language_name, code_file) { + const lang_compile_config = require(`../../language_configs/${language_name}.compile.js`); + const lang_runtime_config = require(`../../language_configs/${language_name}.runtime.js`); - // Take the string of code, and generate a stream of tokens using the antlr lexer. - // Example: ['if', '(', 'var', '==', '123', ')', '{', ...] - let chars = new antlr.InputStream(code); - let lexer = new LexerClass(chars); - let tokens = new antlr.CommonTokenStream(lexer); + if (typeof prepare_cache[language_name] === 'undefined') { + prepare_cache[language_name] = Promise.all([ + prepareJava(lang_compile_config, lang_runtime_config), + prepareJs(lang_compile_config, lang_runtime_config), + ]); + } - // Take the stream of tokens, and create a parser class using the antlr parser. - // Doesn't execute it yet. - let parser = new ParserClass(tokens); + const t1 = Date.now(); - // I don't know what this does - parser.buildParseTrees = true; + const res = await prepare_cache[language_name]; + const antlr_result = res[0]; + const js_parser = res[1]; + const t2 = Date.now(); - // Execute the parser and generate a tree. - // This tree has complicated nodes that need to be simplified by our process_node function. - let tree = parser[lang_runtime_config.entry_rule](); + const code = await fs.readFile(code_file, {'encoding': 'utf8'}); + const t3 = Date.now(); - return tree.toStringTree(parser); -*/ -}; + const treeViaJava = await genTreeViaJava(lang_runtime_config, antlr_result, code); + const t4 = Date.now(); -const compare = async function(language_name, code_file) { - const lang_compile_config = require(`../../language_configs/${language_name}.compile.js`); - const lang_runtime_config = require(`../../language_configs/${language_name}.runtime.js`); - const code = await fs.readFile(code_file, {'encoding': 'utf8'}); - const treeViaJava = await genTreeViaJava(lang_compile_config, lang_runtime_config, code); - console.log(treeViaJava); - const treeViaJs = await genTreeViaJs(lang_compile_config, lang_runtime_config, code); - console.log(treeViaJs); - return treeViaJava === treeViaJs; -}; + const treeViaJs = await genTreeViaJs(lang_runtime_config, js_parser, code); + const t5 = Date.now(); + console.log('[' + language_name + '] prepare: ' + (t2 - t1) + 'ms'); + console.log('[' + language_name + '] readFile: ' + (t3 - t2) + 'ms'); + console.log('[' + language_name + '] genJava: ' + (t4 - t3) + 'ms'); + console.log('[' + language_name + '] genJs: ' + (t5 - t4) + 'ms'); + + return treeViaJava.trim() === treeViaJs.trim(); +}; -const prev_timeout = jasmine.getEnv().DEFAULT_TIMEOUT_INTERVAL; -jasmine.getEnv().DEFAULT_TIMEOUT_INTERVAL = 20000; describe('grammars-v4/', () => { + let prev_timeout; + beforeAll(() => { + prev_timeout = jasmine.DEFAULT_TIMEOUT_INTERVAL; + jasmine.DEFAULT_TIMEOUT_INTERVAL = 20000; + }); afterAll(() => { - fs.unlink(config.build_path); + jasmine.DEFAULT_TIMEOUT_INTERVAL = prev_timeout; + return fs.remove(config.build_path).catch(err => { + console.error(err); + }); }); - describe('grammars-v4/java8/', () => { - it(`handles basic hello worlds`, () => { - return compare('java8', __dirname + '/code/snippet.0.java').then(data => { + const test_snippet = function(lang_key, description, code_filename) { + it(description, () => { + expect.assertions(1); + return compare(lang_key, __dirname + '/code/' + code_filename).then(data => { expect(data).toBeTruthy(); }); }); + }; + + describe('grammars-v4/python3/', () => { + test_snippet('python3', 'handles basic hello worlds', 'snippet.voc84cjo.py'); }); -/* + describe('grammars-v4/java8/', () => { - it(`handles basic hello worlds`, () => { - return compare('python3', __dirname + '/code/snippet.1.py').then(data => { - expect(data).toBeTruthy(); - }); - }); + test_snippet('java8', 'handles basic hello worlds', 'snippet.k17eu4f2.java'); }); -*/ }); - -jasmine.getEnv().DEFAULT_TIMEOUT_INTERVAL = prev_timeout; diff --git a/antlr4 b/antlr4 index ce09abb..2ebae94 160000 --- a/antlr4 +++ b/antlr4 @@ -1 +1 @@ -Subproject commit ce09abb480e82c1701f3df91a3f824fbe01cf454 +Subproject commit 2ebae94612110045b86ca5585c8227e7c808d192 diff --git a/config.js b/config.js index ba9696f..c328458 100644 --- a/config.js +++ b/config.js @@ -6,8 +6,8 @@ module.exports = { 'lang_configs_path': path.resolve(__dirname, 'language_configs'), //'build_path': path.resolve(os.tmpdir(), 'codesplain_build'), 'build_path': path.resolve(__dirname, 'build'), - 'resolve_build_dir': function(lang_runtime_config) { + 'resolve_build_dir': function(lang_runtime_config, target_language) { const language_key = lang_runtime_config.language.toLowerCase(); - return path.resolve(this.build_path, language_key); + return path.resolve(this.build_path, language_key + '_to_' + target_language); }, }; diff --git a/grammars-v4 b/grammars-v4 index 727ec61..96255d4 160000 --- a/grammars-v4 +++ b/grammars-v4 @@ -1 +1 @@ -Subproject commit 727ec61bfa476363400a7162f6c681a783ddc573 +Subproject commit 96255d4c1bfcc377263842d35cbba92620154275 diff --git a/language_configs/python3.rules.js b/language_configs/python3.rules.js index 49399f9..766542d 100644 --- a/language_configs/python3.rules.js +++ b/language_configs/python3.rules.js @@ -67,7 +67,6 @@ module.exports = { 'term': {'collapse': true}, 'factor': {'collapse': true}, 'power': {'collapse': true}, - 'trailed_atom': {'collapse': true}, 'atom': {}, 'testlist_comp': {}, 'trailer': {}, diff --git a/language_configs/python3.tree_matcher_specs.js b/language_configs/python3.tree_matcher_specs.js index c3763f1..61bb07c 100644 --- a/language_configs/python3.tree_matcher_specs.js +++ b/language_configs/python3.tree_matcher_specs.js @@ -2,7 +2,7 @@ module.exports = [ { // The pattern specifies which nodes to match. // When a node is matched, actor will be executed. - pattern: 'for_stmt [.FOR, /.NAME:iter, .IN, /trailed_atom [/.NAME="range", trailer\ + pattern: 'for_stmt [.FOR, /.NAME:iter, .IN, /power [/.NAME="range", trailer\ [.OPEN_PAREN, arglist [/.DECIMAL_INTEGER:begin, .COMMA,\ /.DECIMAL_INTEGER:end], .CLOSE_PAREN]], .COLON, suite]', @@ -28,7 +28,7 @@ module.exports = [ }); }, }, { - pattern: 'trailed_atom [/.NAME:name, trailer [.OPEN_PAREN, arglist:args, .CLOSE_PAREN]]', + pattern: 'power [/.NAME:name, trailer [.OPEN_PAREN, arglist:args, .CLOSE_PAREN]]', type: 'function_call', actor: function() { console.log(name.text, args.children); diff --git a/mappings/python3.csv b/mappings/python3.csv index 9bd047a..59f634d 100644 --- a/mappings/python3.csv +++ b/mappings/python3.csv @@ -63,7 +63,6 @@ arith_expr,1,Arithmetic Expression,#FFA500 term,0,, factor,0,, power,0,, -trailed_atom,0,, atom,0,, testlist_comp,0,, trailer,0,, diff --git a/src/runtime.js b/src/runtime.js index 8320be4..816002d 100644 --- a/src/runtime.js +++ b/src/runtime.js @@ -53,8 +53,8 @@ module.exports = function(input, error_callback, options) { // This tree has complicated nodes that need to be simplified by our process_node function. let tree = parser[lang_runtime_config.entry_rule](); - if (options.return_toStringTree) { - return tree.toStringTree(); + if (options.return_antlr_tree) { + return tree; } // Transform the tree and return it diff --git a/src/utils/run_antlr.js b/src/utils/run_antlr.js index b896045..cd3668d 100644 --- a/src/utils/run_antlr.js +++ b/src/utils/run_antlr.js @@ -23,7 +23,7 @@ module.exports = async (lang_compile_config, lang_runtime_config, target_languag const g4_dir = grammar_dir ? grammar_dir : path.resolve(__dirname, '..', '..', 'grammars-v4', language_key); const g4_path = path.resolve(g4_dir, grammar_files[target_language]); - const build_dir = config.resolve_build_dir(lang_runtime_config); + const build_dir = config.resolve_build_dir(lang_runtime_config, target_language); const build_g4_path = path.resolve(build_dir, language + '.g4'); /* ============================ Build Tasks: ============================== */ diff --git a/webpack.config.js b/webpack.config.js index 335f7bf..07da6b7 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -122,6 +122,6 @@ module.exports = async function(env) { console.warn('No languages generated...'); } else { // Otherwise, return a promise that resolves when all of the language promises resolve. - return Promise.all(lang_configs); + return await Promise.all(lang_configs); } }; From 631bbc0e350dbde46dd06079bd72b9a675bb9d57 Mon Sep 17 00:00:00 2001 From: Joel Walker Date: Thu, 4 May 2017 08:59:04 -0500 Subject: [PATCH 18/24] Added python3 tree comparison test --- __tests__/tree/code/snippet.kt29xnfw.py | 7 +++++++ __tests__/tree/compare.test.js | 11 +++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 __tests__/tree/code/snippet.kt29xnfw.py diff --git a/__tests__/tree/code/snippet.kt29xnfw.py b/__tests__/tree/code/snippet.kt29xnfw.py new file mode 100644 index 0000000..5696edc --- /dev/null +++ b/__tests__/tree/code/snippet.kt29xnfw.py @@ -0,0 +1,7 @@ +import sys +try: + total = sum(int(arg) for arg in sys.argv[1:]) + print('sum =', total) +except ValueError: + print('Please supply integer arguments') + diff --git a/__tests__/tree/compare.test.js b/__tests__/tree/compare.test.js index 2bec6bd..59b7c11 100644 --- a/__tests__/tree/compare.test.js +++ b/__tests__/tree/compare.test.js @@ -114,10 +114,10 @@ const compare = async function(language_name, code_file) { const treeViaJs = await genTreeViaJs(lang_runtime_config, js_parser, code); const t5 = Date.now(); - console.log('[' + language_name + '] prepare: ' + (t2 - t1) + 'ms'); - console.log('[' + language_name + '] readFile: ' + (t3 - t2) + 'ms'); - console.log('[' + language_name + '] genJava: ' + (t4 - t3) + 'ms'); - console.log('[' + language_name + '] genJs: ' + (t5 - t4) + 'ms'); + console.log('compare(' + code_file + ') prepare: ' + (t2 - t1) + 'ms'); + console.log('compare(' + code_file + ') readFile: ' + (t3 - t2) + 'ms'); + console.log('compare(' + code_file + ') genJava: ' + (t4 - t3) + 'ms'); + console.log('compare(' + code_file + ') genJs: ' + (t5 - t4) + 'ms'); return treeViaJava.trim() === treeViaJs.trim(); }; @@ -142,12 +142,15 @@ describe('grammars-v4/', () => { expect.assertions(1); return compare(lang_key, __dirname + '/code/' + code_filename).then(data => { expect(data).toBeTruthy(); + }).catch(err => { + console.error(err); }); }); }; describe('grammars-v4/python3/', () => { test_snippet('python3', 'handles basic hello worlds', 'snippet.voc84cjo.py'); + test_snippet('python3', 'handles a script adding the input arguments', 'snippet.kt29xnfw.py'); }); describe('grammars-v4/java8/', () => { From 2f3c4d373cef681cfbe2b2534d1096db69a8bf06 Mon Sep 17 00:00:00 2001 From: Joel Walker Date: Thu, 4 May 2017 09:02:42 -0500 Subject: [PATCH 19/24] Added ruby snippet for future tree comparison --- __tests__/tree/code/snippet.hixfx981.rb | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 __tests__/tree/code/snippet.hixfx981.rb diff --git a/__tests__/tree/code/snippet.hixfx981.rb b/__tests__/tree/code/snippet.hixfx981.rb new file mode 100644 index 0000000..a198637 --- /dev/null +++ b/__tests__/tree/code/snippet.hixfx981.rb @@ -0,0 +1,24 @@ +class Unit + # A list of all the units we know. + @@units = { } + + # Access to @@units. + def Unit.exists(n) + return @@units.has_key?(n) + end + def Unit.named(n) + return @@units[n] + end + + # If this were Java, I'd define an abstract function isbase() which tells + # if this object is a BaseUnit or not. + def initialize(name) + @name = name + @@units[name] = self + @@units[name + 's'] = self + end + attr_reader :name + def alias(*names) + names.each { |n| @@units[n] = self } + end +end From 33c68af6a7b503874d7fda6c0603299d3c992d61 Mon Sep 17 00:00:00 2001 From: Joel Walker Date: Thu, 4 May 2017 09:51:39 -0500 Subject: [PATCH 20/24] Added profile_test script --- .gitignore | 1 + __tests__/tree/compare.test.js | 6 +++--- profile_test | 24 ++++++++++++++++++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) create mode 100755 profile_test diff --git a/.gitignore b/.gitignore index 7228e88..dad9c10 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ bin/ node_modules/ .DS_Store public/langs/* +profile_result.txt diff --git a/__tests__/tree/compare.test.js b/__tests__/tree/compare.test.js index 59b7c11..c14742c 100644 --- a/__tests__/tree/compare.test.js +++ b/__tests__/tree/compare.test.js @@ -129,13 +129,13 @@ describe('grammars-v4/', () => { beforeAll(() => { prev_timeout = jasmine.DEFAULT_TIMEOUT_INTERVAL; jasmine.DEFAULT_TIMEOUT_INTERVAL = 20000; - }); - afterAll(() => { - jasmine.DEFAULT_TIMEOUT_INTERVAL = prev_timeout; return fs.remove(config.build_path).catch(err => { console.error(err); }); }); + afterAll(() => { + jasmine.DEFAULT_TIMEOUT_INTERVAL = prev_timeout; + }); const test_snippet = function(lang_key, description, code_filename) { it(description, () => { diff --git a/profile_test b/profile_test new file mode 100755 index 0000000..a22534a --- /dev/null +++ b/profile_test @@ -0,0 +1,24 @@ +#!/bin/sh + +# Exit if error +set -e + +# Remove any profile logs from previous runs +rm -f isolate-*-v8.log + +# Run profiler +node --prof ./node_modules/.bin/jest + +# The profiler outputs 3 files. +# Find the one that contains results from running /Users/joelwalker/codesplain/build/output/ +match=$(grep --fixed-strings --files-with-matches '/Users/joelwalker/codesplain/build/output/' isolate-*-v8.log) +[ $(wc -l <<< "$match") = "1" ] || { + echo 'Did not find exactly one file with search string in profile output' + echo "$match" + exit 1 +} + +node --prof-process "$match" > profile_result.txt +echo "Wrote profile_result.txt" + +rm isolate-*-v8.log From 14b03ab5696fd6277e6c3deb87a81c022426054a Mon Sep 17 00:00:00 2001 From: Joel Walker Date: Thu, 4 May 2017 10:41:07 -0500 Subject: [PATCH 21/24] Updated circle.yml to sync git submodules --- circle.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/circle.yml b/circle.yml index 352b4c4..2158f01 100644 --- a/circle.yml +++ b/circle.yml @@ -2,6 +2,11 @@ machine: node: version: 7.7.2 +checkout: + post: + - git submodule sync --recursive + - git submodule update --recursive --init + dependencies: override: - yarn From f4d9a0b2480f8c01d95fcb8dcda2e80dcbec7191 Mon Sep 17 00:00:00 2001 From: Joel Walker Date: Thu, 4 May 2017 11:08:10 -0500 Subject: [PATCH 22/24] Make circle play nice with the submodule cache --- circle.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 2158f01..b4bc27f 100644 --- a/circle.yml +++ b/circle.yml @@ -5,7 +5,10 @@ machine: checkout: post: - git submodule sync --recursive - - git submodule update --recursive --init + + # This is a weird hack to get circle to update the submodule when the url changes + # See https://discuss.circleci.com/t/git-submodule-url-isnt-playing-nice-with-the-cache/549/3 + - git submodule update --recursive --init || (rm -fr .git/config .git/modules && git submodule deinit -f . && git submodule update --init --recursive) dependencies: override: From 778d6967a5af26bdccffab9cd272af8ed9a9e79a Mon Sep 17 00:00:00 2001 From: Joel Walker Date: Thu, 4 May 2017 11:25:19 -0500 Subject: [PATCH 23/24] Fixed tree comparison js parser path bug --- __tests__/tree/compare.test.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/__tests__/tree/compare.test.js b/__tests__/tree/compare.test.js index c14742c..ff7d4a5 100644 --- a/__tests__/tree/compare.test.js +++ b/__tests__/tree/compare.test.js @@ -59,6 +59,10 @@ const prepareJs = async function(lang_compile_config, lang_runtime_config) { 'enable_debug': false, }); + if (webpack_config.length !== 1) { + throw new Error('Unexpected webpack output'); + } + const compiler = webpack(webpack_config); await new Promise(function(resolve, reject) { @@ -71,7 +75,7 @@ const prepareJs = async function(lang_compile_config, lang_runtime_config) { }); }); - return require(path.resolve(output_path, lang_runtime_config.language)); + return require(path.resolve(output_path, webpack_config[0].output.filename)); } const genTreeViaJs = async function(lang_runtime_config, js_parser, code) { From 05081f895c1a90788b68281636450192a84cfb40 Mon Sep 17 00:00:00 2001 From: Joel Walker Date: Thu, 4 May 2017 11:32:33 -0500 Subject: [PATCH 24/24] Changed jasmine.DEFAULT_TIMEOUT_INTERVAL because circle is slow --- __tests__/tree/compare.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/__tests__/tree/compare.test.js b/__tests__/tree/compare.test.js index ff7d4a5..0045bd1 100644 --- a/__tests__/tree/compare.test.js +++ b/__tests__/tree/compare.test.js @@ -132,7 +132,7 @@ describe('grammars-v4/', () => { let prev_timeout; beforeAll(() => { prev_timeout = jasmine.DEFAULT_TIMEOUT_INTERVAL; - jasmine.DEFAULT_TIMEOUT_INTERVAL = 20000; + jasmine.DEFAULT_TIMEOUT_INTERVAL = 60000; return fs.remove(config.build_path).catch(err => { console.error(err); });