Merge pull request #64 from maryvilledev/UnitTests61

Refactored & Added Unit Tests

.gitignore

@@ -1,5 +1,7 @@
 _cache/
+build/
 bin/
 node_modules/
 .DS_Store
 public/langs/*
+profile_result.txt

__tests__/mappings.js → __tests__/mappings/mappings.test.js

@@ -8,7 +8,8 @@ describe('mappings', () => {
       const csvFileName = `${lang}.csv`
       describe(csvFileName, () => {
         const encoding = { encoding: 'utf-8' };
-        const csv = fs.readFileSync(`./mappings/${csvFileName}`, encoding)
+        const csvPath = __dirname + `/../../mappings/${csvFileName}`;
+        const csv = fs.readFileSync(csvPath, encoding)
           .split('\n')   // Make array of lines
           .slice(1, -1); // Cut off the header and newline
         it(`should have a definition for all rules`, () => {
@@ -18,12 +19,12 @@ describe('mappings', () => {
 
           // Extract types from tree_matcher_specs.js file for this lang
           const treeMatcherTypes = require(
-            `../language_configs/${lang}.tree_matcher_specs.js`
+            `../../language_configs/${lang}.tree_matcher_specs.js`
           ).reduce((arr, spec) => arr.concat(spec.provides_tags), []);
 
           // Now create a sorted array of rules in the config files, and
           // the tree matcher specs file, then compare the two
-          Object.keys(require(`../language_configs/${lang}.rules.js`))
+          Object.keys(require(`../../language_configs/${lang}.rules.js`))
             .concat(treeMatcherTypes)
             .sort((a, b) => a.localeCompare(b))
             .forEach((rule, i) => expect(rule).toEqual(tokens[i]));

__tests__/transformers/collapse.test.js

@@ -0,0 +1,63 @@
+describe('collapse.js', () => {
+  const collapse = require('../../src/transformers/collapse.js');
+  const { makeNode } = require('../../src/utils/utils.js');
+
+  const start  = 0;
+  const end    = 5;
+  const noDetail = [];
+
+  const bottomNode = makeNode('bottom_node', start, end, noDetail, []);
+  const middleNode = makeNode('middle_node', start, end, noDetail, [ bottomNode ]);
+  const topNode    = makeNode('top_node', start, end, noDetail, [ middleNode ]);
+
+  it(`is a function`, () => {
+    expect(typeof(collapse)).toEqual('function');
+  });
+
+  it(`recursively removes nodes from the tree if those nodes:
+       • are flagged as collapsible in the runtime config, and
+       • have single children who occupy the exact same range`, () => {
+        const langRuntimeConfig1 = {
+          'rules': {
+           'top_node':    { 'collapse': true },
+           'middle_node': { 'collapse': true },
+           'bottom_node': { 'collapse': false },
+          },
+        };
+        expect(collapse(langRuntimeConfig1, topNode)).toBe(bottomNode);
+
+        const langRuntimeConfig2 = {
+          'rules': {
+            'top_node':    { 'collapse': true },
+            'middle_node': { 'collapse': false },
+            'bottom_node': { 'collapse': true }
+          },
+        };
+        expect(collapse(langRuntimeConfig2, topNode)).toBe(middleNode);
+  });
+
+  it(`does NOT remove nodes from the tree if they are NOT flagged as collapsible`,
+     () => {
+        const langRuntimeConfig = {
+          'rules': {
+            'top_node':    {},
+            'middle_node': { 'collapse': true },
+            'bottom_node': { 'collapse': true },
+          },
+        };
+        expect(collapse(langRuntimeConfig, topNode)).toBe(topNode);
+  });
+
+  it(`does NOT remove nodes from the tree if they ARE flagged as collapsible,
+      but whose children do NOT occupt identical ranges`, () => {
+        const childNode  = makeNode('child', 0, 3, noDetail, []);
+        const parentNode = makeNode('parent', 0, 5, noDetail, [ childNode ]);
+        const langRuntimeConfig = {
+          'rules': {
+            'parent': { 'collapse': true },
+            'child':  { 'collapse': true },
+          },
+        };
+        expect(collapse(langRuntimeConfig, parentNode)).toBe(parentNode);
+  });
+});

__tests__/transformers/simplify_node.test.js

@@ -0,0 +1,63 @@
+describe(`simplify_node.js`, () => {
+  const simplify_node = require('../../src/transformers/simplify_node.js');
+  const {
+    makeAntlrNode,
+    makeAntlrTerminal,
+    makeNode,
+    makeTerminal,
+  } = require('../../src/utils/utils.js');
+  const lang_runtime_config = {
+    symbol_name_map: [
+      '.BLAH',
+      '.FOOBAR',
+      '.QUXBAZ',
+    ],
+    rule_name_map: [
+      'floop_type',
+      'groop_loop',
+    ],
+  };
+
+  it(`is a function`, () => {
+    expect(typeof(simplify_node)).toEqual('function');
+  });
+
+  it(`recursively extracts the following properties from non-terminal
+      input nodes:
+        • type
+        • begin
+        • end
+        • tags
+        • children`, () => {
+    // Different forms for starts and stops are intentionally
+    // used here, as both can be found in real ANTLR nodes.
+    const input_node = (
+      makeAntlrNode(0, { start: 0 }, { stop: 5 }, [
+        makeAntlrNode(1, { start: 0, stop: 4 }, undefined, undefined),
+      ]));
+
+    const { rule_name_map } = lang_runtime_config;
+    const expected = (
+      makeNode(rule_name_map[0], 0, 5 + 1, [], [
+        makeNode(rule_name_map[1], 0, 4 + 1, [], []),
+      ]));
+    expect(simplify_node(lang_runtime_config, input_node)).toEqual(expected);
+  });
+
+  it(`extracts the following properties from terminal input nodes:
+      • type
+      • begin
+      • end
+      • tags
+      • children
+      • text`, () => {
+    const input_terminal = (
+      makeAntlrTerminal(0, 0, 5, 'foobar')
+    );
+    const { symbol_name_map } = lang_runtime_config;
+    const expected = (
+      makeTerminal(symbol_name_map[0 + 2], 0, 6, 'foobar', [])
+    );
+    expect(simplify_node(lang_runtime_config, input_terminal)).toEqual(expected);
+  });
+});

__tests__/tree/code/rename_random.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+mv "$1" "snippet.`node --print "Math.random().toString(36).substr(2, 8)"`.${1##*.}"

__tests__/tree/code/snippet.hixfx981.rb

@@ -0,0 +1,24 @@
+class Unit
+  # A list of all the units we know.
+  @@units = { }
+
+  # Access to @@units.
+  def Unit.exists(n)
+    return @@units.has_key?(n)
+  end
+  def Unit.named(n)
+    return @@units[n]
+  end
+
+  # If this were Java, I'd define an abstract function isbase() which tells
+  # if this object is a BaseUnit or not.
+  def initialize(name)
+    @name = name
+    @@units[name] = self
+    @@units[name + 's'] = self
+  end
+  attr_reader :name
+  def alias(*names)
+    names.each { |n| @@units[n] = self }
+  end
+end

__tests__/tree/code/snippet.k17eu4f2.java

@@ -0,0 +1,5 @@
+public class HelloWorldApp {
+    public static void main(String[] args) {
+        System.out.println("Hello World!");
+    }
+}

__tests__/tree/code/snippet.kt29xnfw.py

@@ -0,0 +1,7 @@
+import sys
+try:
+    total = sum(int(arg) for arg in sys.argv[1:])
+    print('sum =', total)
+except ValueError:
+    print('Please supply integer arguments')
+

__tests__/tree/code/snippet.voc84cjo.py

@@ -0,0 +1 @@
+print('Hello, world!')

__tests__/tree/compare.test.js

@@ -0,0 +1,163 @@
+const path = require('path');
+const fs = require('fs-promise');
+const antlr = require('antlr4');
+const expect_error = require('../../src/utils/expect_error.js');
+const run_antlr = require('../../src/utils/run_antlr.js');
+const run_java = require('../../src/utils/run_java.js');
+const webpack = require('webpack');
+
+const config = require('../../config');
+
+const prepareJava = async function(lang_compile_config, lang_runtime_config) {
+  return await run_antlr(lang_compile_config, lang_runtime_config, 'Java');
+}
+
+const genTreeViaJava = async function(lang_runtime_config, antlr_result, code) {
+  const {
+    build_dir
+  } = antlr_result;
+
+  const java_sources = [
+    lang_runtime_config.language + 'Lexer.java',
+    lang_runtime_config.language + 'Parser.java',
+  ];
+
+  // Add CLASSPATH to environment
+  const environment = Object.create(process.env);
+  const classpath = environment.CLASSPATH ? environment.CLASSPATH.split(':') : [];
+  classpath.unshift(path.resolve(__dirname, '../../bin/antlr-4.6-complete.jar'));
+  classpath.unshift('.');
+  environment.CLASSPATH = classpath.join(':');
+
+  const result = await run_java(
+    java_sources,
+    'org.antlr.v4.gui.TestRig',
+    [lang_runtime_config.language, lang_runtime_config.entry_rule, '-tree'],
+    {
+      'cwd': build_dir,
+      'env': environment,
+      //'stdio': ['pipe', 'pipe', process.stderr],
+    },
+    code
+  );
+
+  if (result.code) {
+    throw result.stderr.toString();
+  } else {
+    return result.stdout.toString();
+  }
+};
+
+const prepareJs = async function(lang_compile_config, lang_runtime_config) {
+  const output_path = path.resolve(config.build_path, 'output');
+
+  const webpack_config = await require('../../webpack.config.js')({
+    'langs': lang_runtime_config.language,
+    'optimize': 0,
+    'libraryTarget': 'commonjs2',
+    'outputPath': output_path,
+    'enable_debug': false,
+  });
+
+  if (webpack_config.length !== 1) {
+    throw new Error('Unexpected webpack output');
+  }
+
+  const compiler = webpack(webpack_config);
+
+  await new Promise(function(resolve, reject) {
+    compiler.run(function(err, stats) {
+      if (err) {
+        reject(err);
+      } else {
+        resolve(stats);
+      }
+    });
+  });
+
+  return require(path.resolve(output_path, webpack_config[0].output.filename));
+}
+
+const genTreeViaJs = async function(lang_runtime_config, js_parser, code) {
+  const tree = js_parser(code, function(err) {
+    throw err;
+  }, {
+    'return_antlr_tree': true,
+  });
+
+  return tree.toStringTree(tree.parser.ruleNames);
+};
+
+
+let prepare_cache = {};
+
+const compare = async function(language_name, code_file) {
+  const lang_compile_config = require(`../../language_configs/${language_name}.compile.js`);
+  const lang_runtime_config = require(`../../language_configs/${language_name}.runtime.js`);
+
+  if (typeof prepare_cache[language_name] === 'undefined') {
+    prepare_cache[language_name] = Promise.all([
+      prepareJava(lang_compile_config, lang_runtime_config),
+      prepareJs(lang_compile_config, lang_runtime_config),
+    ]);
+  }
+
+  const t1 = Date.now();
+
+  const res = await prepare_cache[language_name];
+  const antlr_result = res[0];
+  const js_parser = res[1];
+  const t2 = Date.now();
+
+  const code = await fs.readFile(code_file, {'encoding': 'utf8'});
+  const t3 = Date.now();
+
+  const treeViaJava = await genTreeViaJava(lang_runtime_config, antlr_result, code);
+  const t4 = Date.now();
+
+  const treeViaJs = await genTreeViaJs(lang_runtime_config, js_parser, code);
+  const t5 = Date.now();
+
+  console.log('compare(' + code_file + ') prepare:  ' + (t2 - t1) + 'ms');
+  console.log('compare(' + code_file + ') readFile: ' + (t3 - t2) + 'ms');
+  console.log('compare(' + code_file + ') genJava:  ' + (t4 - t3) + 'ms');
+  console.log('compare(' + code_file + ') genJs:    ' + (t5 - t4) + 'ms');
+
+  return treeViaJava.trim() === treeViaJs.trim();
+};
+
+
+
+describe('grammars-v4/', () => {
+  let prev_timeout;
+  beforeAll(() => {
+    prev_timeout = jasmine.DEFAULT_TIMEOUT_INTERVAL;
+    jasmine.DEFAULT_TIMEOUT_INTERVAL = 60000;
+    return fs.remove(config.build_path).catch(err => {
+      console.error(err);
+    });
+  });
+  afterAll(() => {
+    jasmine.DEFAULT_TIMEOUT_INTERVAL = prev_timeout;
+  });
+
+  const test_snippet = function(lang_key, description, code_filename) {
+    it(description, () => {
+      expect.assertions(1);
+      return compare(lang_key, __dirname + '/code/' + code_filename).then(data => {
+        expect(data).toBeTruthy();
+      }).catch(err => {
+        console.error(err);
+      });
+    });
+  };
+
+  describe('grammars-v4/python3/', () => {
+    test_snippet('python3', 'handles basic hello worlds', 'snippet.voc84cjo.py');
+    test_snippet('python3', 'handles a script adding the input arguments', 'snippet.kt29xnfw.py');
+  });
+
+  describe('grammars-v4/java8/', () => {
+    test_snippet('java8', 'handles basic hello worlds', 'snippet.k17eu4f2.java');
+  });
+});

__tests__/utils/__snapshots__/utils.test.js.snap

@@ -0,0 +1,14 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`test-utils.js makeNode returns the expected object 1`] = `
+Object {
+  "ast_type": "Some Type",
+  "begin": 0,
+  "children": Array [],
+  "detail": Array [],
+  "end": 5,
+  "tags": Array [
+    "Some Type",
+  ],
+}
+`;

__tests__/utils/utils.test.js

@@ -0,0 +1,10 @@
+const { makeNode } = require('../../src/utils/utils.js');
+
+describe(`test-utils.js`, () => {
+  describe(`makeNode`, () => {
+    it(`returns the expected object`, () => {
+      const node = makeNode('Some Type', 0, 5, [], []);
+      expect(node).toMatchSnapshot();
+    });
+  });
+});