From f2fcddc4c916124d78fc99ed9dc217b39d1ee1c4 Mon Sep 17 00:00:00 2001 From: sblundy Date: Sun, 11 Aug 2019 06:54:26 -0700 Subject: [PATCH] Add comma as bareword char, except in braced lists --- build.gradle.kts | 2 +- .../sblundy/elvish/lang/ElvishLexer.java | 347 +++++++++--------- .../sblundy/elvish/lang/ElvishParser.java | 47 ++- .../sblundy/elvish/psi/ElvishBraced.java | 2 +- .../sblundy/elvish/psi/ElvishTypes.java | 6 +- .../sblundy/elvish/psi/ElvishVisitor.java | 4 + .../elvish/psi/impl/ElvishBracedImpl.java | 4 +- src/main/grammars/Elvish.bnf | 9 +- src/main/grammars/Elvish.flex | 1 + .../github/sblundy/elvish/lang/parserUtils.kt | 7 - .../elvish/lang/bareword-with-comma.elv | 1 + .../elvish/lang/bareword-with-comma.txt | 21 ++ .../elvish/lang/braced-list.tokens.txt | 4 +- .../sblundy/elvish/lang/braced-list.txt | 10 +- 14 files changed, 262 insertions(+), 203 deletions(-) create mode 100644 src/test/resources/com/github/sblundy/elvish/lang/bareword-with-comma.elv create mode 100644 src/test/resources/com/github/sblundy/elvish/lang/bareword-with-comma.txt diff --git a/build.gradle.kts b/build.gradle.kts index a49ac72..b80e64a 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -8,7 +8,7 @@ plugins { } group = "com.github.sblundy" -version = "1.0-beta-2" +version = "1.0-beta-3" repositories { mavenCentral() diff --git a/src/gen/com/github/sblundy/elvish/lang/ElvishLexer.java b/src/gen/com/github/sblundy/elvish/lang/ElvishLexer.java index d8afdb3..26d8c7b 100644 --- a/src/gen/com/github/sblundy/elvish/lang/ElvishLexer.java +++ b/src/gen/com/github/sblundy/elvish/lang/ElvishLexer.java @@ -96,11 +96,11 @@ public static int ZZ_CMAP(int ch) { /* The ZZ_CMAP_A table has 664 entries */ static final char ZZ_CMAP_A[] = zzUnpackCMap( - "\11\0\1\46\1\1\2\64\1\45\2\0\1\46\1\36\1\57\1\2\1\47\1\36\1\60\1\56\1\54\1"+ - "\55\1\44\1\36\1\0\1\41\2\36\10\7\2\5\1\41\1\63\1\40\1\37\1\40\1\53\1\43\6"+ - "\14\16\42\1\4\5\42\1\52\1\3\1\13\1\11\1\42\1\0\1\31\1\16\1\12\1\35\1\17\1"+ + "\11\0\1\47\1\1\2\65\1\46\2\0\1\47\1\36\1\60\1\2\1\50\1\36\1\61\1\57\1\55\1"+ + "\56\1\45\1\36\1\44\1\41\2\36\10\7\2\5\1\41\1\64\1\40\1\37\1\40\1\54\1\43\6"+ + "\14\16\42\1\4\5\42\1\53\1\3\1\13\1\11\1\42\1\0\1\31\1\16\1\12\1\35\1\17\1"+ "\23\1\41\1\25\1\22\2\41\1\20\1\41\1\30\1\33\1\26\1\41\1\34\1\21\1\27\1\6\1"+ - "\15\1\24\1\10\1\32\1\41\1\50\1\62\1\51\1\61\6\0\1\64\2\0\10\41\2\0\6\41\4"+ + "\15\1\24\1\10\1\32\1\41\1\51\1\63\1\52\1\62\6\0\1\65\2\0\10\41\2\0\6\41\4"+ "\0\7\41\1\0\1\41\1\0\4\41\1\0\5\41\1\0\16\41\2\0\2\41\2\0\6\41\5\0\5\41\3"+ "\0\5\41\1\0\10\41\1\0\4\41\2\0\5\41\6\0\6\41\2\0\4\41\2\0\1\41\1\0\4\41\1"+ "\0\10\41\2\0\2\41\2\0\6\41\1\0\7\41\1\0\1\41\3\0\4\41\2\0\5\41\2\0\4\41\10"+ @@ -111,7 +111,7 @@ public static int ZZ_CMAP(int ch) { "\5\0\2\41\7\0\1\41\2\0\2\41\5\0\4\41\1\0\1\41\4\0\1\41\4\0\6\41\1\0\1\41\3"+ "\0\3\41\4\0\2\41\1\0\1\41\2\0\2\41\1\0\1\41\2\0\1\41\3\0\3\41\1\0\1\41\1\0"+ "\1\41\5\0\1\41\2\0\1\41\1\0\2\41\4\0\1\41\3\0\4\41\3\0\5\41\1\0\1\41\1\0\1"+ - "\41\1\0\1\41\1\0\1\41\2\0\3\41\1\0\2\41\2\64\6\41\5\0\3\41\1\0\6\41\1\0\2"+ + "\41\1\0\1\41\1\0\1\41\2\0\3\41\1\0\2\41\2\65\6\41\5\0\3\41\1\0\6\41\1\0\2"+ "\41\1\0\2\41\1\0\5\41\1\0\4\41\1\0\5\41\2\0"); /** @@ -121,16 +121,16 @@ public static int ZZ_CMAP(int ch) { private static final String ZZ_ACTION_PACKED_0 = "\3\0\1\1\1\2\1\3\1\1\2\4\1\5\1\6"+ - "\6\4\1\7\1\10\1\11\1\12\1\2\1\13\1\14"+ + "\6\4\1\7\1\10\1\11\1\12\1\13\1\2\1\14"+ "\1\15\1\16\1\17\1\20\1\21\1\22\1\23\1\24"+ - "\1\25\1\26\1\27\1\30\1\31\1\32\1\31\1\0"+ - "\1\33\1\3\2\34\3\4\1\35\1\4\1\36\4\4"+ - "\1\37\6\40\1\41\4\4\1\42\1\4\1\43\1\44"+ - "\4\0\1\4\1\45\1\46\2\4\2\0\2\4\1\47"+ - "\1\0\1\50\1\4\1\0\1\51"; + "\1\25\1\26\1\27\1\30\1\31\1\32\1\33\1\32"+ + "\1\0\1\34\1\3\2\35\3\4\1\36\1\4\1\37"+ + "\4\4\1\40\6\41\1\42\4\4\1\43\1\4\1\44"+ + "\1\45\4\0\1\4\1\46\1\47\2\4\2\0\2\4"+ + "\1\50\1\0\1\51\1\4\1\0\1\52"; private static int [] zzUnpackAction() { - int [] result = new int[89]; + int [] result = new int[90]; int offset = 0; offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result); return result; @@ -155,21 +155,21 @@ private static int zzUnpackAction(String packed, int offset, int [] result) { private static final int [] ZZ_ROWMAP = zzUnpackRowMap(); private static final String ZZ_ROWMAP_PACKED_0 = - "\0\0\0\65\0\152\0\237\0\237\0\324\0\u0109\0\u013e"+ - "\0\u0173\0\u01a8\0\237\0\u01dd\0\u0212\0\u0247\0\u027c\0\u02b1"+ - "\0\u02e6\0\u031b\0\237\0\u031b\0\u01a8\0\u0350\0\u0385\0\237"+ - "\0\237\0\237\0\237\0\237\0\237\0\237\0\237\0\237"+ - "\0\237\0\u013e\0\237\0\237\0\u03ba\0\u03ef\0\u0424\0\u0459"+ - "\0\237\0\u048e\0\237\0\u04c3\0\u04f8\0\u052d\0\u0562\0\u013e"+ - "\0\u0597\0\u013e\0\u05cc\0\u0601\0\u0636\0\u066b\0\237\0\237"+ - "\0\u06a0\0\u06d5\0\u070a\0\u073f\0\u0774\0\u013e\0\u07a9\0\u07de"+ - "\0\u0813\0\u0848\0\u013e\0\u087d\0\u013e\0\u013e\0\u08b2\0\u08e7"+ - "\0\u091c\0\u0951\0\u0986\0\u013e\0\u013e\0\u09bb\0\u09f0\0\u0a25"+ - "\0\u073f\0\u0a5a\0\u0a8f\0\u013e\0\u0ac4\0\u013e\0\u0af9\0\u06d5"+ - "\0\u013e"; + "\0\0\0\66\0\154\0\242\0\242\0\330\0\u010e\0\u0144"+ + "\0\u017a\0\u01b0\0\242\0\u01e6\0\u021c\0\u0252\0\u0288\0\u02be"+ + "\0\u02f4\0\u032a\0\242\0\u032a\0\242\0\u01b0\0\u0360\0\u0396"+ + "\0\242\0\242\0\242\0\242\0\242\0\242\0\242\0\242"+ + "\0\242\0\242\0\u0144\0\242\0\242\0\u03cc\0\u0402\0\u0438"+ + "\0\u046e\0\242\0\u04a4\0\242\0\u04da\0\u0510\0\u0546\0\u057c"+ + "\0\u0144\0\u05b2\0\u0144\0\u05e8\0\u061e\0\u0654\0\u068a\0\242"+ + "\0\242\0\u06c0\0\u06f6\0\u072c\0\u0762\0\u0798\0\u0144\0\u07ce"+ + "\0\u0804\0\u083a\0\u0870\0\u0144\0\u08a6\0\u0144\0\u0144\0\u08dc"+ + "\0\u0912\0\u0948\0\u097e\0\u09b4\0\u0144\0\u0144\0\u09ea\0\u0a20"+ + "\0\u0a56\0\u0762\0\u0a8c\0\u0ac2\0\u0144\0\u0af8\0\u0144\0\u0b2e"+ + "\0\u06f6\0\u0144"; private static int [] zzUnpackRowMap() { - int [] result = new int[89]; + int [] result = new int[90]; int offset = 0; offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result); return result; @@ -197,77 +197,77 @@ private static int zzUnpackRowMap(String packed, int offset, int [] result) { "\2\10\1\20\5\10\1\21\1\22\1\23\1\12\2\10"+ "\1\24\1\25\1\26\1\27\1\30\1\31\1\32\1\33"+ "\1\34\1\35\1\36\1\37\1\40\1\41\1\42\1\43"+ - "\1\44\1\4\56\45\1\46\6\45\3\47\1\50\53\47"+ - "\1\51\5\47\65\0\1\6\1\52\63\6\1\0\1\53"+ - "\43\0\1\54\23\0\5\10\1\0\1\10\1\0\22\10"+ - "\3\0\2\10\16\0\1\10\7\0\5\10\1\0\1\10"+ - "\1\0\5\10\1\55\14\10\3\0\2\10\16\0\1\10"+ - "\14\0\1\12\26\0\1\12\3\0\1\12\24\0\4\10"+ - "\1\56\1\0\1\10\1\0\4\10\1\57\15\10\3\0"+ - "\2\10\16\0\1\10\7\0\5\10\1\0\1\10\1\0"+ - "\7\10\1\60\12\10\3\0\2\10\16\0\1\10\7\0"+ - "\5\10\1\0\1\10\1\0\6\10\1\61\5\10\1\62"+ - "\2\10\1\63\2\10\3\0\2\10\16\0\1\10\7\0"+ - "\5\10\1\0\1\10\1\0\11\10\1\64\10\10\3\0"+ - "\2\10\16\0\1\10\7\0\5\10\1\0\1\10\1\0"+ - "\20\10\1\65\1\10\3\0\2\10\16\0\1\10\7\0"+ - "\5\10\1\0\1\10\1\0\3\10\1\66\16\10\3\0"+ - "\2\10\16\0\1\10\41\0\1\22\4\0\1\22\22\0"+ - "\1\5\131\0\1\27\16\0\56\45\1\0\6\45\56\0"+ - "\1\67\6\0\3\47\1\0\53\47\1\0\5\47\1\70"+ - "\1\0\1\70\1\67\1\71\1\70\1\72\1\73\1\74"+ - "\2\75\2\70\3\67\3\70\1\67\3\70\3\67\2\70"+ - "\1\67\10\70\1\0\11\70\1\67\4\70\2\0\1\52"+ - "\43\0\1\52\20\0\1\53\67\0\5\10\1\0\1\10"+ - "\1\0\3\10\1\76\16\10\3\0\2\10\16\0\1\10"+ - "\7\0\5\10\1\0\1\77\1\0\22\10\3\0\2\10"+ - "\16\0\1\10\7\0\5\10\1\0\1\10\1\0\5\10"+ - "\1\100\1\101\13\10\3\0\2\10\16\0\1\10\7\0"+ - "\5\10\1\0\1\10\1\0\14\10\1\102\5\10\3\0"+ - "\2\10\16\0\1\10\7\0\5\10\1\0\1\10\1\0"+ - "\20\10\1\103\1\10\3\0\2\10\16\0\1\10\7\0"+ - "\5\10\1\0\1\10\1\0\6\10\1\104\13\10\3\0"+ - "\2\10\16\0\1\10\7\0\5\10\1\0\1\10\1\0"+ - "\16\10\1\105\3\10\3\0\2\10\16\0\1\10\7\0"+ - "\5\10\1\0\1\10\1\0\4\10\1\106\15\10\3\0"+ - "\2\10\16\0\1\10\10\0\1\107\1\0\1\107\2\0"+ - "\1\107\1\0\1\107\1\0\2\107\3\0\1\107\5\0"+ - "\1\107\3\0\1\107\34\0\1\110\1\0\1\110\2\0"+ - "\1\110\1\0\1\110\1\0\2\110\3\0\1\110\5\0"+ - "\1\110\3\0\1\110\36\0\1\111\62\0\1\112\1\0"+ - "\1\112\2\0\1\112\1\0\1\112\1\0\2\112\3\0"+ - "\1\112\5\0\1\112\3\0\1\112\32\0\2\67\4\0"+ - "\1\67\1\0\2\67\25\0\2\67\6\0\1\67\16\0"+ - "\5\10\1\0\1\10\1\0\3\10\1\113\16\10\3\0"+ - "\2\10\16\0\1\10\7\0\5\10\1\0\1\10\1\0"+ - "\3\10\1\114\16\10\3\0\2\10\16\0\1\10\7\0"+ - "\5\10\1\0\1\10\1\0\7\10\1\115\12\10\3\0"+ - "\2\10\16\0\1\10\7\0\5\10\1\0\1\10\1\0"+ - "\15\10\1\116\4\10\3\0\2\10\16\0\1\10\7\0"+ - "\5\10\1\0\1\10\1\0\4\10\1\117\15\10\3\0"+ - "\2\10\16\0\1\10\10\0\1\120\1\0\1\120\2\0"+ - "\1\120\1\0\1\120\1\0\2\120\3\0\1\120\5\0"+ - "\1\120\3\0\1\120\34\0\1\121\1\0\1\121\2\0"+ - "\1\121\1\0\1\121\1\0\2\121\3\0\1\121\5\0"+ - "\1\121\3\0\1\121\36\0\1\67\62\0\1\67\1\0"+ - "\1\67\2\0\1\67\1\0\1\67\1\0\2\67\3\0"+ - "\1\67\5\0\1\67\3\0\1\67\33\0\5\10\1\0"+ - "\1\10\1\0\12\10\1\122\7\10\3\0\2\10\16\0"+ - "\1\10\7\0\5\10\1\0\1\10\1\0\4\10\1\123"+ - "\15\10\3\0\2\10\16\0\1\10\7\0\5\10\1\0"+ - "\1\10\1\0\3\10\1\124\16\10\3\0\2\10\16\0"+ - "\1\10\10\0\1\125\1\0\1\125\2\0\1\125\1\0"+ - "\1\125\1\0\2\125\3\0\1\125\5\0\1\125\3\0"+ - "\1\125\33\0\5\10\1\0\1\10\1\0\13\10\1\126"+ - "\6\10\3\0\2\10\16\0\1\10\7\0\5\10\1\0"+ - "\1\10\1\0\4\10\1\127\15\10\3\0\2\10\16\0"+ - "\1\10\10\0\1\130\1\0\1\130\2\0\1\130\1\0"+ - "\1\130\1\0\2\130\3\0\1\130\5\0\1\130\3\0"+ - "\1\130\33\0\5\10\1\0\1\10\1\0\16\10\1\131"+ - "\3\10\3\0\2\10\16\0\1\10\3\0"; + "\1\44\1\45\1\4\57\46\1\47\6\46\3\50\1\51"+ + "\54\50\1\52\5\50\66\0\1\6\1\53\64\6\1\0"+ + "\1\54\44\0\1\55\23\0\5\10\1\0\1\10\1\0"+ + "\22\10\3\0\2\10\17\0\1\10\7\0\5\10\1\0"+ + "\1\10\1\0\5\10\1\56\14\10\3\0\2\10\17\0"+ + "\1\10\14\0\1\12\26\0\1\12\4\0\1\12\24\0"+ + "\4\10\1\57\1\0\1\10\1\0\4\10\1\60\15\10"+ + "\3\0\2\10\17\0\1\10\7\0\5\10\1\0\1\10"+ + "\1\0\7\10\1\61\12\10\3\0\2\10\17\0\1\10"+ + "\7\0\5\10\1\0\1\10\1\0\6\10\1\62\5\10"+ + "\1\63\2\10\1\64\2\10\3\0\2\10\17\0\1\10"+ + "\7\0\5\10\1\0\1\10\1\0\11\10\1\65\10\10"+ + "\3\0\2\10\17\0\1\10\7\0\5\10\1\0\1\10"+ + "\1\0\20\10\1\66\1\10\3\0\2\10\17\0\1\10"+ + "\7\0\5\10\1\0\1\10\1\0\3\10\1\67\16\10"+ + "\3\0\2\10\17\0\1\10\41\0\1\22\4\0\1\22"+ + "\23\0\1\5\133\0\1\30\16\0\57\46\1\0\6\46"+ + "\57\0\1\70\6\0\3\50\1\0\54\50\1\0\5\50"+ + "\1\71\1\0\1\71\1\70\1\72\1\71\1\73\1\74"+ + "\1\75\2\76\2\71\3\70\3\71\1\70\3\71\3\70"+ + "\2\71\1\70\11\71\1\0\11\71\1\70\4\71\2\0"+ + "\1\53\44\0\1\53\20\0\1\54\70\0\5\10\1\0"+ + "\1\10\1\0\3\10\1\77\16\10\3\0\2\10\17\0"+ + "\1\10\7\0\5\10\1\0\1\100\1\0\22\10\3\0"+ + "\2\10\17\0\1\10\7\0\5\10\1\0\1\10\1\0"+ + "\5\10\1\101\1\102\13\10\3\0\2\10\17\0\1\10"+ + "\7\0\5\10\1\0\1\10\1\0\14\10\1\103\5\10"+ + "\3\0\2\10\17\0\1\10\7\0\5\10\1\0\1\10"+ + "\1\0\20\10\1\104\1\10\3\0\2\10\17\0\1\10"+ + "\7\0\5\10\1\0\1\10\1\0\6\10\1\105\13\10"+ + "\3\0\2\10\17\0\1\10\7\0\5\10\1\0\1\10"+ + "\1\0\16\10\1\106\3\10\3\0\2\10\17\0\1\10"+ + "\7\0\5\10\1\0\1\10\1\0\4\10\1\107\15\10"+ + "\3\0\2\10\17\0\1\10\10\0\1\110\1\0\1\110"+ + "\2\0\1\110\1\0\1\110\1\0\2\110\3\0\1\110"+ + "\5\0\1\110\3\0\1\110\35\0\1\111\1\0\1\111"+ + "\2\0\1\111\1\0\1\111\1\0\2\111\3\0\1\111"+ + "\5\0\1\111\3\0\1\111\37\0\1\112\63\0\1\113"+ + "\1\0\1\113\2\0\1\113\1\0\1\113\1\0\2\113"+ + "\3\0\1\113\5\0\1\113\3\0\1\113\33\0\2\70"+ + "\4\0\1\70\1\0\2\70\25\0\2\70\7\0\1\70"+ + "\16\0\5\10\1\0\1\10\1\0\3\10\1\114\16\10"+ + "\3\0\2\10\17\0\1\10\7\0\5\10\1\0\1\10"+ + "\1\0\3\10\1\115\16\10\3\0\2\10\17\0\1\10"+ + "\7\0\5\10\1\0\1\10\1\0\7\10\1\116\12\10"+ + "\3\0\2\10\17\0\1\10\7\0\5\10\1\0\1\10"+ + "\1\0\15\10\1\117\4\10\3\0\2\10\17\0\1\10"+ + "\7\0\5\10\1\0\1\10\1\0\4\10\1\120\15\10"+ + "\3\0\2\10\17\0\1\10\10\0\1\121\1\0\1\121"+ + "\2\0\1\121\1\0\1\121\1\0\2\121\3\0\1\121"+ + "\5\0\1\121\3\0\1\121\35\0\1\122\1\0\1\122"+ + "\2\0\1\122\1\0\1\122\1\0\2\122\3\0\1\122"+ + "\5\0\1\122\3\0\1\122\37\0\1\70\63\0\1\70"+ + "\1\0\1\70\2\0\1\70\1\0\1\70\1\0\2\70"+ + "\3\0\1\70\5\0\1\70\3\0\1\70\34\0\5\10"+ + "\1\0\1\10\1\0\12\10\1\123\7\10\3\0\2\10"+ + "\17\0\1\10\7\0\5\10\1\0\1\10\1\0\4\10"+ + "\1\124\15\10\3\0\2\10\17\0\1\10\7\0\5\10"+ + "\1\0\1\10\1\0\3\10\1\125\16\10\3\0\2\10"+ + "\17\0\1\10\10\0\1\126\1\0\1\126\2\0\1\126"+ + "\1\0\1\126\1\0\2\126\3\0\1\126\5\0\1\126"+ + "\3\0\1\126\34\0\5\10\1\0\1\10\1\0\13\10"+ + "\1\127\6\10\3\0\2\10\17\0\1\10\7\0\5\10"+ + "\1\0\1\10\1\0\4\10\1\130\15\10\3\0\2\10"+ + "\17\0\1\10\10\0\1\131\1\0\1\131\2\0\1\131"+ + "\1\0\1\131\1\0\2\131\3\0\1\131\5\0\1\131"+ + "\3\0\1\131\34\0\5\10\1\0\1\10\1\0\16\10"+ + "\1\132\3\10\3\0\2\10\17\0\1\10\3\0"; private static int [] zzUnpackTrans() { - int [] result = new int[2862]; + int [] result = new int[2916]; int offset = 0; offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result); return result; @@ -305,13 +305,13 @@ private static int zzUnpackTrans(String packed, int offset, int [] result) { private static final int [] ZZ_ATTRIBUTE = zzUnpackAttribute(); private static final String ZZ_ATTRIBUTE_PACKED_0 = - "\3\0\2\11\5\1\1\11\7\1\1\11\4\1\12\11"+ - "\1\1\2\11\3\1\1\0\1\11\1\1\1\11\13\1"+ - "\2\11\16\1\4\0\5\1\2\0\3\1\1\0\2\1"+ - "\1\0\1\1"; + "\3\0\2\11\5\1\1\11\7\1\1\11\1\1\1\11"+ + "\3\1\12\11\1\1\2\11\3\1\1\0\1\11\1\1"+ + "\1\11\13\1\2\11\16\1\4\0\5\1\2\0\3\1"+ + "\1\0\2\1\1\0\1\1"; private static int [] zzUnpackAttribute() { - int [] result = new int[89]; + int [] result = new int[90]; int offset = 0; offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result); return result; @@ -630,211 +630,216 @@ else if (zzAtEOF) { { return TokenType.BAD_CHARACTER; } // fall through - case 42: break; + case 43: break; case 2: { return ElvishTypes.EOL; } // fall through - case 43: break; + case 44: break; case 3: { return ElvishTypes.COMMENT; } // fall through - case 44: break; + case 45: break; case 4: { return ElvishTypes.VARIABLE_CHAR; } // fall through - case 45: break; + case 46: break; case 5: { return ElvishTypes.COMMAND_BAREWORD_CHAR; } // fall through - case 46: break; + case 47: break; case 6: { return ElvishTypes.CLOSE_BRACKET; } // fall through - case 47: break; + case 48: break; case 7: { return ElvishTypes.BAREWORD_CHAR; } // fall through - case 48: break; + case 49: break; case 8: { return ElvishTypes.EQUALS; } // fall through - case 49: break; + case 50: break; case 9: { return ElvishTypes.AT_SYMBOL; } // fall through - case 50: break; + case 51: break; case 10: - { return ElvishTypes.WILDCARD; + { return ElvishTypes.COMMA; } // fall through - case 51: break; + case 52: break; case 11: - { return ElvishTypes.INLINE_WHITESPACE; + { return ElvishTypes.WILDCARD; } // fall through - case 52: break; + case 53: break; case 12: - { return ElvishTypes.DOLLAR_SIGN; + { return ElvishTypes.INLINE_WHITESPACE; } // fall through - case 53: break; + case 54: break; case 13: - { return ElvishTypes.OPEN_BRACE; + { return ElvishTypes.DOLLAR_SIGN; } // fall through - case 54: break; + case 55: break; case 14: - { return ElvishTypes.CLOSE_BRACE; + { return ElvishTypes.OPEN_BRACE; } // fall through - case 55: break; + case 56: break; case 15: - { return ElvishTypes.OPEN_BRACKET; + { return ElvishTypes.CLOSE_BRACE; } // fall through - case 56: break; + case 57: break; case 16: - { return ElvishTypes.QUESTION; + { return ElvishTypes.OPEN_BRACKET; } // fall through - case 57: break; + case 58: break; case 17: - { return ElvishTypes.OPEN_PARAN; + { return ElvishTypes.QUESTION; } // fall through - case 58: break; + case 59: break; case 18: - { return ElvishTypes.CLOSE_PARAN; + { return ElvishTypes.OPEN_PARAN; } // fall through - case 59: break; + case 60: break; case 19: + { return ElvishTypes.CLOSE_PARAN; + } + // fall through + case 61: break; + case 20: { yyPushState(IN_SINGLE_QUOTE_STRING); return ElvishTypes.SINGLE_QUOTE; } // fall through - case 60: break; - case 20: + case 62: break; + case 21: { yyPushState(IN_DOUBLE_QUOTE_STRING); return ElvishTypes.DOUBLE_QUOTE; } // fall through - case 61: break; - case 21: + case 63: break; + case 22: { return ElvishTypes.AMPERSAND; } // fall through - case 62: break; - case 22: + case 64: break; + case 23: { return ElvishTypes.TILDE; } // fall through - case 63: break; - case 23: + case 65: break; + case 24: { return ElvishTypes.PIPE; } // fall through - case 64: break; - case 24: + case 66: break; + case 25: { return ElvishTypes.SEMICOLON; } // fall through - case 65: break; - case 25: + case 67: break; + case 26: { return ElvishTypes.TEXT; } // fall through - case 66: break; - case 26: + case 68: break; + case 27: { yyPopState(); return ElvishTypes.SINGLE_QUOTE; } // fall through - case 67: break; - case 27: + case 69: break; + case 28: { yyPopState(); return ElvishTypes.DOUBLE_QUOTE; } // fall through - case 68: break; - case 28: + case 70: break; + case 29: { return ElvishTypes.CONTINUATION; } // fall through - case 69: break; - case 29: + case 71: break; + case 30: { return ElvishTypes.KEYWORD_IF; } // fall through - case 70: break; - case 30: + case 72: break; + case 31: { return ElvishTypes.KEYWORD_FN; } // fall through - case 71: break; - case 31: + case 73: break; + case 32: { return ElvishTypes.ESCAPED_QUOTED_TEXT; } // fall through - case 72: break; - case 32: + case 74: break; + case 33: { return ElvishTypes.INVALID_ESCAPED_QUOTED_TEXT; } // fall through - case 73: break; - case 33: + case 75: break; + case 34: { return ElvishTypes.KEYWORD_USE; } // fall through - case 74: break; - case 34: + case 76: break; + case 35: { return ElvishTypes.KEYWORD_FOR; } // fall through - case 75: break; - case 35: + case 77: break; + case 36: { return ElvishTypes.KEYWORD_TRY; } // fall through - case 76: break; - case 36: + case 78: break; + case 37: { return ElvishTypes.KEYWORD_DEL; } // fall through - case 77: break; - case 37: + case 79: break; + case 38: { return ElvishTypes.KEYWORD_ELSE; } // fall through - case 78: break; - case 38: + case 80: break; + case 39: { return ElvishTypes.KEYWORD_ELIF; } // fall through - case 79: break; - case 39: + case 81: break; + case 40: { return ElvishTypes.KEYWORD_WHILE; } // fall through - case 80: break; - case 40: + case 82: break; + case 41: { return ElvishTypes.KEYWORD_EXCEPT; } // fall through - case 81: break; - case 41: + case 83: break; + case 42: { return ElvishTypes.KEYWORD_FINALLY; } // fall through - case 82: break; + case 84: break; default: zzScanError(ZZ_NO_MATCH); } diff --git a/src/gen/com/github/sblundy/elvish/lang/ElvishParser.java b/src/gen/com/github/sblundy/elvish/lang/ElvishParser.java index a9218bd..6f5ceb2 100644 --- a/src/gen/com/github/sblundy/elvish/lang/ElvishParser.java +++ b/src/gen/com/github/sblundy/elvish/lang/ElvishParser.java @@ -202,7 +202,7 @@ private static boolean AssignmentOnlyForm_2(PsiBuilder builder_, int level_) { } /* ********************************************************** */ - // (VARIABLE_CHAR|BAREWORD_CHAR|<>)+ + // (VARIABLE_CHAR|BAREWORD_CHAR|COMMA|<>)+ public static boolean Bareword(PsiBuilder builder_, int level_) { if (!recursion_guard_(builder_, level_, "Bareword")) return false; boolean result_; @@ -217,20 +217,21 @@ public static boolean Bareword(PsiBuilder builder_, int level_) { return result_; } - // VARIABLE_CHAR|BAREWORD_CHAR|<> + // VARIABLE_CHAR|BAREWORD_CHAR|COMMA|<> private static boolean Bareword_0(PsiBuilder builder_, int level_) { if (!recursion_guard_(builder_, level_, "Bareword_0")) return false; boolean result_; Marker marker_ = enter_section_(builder_); result_ = consumeToken(builder_, VARIABLE_CHAR); if (!result_) result_ = consumeToken(builder_, BAREWORD_CHAR); + if (!result_) result_ = consumeToken(builder_, COMMA); if (!result_) result_ = parseKeywordAsBareword(builder_, level_ + 1); exit_section_(builder_, marker_, null, result_); return result_; } /* ********************************************************** */ - // OPEN_BRACE (Space | EOL)* Bareword (BracedSep* Bareword)* (Space | EOL)* CLOSE_BRACE + // OPEN_BRACE (Space | EOL)* BracedBareword (BracedSep* BracedBareword)* (Space | EOL)* CLOSE_BRACE public static boolean Braced(PsiBuilder builder_, int level_) { if (!recursion_guard_(builder_, level_, "Braced")) return false; if (!nextTokenIs(builder_, OPEN_BRACE)) return false; @@ -238,7 +239,7 @@ public static boolean Braced(PsiBuilder builder_, int level_) { Marker marker_ = enter_section_(builder_); result_ = consumeToken(builder_, OPEN_BRACE); result_ = result_ && Braced_1(builder_, level_ + 1); - result_ = result_ && Bareword(builder_, level_ + 1); + result_ = result_ && BracedBareword(builder_, level_ + 1); result_ = result_ && Braced_3(builder_, level_ + 1); result_ = result_ && Braced_4(builder_, level_ + 1); result_ = result_ && consumeToken(builder_, CLOSE_BRACE); @@ -266,7 +267,7 @@ private static boolean Braced_1_0(PsiBuilder builder_, int level_) { return result_; } - // (BracedSep* Bareword)* + // (BracedSep* BracedBareword)* private static boolean Braced_3(PsiBuilder builder_, int level_) { if (!recursion_guard_(builder_, level_, "Braced_3")) return false; while (true) { @@ -277,13 +278,13 @@ private static boolean Braced_3(PsiBuilder builder_, int level_) { return true; } - // BracedSep* Bareword + // BracedSep* BracedBareword private static boolean Braced_3_0(PsiBuilder builder_, int level_) { if (!recursion_guard_(builder_, level_, "Braced_3_0")) return false; boolean result_; Marker marker_ = enter_section_(builder_); result_ = Braced_3_0_0(builder_, level_ + 1); - result_ = result_ && Bareword(builder_, level_ + 1); + result_ = result_ && BracedBareword(builder_, level_ + 1); exit_section_(builder_, marker_, null, result_); return result_; } @@ -320,13 +321,41 @@ private static boolean Braced_4_0(PsiBuilder builder_, int level_) { } /* ********************************************************** */ - // (Space | EOL)* <> (Space | EOL)* + // (VARIABLE_CHAR|BAREWORD_CHAR|<>)+ + public static boolean BracedBareword(PsiBuilder builder_, int level_) { + if (!recursion_guard_(builder_, level_, "BracedBareword")) return false; + boolean result_; + Marker marker_ = enter_section_(builder_, level_, _NONE_, BRACED_BAREWORD, ""); + result_ = BracedBareword_0(builder_, level_ + 1); + while (result_) { + int pos_ = current_position_(builder_); + if (!BracedBareword_0(builder_, level_ + 1)) break; + if (!empty_element_parsed_guard_(builder_, "BracedBareword", pos_)) break; + } + exit_section_(builder_, level_, marker_, result_, false, null); + return result_; + } + + // VARIABLE_CHAR|BAREWORD_CHAR|<> + private static boolean BracedBareword_0(PsiBuilder builder_, int level_) { + if (!recursion_guard_(builder_, level_, "BracedBareword_0")) return false; + boolean result_; + Marker marker_ = enter_section_(builder_); + result_ = consumeToken(builder_, VARIABLE_CHAR); + if (!result_) result_ = consumeToken(builder_, BAREWORD_CHAR); + if (!result_) result_ = parseKeywordAsBareword(builder_, level_ + 1); + exit_section_(builder_, marker_, null, result_); + return result_; + } + + /* ********************************************************** */ + // (Space | EOL)* COMMA (Space | EOL)* public static boolean BracedSep(PsiBuilder builder_, int level_) { if (!recursion_guard_(builder_, level_, "BracedSep")) return false; boolean result_; Marker marker_ = enter_section_(builder_, level_, _NONE_, BRACED_SEP, ""); result_ = BracedSep_0(builder_, level_ + 1); - result_ = result_ && parseCommaAsSeparator(builder_, level_ + 1); + result_ = result_ && consumeToken(builder_, COMMA); result_ = result_ && BracedSep_2(builder_, level_ + 1); exit_section_(builder_, level_, marker_, result_, false, null); return result_; diff --git a/src/gen/com/github/sblundy/elvish/psi/ElvishBraced.java b/src/gen/com/github/sblundy/elvish/psi/ElvishBraced.java index 2be1c8d..436253b 100644 --- a/src/gen/com/github/sblundy/elvish/psi/ElvishBraced.java +++ b/src/gen/com/github/sblundy/elvish/psi/ElvishBraced.java @@ -8,7 +8,7 @@ public interface ElvishBraced extends PsiElement { @NotNull - List getBarewordList(); + List getBracedBarewordList(); @NotNull List getBracedSepList(); diff --git a/src/gen/com/github/sblundy/elvish/psi/ElvishTypes.java b/src/gen/com/github/sblundy/elvish/psi/ElvishTypes.java index e5f8af2..be35fdf 100644 --- a/src/gen/com/github/sblundy/elvish/psi/ElvishTypes.java +++ b/src/gen/com/github/sblundy/elvish/psi/ElvishTypes.java @@ -13,6 +13,7 @@ public interface ElvishTypes { IElementType ASSIGNMENT = new ElvishElementType("ASSIGNMENT"); IElementType BAREWORD = new ElvishElementType("BAREWORD"); IElementType BRACED = new ElvishElementType("BRACED"); + IElementType BRACED_BAREWORD = new ElvishElementType("BRACED_BAREWORD"); IElementType BRACED_SEP = new ElvishElementType("BRACED_SEP"); IElementType CHUNK = new ElvishElementType("CHUNK"); IElementType COLLECTION = new ElvishElementType("COLLECTION"); @@ -51,10 +52,10 @@ public interface ElvishTypes { IElementType AMPERSAND = new ElvishTokenType("&"); IElementType AT_SYMBOL = new ElvishTokenType("@"); IElementType BAREWORD_CHAR = new ElvishTokenType("BAREWORD_CHAR"); - IElementType BRACED_SEPARATOR = new ElvishTokenType("[, ]+"); IElementType CLOSE_BRACE = new ElvishTokenType("}"); IElementType CLOSE_BRACKET = new ElvishTokenType("]"); IElementType CLOSE_PARAN = new ElvishTokenType(")"); + IElementType COMMA = new ElvishTokenType(","); IElementType COMMAND_BAREWORD_CHAR = new ElvishTokenType("COMMAND_BAREWORD_CHAR"); IElementType COMMENT = new ElvishTokenType("COMMENT"); IElementType CONTINUATION = new ElvishTokenType("CONTINUATION"); @@ -106,6 +107,9 @@ else if (type == BAREWORD) { else if (type == BRACED) { return new ElvishBracedImpl(node); } + else if (type == BRACED_BAREWORD) { + return new ElvishBracedBarewordImpl(node); + } else if (type == BRACED_SEP) { return new ElvishBracedSepImpl(node); } diff --git a/src/gen/com/github/sblundy/elvish/psi/ElvishVisitor.java b/src/gen/com/github/sblundy/elvish/psi/ElvishVisitor.java index 212b150..9b2cd1f 100644 --- a/src/gen/com/github/sblundy/elvish/psi/ElvishVisitor.java +++ b/src/gen/com/github/sblundy/elvish/psi/ElvishVisitor.java @@ -23,6 +23,10 @@ public void visitBraced(@NotNull ElvishBraced o) { visitPsiElement(o); } + public void visitBracedBareword(@NotNull ElvishBracedBareword o) { + visitPsiElement(o); + } + public void visitBracedSep(@NotNull ElvishBracedSep o) { visitPsiElement(o); } diff --git a/src/gen/com/github/sblundy/elvish/psi/impl/ElvishBracedImpl.java b/src/gen/com/github/sblundy/elvish/psi/impl/ElvishBracedImpl.java index 38de8ac..4187094 100644 --- a/src/gen/com/github/sblundy/elvish/psi/impl/ElvishBracedImpl.java +++ b/src/gen/com/github/sblundy/elvish/psi/impl/ElvishBracedImpl.java @@ -28,8 +28,8 @@ public void accept(@NotNull PsiElementVisitor visitor) { @Override @NotNull - public List getBarewordList() { - return PsiTreeUtil.getChildrenOfTypeAsList(this, ElvishBareword.class); + public List getBracedBarewordList() { + return PsiTreeUtil.getChildrenOfTypeAsList(this, ElvishBracedBareword.class); } @Override diff --git a/src/main/grammars/Elvish.bnf b/src/main/grammars/Elvish.bnf index a7e1015..7cc6a72 100644 --- a/src/main/grammars/Elvish.bnf +++ b/src/main/grammars/Elvish.bnf @@ -43,7 +43,7 @@ BAREWORD_CHAR="regexp:[./@%+!]" VARIABLE_CHAR="regexp:[0-9a-zA-Z\-_:~]" COMMAND_BAREWORD_CHAR="regexp:[<>*^]" - BRACED_SEPARATOR="[, ]+" + COMMA="," WILDCARD="*" KEYWORD_ELSE="else" KEYWORD_ELIF="elif" @@ -77,7 +77,8 @@ DoubleQuoted ::= DOUBLE_QUOTE (TEXT|ESCAPED_QUOTED_TEXT|INVALID_ESCAPED_QUOTED_T private Primary ::= Bareword | SingleQuoted | DoubleQuoted | VariableRef | WILDCARD | TILDE | ExceptionCapture | OutputCapture | Lambda | List | Map | Braced Variable ::= VARIABLE_CHAR+ VariableRef ::= DOLLAR_SIGN AT_SYMBOL? VARIABLE_CHAR+ Index* -Bareword ::= (VARIABLE_CHAR|BAREWORD_CHAR|<>)+ +Bareword ::= (VARIABLE_CHAR|BAREWORD_CHAR|COMMA|<>)+ +BracedBareword ::= (VARIABLE_CHAR|BAREWORD_CHAR|<>)+ CommandBareword ::= (VARIABLE_CHAR|BAREWORD_CHAR|COMMAND_BAREWORD_CHAR|WILDCARD|EQUALS)+ // Maps and lists @@ -114,8 +115,8 @@ OutputCapture ::= OPEN_PARAN Chunk CLOSE_PARAN Lambda ::= LambdaArguments? OPEN_BRACE Chunk CLOSE_BRACE LambdaArguments ::=OPEN_BRACKET Space? ((parameter | MapPair) Space?)* CLOSE_BRACKET parameter ::= AT_SYMBOL? Compound -Braced ::= OPEN_BRACE (Space | EOL)* Bareword (BracedSep* Bareword)* (Space | EOL)* CLOSE_BRACE -BracedSep ::= (Space | EOL)* <> (Space | EOL)* +Braced ::= OPEN_BRACE (Space | EOL)* BracedBareword (BracedSep* BracedBareword)* (Space | EOL)* CLOSE_BRACE +BracedSep ::= (Space | EOL)* COMMA (Space | EOL)* // Special Commands diff --git a/src/main/grammars/Elvish.flex b/src/main/grammars/Elvish.flex index 8428e2c..9b71092 100644 --- a/src/main/grammars/Elvish.flex +++ b/src/main/grammars/Elvish.flex @@ -113,6 +113,7 @@ WHITESPACE=({INLINE_WHITESPACE_CHAR}|{EOL})+ {KEYWORD_USE} { return ElvishTypes.KEYWORD_USE; } {VARIABLE_CHAR}+ { return ElvishTypes.VARIABLE_CHAR; } {BAREWORD_CHAR}+ { return ElvishTypes.BAREWORD_CHAR; } + {BRACED_BAREWORD_CHAR} { return ElvishTypes.COMMA; } {COMMAND_BAREWORD_CHAR}+ { return ElvishTypes.COMMAND_BAREWORD_CHAR; } } diff --git a/src/main/kotlin/com/github/sblundy/elvish/lang/parserUtils.kt b/src/main/kotlin/com/github/sblundy/elvish/lang/parserUtils.kt index 0afe1bd..fdd72c4 100644 --- a/src/main/kotlin/com/github/sblundy/elvish/lang/parserUtils.kt +++ b/src/main/kotlin/com/github/sblundy/elvish/lang/parserUtils.kt @@ -17,13 +17,6 @@ private val keywords = setOf( ElvishTypes.KEYWORD_TRY, ElvishTypes.KEYWORD_WHILE) -fun parseCommaAsSeparator(builder: PsiBuilder, level: Int): Boolean { - if (builder.tokenText != ",") return false - builder.remapCurrentToken(ElvishTypes.BRACED_SEPARATOR) - builder.advanceLexer() - return true -} - fun parseKeywordAsBareword(builder: PsiBuilder, level: Int): Boolean { if (!(builder.tokenType in keywords)) return false builder.remapCurrentToken(ElvishTypes.BAREWORD_CHAR) diff --git a/src/test/resources/com/github/sblundy/elvish/lang/bareword-with-comma.elv b/src/test/resources/com/github/sblundy/elvish/lang/bareword-with-comma.elv new file mode 100644 index 0000000..92bc75f --- /dev/null +++ b/src/test/resources/com/github/sblundy/elvish/lang/bareword-with-comma.elv @@ -0,0 +1 @@ +echo text, with comma \ No newline at end of file diff --git a/src/test/resources/com/github/sblundy/elvish/lang/bareword-with-comma.txt b/src/test/resources/com/github/sblundy/elvish/lang/bareword-with-comma.txt new file mode 100644 index 0000000..1046b5b --- /dev/null +++ b/src/test/resources/com/github/sblundy/elvish/lang/bareword-with-comma.txt @@ -0,0 +1,21 @@ +Elvish File + ElvishChunkImpl(CHUNK) + ElvishHeadImpl(HEAD) + ElvishCommandBarewordImpl(COMMAND_BAREWORD) + PsiElement(ElvishTokenType.VARIABLE_CHAR)('echo') + PsiElement(ElvishTokenType.INLINE_WHITESPACE)(' ') + ElvishArgumentImpl(ARGUMENT) + ElvishCompoundImpl(COMPOUND) + ElvishBarewordImpl(BAREWORD) + PsiElement(ElvishTokenType.VARIABLE_CHAR)('text') + PsiElement(ElvishTokenType.,)(',') + PsiElement(ElvishTokenType.INLINE_WHITESPACE)(' ') + ElvishArgumentImpl(ARGUMENT) + ElvishCompoundImpl(COMPOUND) + ElvishBarewordImpl(BAREWORD) + PsiElement(ElvishTokenType.VARIABLE_CHAR)('with') + PsiElement(ElvishTokenType.INLINE_WHITESPACE)(' ') + ElvishArgumentImpl(ARGUMENT) + ElvishCompoundImpl(COMPOUND) + ElvishBarewordImpl(BAREWORD) + PsiElement(ElvishTokenType.VARIABLE_CHAR)('comma') \ No newline at end of file diff --git a/src/test/resources/com/github/sblundy/elvish/lang/braced-list.tokens.txt b/src/test/resources/com/github/sblundy/elvish/lang/braced-list.tokens.txt index 06260a2..15b69dd 100644 --- a/src/test/resources/com/github/sblundy/elvish/lang/braced-list.tokens.txt +++ b/src/test/resources/com/github/sblundy/elvish/lang/braced-list.tokens.txt @@ -4,9 +4,9 @@ ElvishTokenType.VARIABLE_CHAR ('01') ElvishTokenType.BAREWORD_CHAR ('/') ElvishTokenType.{ ('{') ElvishTokenType.VARIABLE_CHAR ('03') -BAD_CHARACTER (',') +ElvishTokenType., (',') ElvishTokenType.VARIABLE_CHAR ('10') -BAD_CHARACTER (',') +ElvishTokenType., (',') ElvishTokenType.VARIABLE_CHAR ('17') ElvishTokenType.} ('}') ElvishTokenType.BAREWORD_CHAR ('/') diff --git a/src/test/resources/com/github/sblundy/elvish/lang/braced-list.txt b/src/test/resources/com/github/sblundy/elvish/lang/braced-list.txt index 5ab8286..de59bf4 100644 --- a/src/test/resources/com/github/sblundy/elvish/lang/braced-list.txt +++ b/src/test/resources/com/github/sblundy/elvish/lang/braced-list.txt @@ -11,15 +11,15 @@ Elvish File PsiElement(ElvishTokenType.BAREWORD_CHAR)('/') ElvishBracedImpl(BRACED) PsiElement(ElvishTokenType.{)('{') - ElvishBarewordImpl(BAREWORD) + ElvishBracedBarewordImpl(BRACED_BAREWORD) PsiElement(ElvishTokenType.VARIABLE_CHAR)('03') ElvishBracedSepImpl(BRACED_SEP) - PsiElement(ElvishTokenType.[, ]+)(',') - ElvishBarewordImpl(BAREWORD) + PsiElement(ElvishTokenType.,)(',') + ElvishBracedBarewordImpl(BRACED_BAREWORD) PsiElement(ElvishTokenType.VARIABLE_CHAR)('10') ElvishBracedSepImpl(BRACED_SEP) - PsiElement(ElvishTokenType.[, ]+)(',') - ElvishBarewordImpl(BAREWORD) + PsiElement(ElvishTokenType.,)(',') + ElvishBracedBarewordImpl(BRACED_BAREWORD) PsiElement(ElvishTokenType.VARIABLE_CHAR)('17') PsiElement(ElvishTokenType.})('}') ElvishBarewordImpl(BAREWORD)