[WIP]compiler based nim syntax highlight

src/moe.nim.cfg

@@ -2,3 +2,4 @@
 -d:strip
 -d:lto
 --gc:orc
+-d:nimpretty

src/moepkg/highlight.nim

@@ -2,6 +2,7 @@ import sequtils, os, strformat, parseutils
 import syntax/highlite
 import unicodeext, color
 from strutils import find
+import syntax/syntaxnim2
 
 type ColorSegment* = object
   firstRow*, firstColumn*, lastRow*, lastColumn*: int
@@ -145,23 +146,23 @@ iterator parseReservedWord(
     yield (buffer[pos ..< last], reservedWord.color)
     buffer = buffer[last ..^ 1]
 
-proc getEditorColorPairInNim(kind: TokenClass,
-                             isProcName: bool): EditorColorPair =
+proc getEditorColorPairInNim(kind: TokenClass ): EditorColorPair =
 
   case kind:
     of gtKeyword: EditorColorPair.keyword
     of gtBoolean: EditorColorPair.boolean
     of gtSpecialVar: EditorColorPair.specialVar
+    of gtOperator: EditorColorPair.functionName
     of gtBuiltin: EditorColorPair.builtin
     of gtStringLit: EditorColorPair.stringLit
     of gtDecNumber: EditorColorPair.decNumber
     of gtComment: EditorColorPair.comment
     of gtLongComment: EditorColorPair.longComment
     of gtPreprocessor: EditorColorPair.preprocessor
     of gtWhitespace, gtPunctuation: EditorColorPair.defaultChar
+    of gtFunctionName: EditorColorPair.functionName
     else:
-      if isProcName: EditorColorPair.functionName
-      else: EditorColorPair.defaultChar
+      EditorColorPair.defaultChar
 
 proc getEditorColorPair(kind: TokenClass,
                         language: SourceLanguage): EditorColorPair =
@@ -225,16 +226,19 @@ proc initHighlight*(buffer: string,
      language == SourceLanguage.langMarkDown:
     splitByNewline(buffer, EditorColorPair.defaultChar)
     return result
-
+  
   var token = GeneralTokenizer()
-  token.initGeneralTokenizer(buffer)
+
+  if language == SourceLanguage.langNim:
+    splitByNewline(buffer, EditorColorPair.defaultChar)
+    var tokens = parseTokens(buffer)
+    return result
+  else:
+    token.initGeneralTokenizer(buffer)
   var pad: string
   if buffer.parseWhile(pad, {' ', '\x09'..'\x0D'}) > 0:
     splitByNewline(pad, EditorColorPair.defaultChar)
 
-  # Only use in nim
-  var isProcName = false
-
   while true:
     try:
       token.getNextToken(language)
@@ -251,20 +255,7 @@ proc initHighlight*(buffer: string,
       currentColumn = 0
       continue
 
-    let color = if language == SourceLanguage.langNim:
-                  getEditorColorPairInNim(token.kind, isProcName)
-                else:
-                  getEditorColorPair(token.kind, language)
-
-    isProcName = if (language == SourceLanguage.langNim) and
-                   (buffer[first.. last] == "proc" or
-                   buffer[first.. last] == "macro" or
-                   buffer[first.. last] == "template" or
-                   buffer[first.. last] == "func"): true
-                  elif language == SourceLanguage.langNim and
-                       isProcName and
-                       token.kind == gtWhitespace: true
-                 else: false
+    let color = getEditorColorPair(token.kind, language)
 
     if token.kind == gtComment:
       for r in buffer[first..last].parseReservedWord(reservedWords, color):
@@ -324,3 +315,6 @@ proc detectLanguage*(filename: string): SourceLanguage =
     return SourceLanguage.langMarkDown
   else:
     return SourceLanguage.langNone
+
+when isMainModule:
+  echo initHighlight(readFile(currentSourcePath),@[],langNim).colorSegments

src/moepkg/syntax/highlite.nim

@@ -76,14 +76,17 @@ type
     gtOperator, gtPunctuation, gtComment, gtLongComment, gtRegularExpression,
     gtTagStart, gtTagEnd, gtKey, gtValue, gtRawData, gtAssembler,
     gtPreprocessor, gtDirective, gtCommand, gtRule, gtHyperlink, gtLabel,
-    gtReference, gtOther, gtBoolean, gtSpecialVar, gtBuiltin
+    gtReference, gtOther, gtBoolean, gtSpecialVar, gtBuiltin, gtFunctionName, gtTypeName
 
   GeneralTokenizer* = object of RootObj
     kind*: TokenClass
     start*, length*: int
     buf*: cstring
     pos*: int
     state*: TokenClass
+    # for nim
+    line*: uint16
+    col*: int16
 
   SourceLanguage* = enum
     langNone, langNim, langCpp, langCsharp, langC, langJava,

src/moepkg/syntax/mimport.nim

@@ -0,0 +1,10 @@
+import macros
+import os 
+
+const explicitSourcePath {.strdefine.} = os.parentDir(os.parentDir( os.getCurrentCompilerExe()))
+
+macro mImport*(path: static[string]): untyped =
+  result = newNimNode(nnkStmtList)
+  result.add(quote do:
+    import `explicitSourcePath` /  `path`
+  )

src/moepkg/syntax/pnode_parse.nim

@@ -0,0 +1,44 @@
+import ./mimport
+from os import nil
+mImport(os.joinPath( "compiler" , "parser.nim"))
+mImport(os.joinPath( "compiler" , "llstream.nim"))
+mImport(os.joinPath( "compiler" , "idents.nim"))
+mImport(os.joinPath( "compiler" , "options.nim"))
+mImport(os.joinPath( "compiler" , "pathutils.nim"))
+mImport(os.joinPath( "compiler" , "lineinfos.nim"))
+mImport(os.joinPath( "compiler" , "ast.nim"))
+export ast
+
+type ParseError = ref object of CatchableError
+const DevNullDir = when defined(windows):"c:\\" else: "/dev"
+const DevNullFile = when defined(windows):"nul" else: "null"
+
+proc parsePNodeStr*(str: string): PNode =
+  let cache: IdentCache = newIdentCache()
+  let config: ConfigRef = newConfigRef()
+  var pars: Parser
+
+  config.verbosity = 0
+  config.options.excl optHints
+  when defined(nimpretty):
+    config.outDir = toAbsoluteDir(DevNullDir)
+    config.outFile = RelativeFile(DevNullFile)
+  openParser(
+    p = pars,
+    filename = AbsoluteFile(currentSourcePath),
+    inputStream = llStreamOpen(str),
+    cache = cache,
+    config = config
+  )
+
+  pars.lex.errorHandler =
+    proc(conf: ConfigRef; info: TLineInfo; msg: TMsgKind; arg: string) =
+      if msg notin {hintLineTooLong}:
+        raise ParseError(msg: arg)
+
+  try:
+    result = parseAll(pars)
+    closeParser(pars)
+
+  except ParseError:
+    return nil

src/moepkg/syntax/syntaxnim2.nim

@@ -0,0 +1,224 @@
+
+import strutils
+import pnode_parse
+from algorithm import binarySearch
+import highlite
+include syntaxnim
+
+proc initNimToken(kind: TokenClass; start: int, buf: string, line:uint16, col:int16): GeneralTokenizer {.inline.} =
+  result = GeneralTokenizer(kind: kind, start: start, length: buf.len, buf: buf.cstring, line: line, col: col)
+
+proc initNimKeyword(start: int, buf: string, line: uint16, col: int16): GeneralTokenizer {.inline.} =
+  result = GeneralTokenizer(kind: TokenClass.gtKeyword, start: start, length: buf.len, buf: buf.cstring, line: line, col: col)
+
+proc initNimKeyword(n: PNode, buf: string): GeneralTokenizer =
+  let start = n.info.offsetA
+  let length = if n.info.offsetB == n.info.offsetA: buf.len else: n.info.offsetB - n.info.offsetA + 1
+  result = GeneralTokenizer(kind: TokenClass.gtKeyword, start: start, length: length, buf: buf.cstring, line: n.info.line, col: n.info.col)
+
+const CallNodes = {nkCall, nkInfix, nkPrefix, nkPostfix, nkCommand,
+             nkCallStrLit, nkHiddenCallConv}
+
+proc flatNode(par: PNode, outNodes: var seq[PNode]) =
+  # outNodes.add par
+  var d: PNode
+  for n in par:
+    d = n
+    case n.kind
+    of nkEmpty:
+      continue
+    of nkForStmt:
+      for s in n.sons[^2 .. ^1]:
+        flatNode(s, outNodes)
+      outNodes.add d
+      continue
+    of nkProcDef:
+      for s in n.sons[1 .. ^1]:
+        flatNode(s, outNodes)
+      d.sons.setLen(1)
+      outNodes.add d
+      continue
+    of {nkCall, nkCommand}:
+      d.sons.setLen(1)
+      outNodes.add d
+      for s in n.sons[1 .. ^1]:
+        flatNode(s, outNodes)
+      continue
+    else:
+      discard
+    outNodes.add n
+    flatNode(n, outNodes)
+
+proc `$`*(node: PNode): string =
+  ## Get the string of an identifier node.
+  case node.kind
+  of nkPostfix, nkInfix:
+    result = $node[0].ident.s
+  of nkIdent:
+    result = $node.ident.s
+  of nkPrefix:
+    result = $node.ident.s
+  of nkStrLit..nkTripleStrLit, nkCommentStmt, nkSym:
+    result = node.strVal
+  # of nnkOpenSymChoice, nnkClosedSymChoice:
+  #   result = $node[0]
+  of nkAccQuoted:
+    result = $node[0]
+  else:
+    discard
+
+proc basename*(a: PNode): PNode {.raises: [].} =
+  ## Pull an identifier from prefix/postfix expressions.
+  case a.kind
+  of nkIdent: result = a
+  of nkPostfix, nkPrefix:result = a[1]
+  of nkPragmaExpr: result = basename(a[0])
+  of nkExprColonExpr: result = a[0]
+  else:
+    discard
+
+proc parseTokens*(source: string): seq[GeneralTokenizer] =
+  let node = parsePNodeStr(source)
+  var outNodes = newSeq[PNode]()
+  flatNode(node, outNodes)
+
+  for n in outNodes:
+    case n.kind
+    of nkObjConstr:
+      result.add initNimToken(TokenClass.gtTypeName, n[0].info.offsetA, n[0].ident.s, n[0].info.line, n[0].info.col)
+    of nkEmpty, nkPar, nkBracket, nkAsgn, nkConstSection:
+      continue
+    of nkYieldStmt:
+      result.add initNimKeyword(n, "yield")
+    of nkVarTy:
+      result.add initNimKeyword(n, "var")
+    of nkPragma:
+      result.add initNimKeyword(n[0], $n[0].basename())
+    of nkProcDef:
+      result.add initNimKeyword(n, "proc")
+      result.add initNimToken(TokenClass.gtFunctionName, n[0].info.offsetA, $ n[0].basename(), n[0].info.line, n[0].info.col)
+    of nkIncludeStmt:
+      result.add initNimKeyword(n, "include")
+    of nkFromStmt:
+      result.add initNimKeyword(n[0].info.offsetA, "from", n[0].info.line, n[0].info.col)
+    of nkImportExceptStmt:
+      result.add initNimKeyword(n, "import")
+      let inStart = n[0].info.offsetB
+      result.add initNimKeyword(inStart, "except", n[0].info.line, n[0].info.col)
+    of nkExportStmt:
+      result.add initNimKeyword(n, "export")
+    of nkExportExceptStmt:
+      result.add initNimKeyword(n, "export")
+    of nkConstDef:
+      result.add initNimKeyword(n, "const")
+    of nkMacroDef:
+      result.add initNimKeyword(n, "macro")
+    of nkVarSection:
+      result.add initNimKeyword(n, "var")
+    of nkLetSection:
+      result.add initNimKeyword(n, "let")
+    of nkIteratorDef:
+      result.add initNimKeyword(n, "iterator")
+    of nkIfStmt:
+      result.add initNimKeyword(n, "if")
+    of nkReturnStmt:
+      result.add initNimKeyword(n, "return")
+    of nkBlockStmt:
+      result.add initNimKeyword(n, "block")
+    of nkExceptBranch:
+      result.add initNimKeyword(n, "except")
+    of nkWhileStmt:
+      result.add initNimKeyword(n, "while")
+    of nkTryStmt:
+      result.add initNimKeyword(n, "try")
+    of nkForStmt:
+      let inStart = n[^2].info.offsetA - 3
+      result.add initNimKeyword(n, "for")
+      result.add initNimKeyword(inStart, "in", n[^2].info.line, n[^2].info.col)
+    of nkCaseStmt:
+      result.add initNimKeyword(n, "case")
+    of nkContinueStmt:
+      result.add initNimKeyword(n, "continue")
+    of nkAsmStmt:
+      result.add initNimKeyword(n, "asm")
+    of nkDiscardStmt:
+      result.add initNimKeyword(n, "discard")
+    of nkBreakStmt:
+      result.add initNimKeyword(n, "break")
+    of nkElifBranch:
+      result.add initNimKeyword(n, "elif")
+    of nkElse:
+      result.add initNimKeyword(n, "else")
+    of nkOfBranch:
+      result.add initNimKeyword(n, "of")
+    of nkCast:
+      result.add initNimKeyword(n, "cast")
+    of nkMixinStmt:
+      result.add initNimKeyword(n, "mixin")
+    of nkTemplateDef:
+      result.add initNimKeyword(n, "template")
+    of nkImportStmt:
+      result.add initNimKeyword(n, "import")
+    of nkNilLit:
+      result.add initNimKeyword(n, "nil")
+    of nkCharLit:
+      let val = $n.intVal.char
+      result.add initNimToken(TokenClass.gtOctNumber, n.info.offsetA, val, n.info.line, n.info.col)
+    of nkIntLit .. nkUInt64Lit:
+      # intVal
+      let val = $n.getInt
+      result.add initNimToken(TokenClass.gtOctNumber, n.info.offsetA, val, n.info.line, n.info.col)
+    of nkFloatLit..nkFloat128Lit:
+      # floatVal*: BiggestFloat
+      result.add initNimToken(TokenClass.gtDecNumber, n.info.offsetA, $n.floatVal, n.info.line, n.info.col)
+    of nkStrLit .. nkTripleStrLit:
+      # strVal*: string
+      result.add initNimToken(TokenClass.gtStringLit, n.info.offsetA, n.strVal, n.info.line, n.info.col)
+    of nkTypeSection:
+      discard
+    of nkGenericParams:
+      discard
+    of nkDotExpr:
+      discard
+    of nkTypeClassTy:
+      discard
+    of nkBracketExpr:
+      discard
+    of nkFormalParams:
+      discard
+    of nkProcTy:
+      result.add initNimKeyword(n, "proc")
+    of nkTypeDef:
+      result.add initNimKeyword(n, "type")
+    of nkObjectTy:
+      result.add initNimKeyword(n, "object")
+    of nkStmtList:
+      discard
+    of nkRecList:
+      discard
+    of nkIdentDefs:
+      discard
+    of nkExprColonExpr:
+      discard
+    of nkTableConstr:
+      discard
+    of nkIdentKinds - {nkAccQuoted}:
+      # ident*: PIdent
+      result.add initNimToken(nimGetKeyword(n.ident.s), n.info.offsetA, n.ident.s, n.info.line, n.info.col)
+    of nkAccQuoted:
+      discard
+    of nkCallKinds - {nkInfix, nkPostfix, nkDotExpr}:
+      let id = $n[0]
+      let tok = initNimToken(TokenClass.gtFunctionName, n[0].info.offsetA, id, n[0].info.line, n[0].info.col)
+      if tok.buf.len > 0:
+        result.add tok
+    of nkInfix:
+      if $n[0] == "as":
+        result.add initNimKeyword(n[0].info.offsetA, "as", n[0].info.line, n[0].info.col)
+      else:
+        result.add initNimToken(TokenClass.gtOperator, n[0].info.offsetA, $n, n[0].info.line, n[0].info.col)
+    of nkPostfix:
+      result.add initNimToken(TokenClass.gtSpecialVar, n[0].info.offsetA, $n, n[0].info.line, n[0].info.col)
+    else:
+      result.add initNimToken(TokenClass.gtIdentifier, n.info.offsetA, $n, n.info.line, n.info.col)
+