Offsets and Locations

Some work to improve the handling of text locations by the
`TextDocument`. This makes sure that locations are allways at a
consistent line colum offset. Previously the first line was 0-based and
the remaining lines 1-based.

Still not totally sure if we want the position objects to be 0-based
like Firethorn or 1-based ready for output to the screen.

src/Feersum.CompilerServices/Syntax/Lex.fs

@@ -286,6 +286,8 @@ let public tokenise input =
 
     let mutable state = Start
     let mutable lexeme = StringBuilder()
+    // TODO: we should probably stop tracking offset in the tokens and instead
+    ///      keep track as we eat their lexemes in the parser.
     let mutable offset = 0
 
     seq {

src/Feersum.CompilerServices/Syntax/Parse.fs

@@ -86,7 +86,11 @@ module private ParserState =
         let pos =
             match List.tryHead state.Tokens with
             | Some token -> token.Offset
-            | _ -> 0
+            | _ ->
+                // FIXME: If we have no tokens let the diagnostic should
+                // probably point to the beginning of the file rather than the
+                // end of it.
+                0
             |> TextDocument.offsetToPoint state.Document
             |> TextLocation.Point
 

src/Feersum.CompilerServices/Text.fs

@@ -57,19 +57,22 @@ module public TextDocument =
         { Path = path
           LineStarts = lineStarts body }
 
+    /// Turn a character offset into a document into a hunan line column value.
+    /// That is a 0-baesd character offset into the file ebcomes a 1-based pair
+    /// of line and column.
     let private offsetToLineCol lines offset =
-        // TODO: This method returns 1-based line indexes, and 0-based column
-        //       indexes. Is that what we _want_?
-        match List.tryFindIndex (fun x -> x > offset) lines with
-        | Some(0) -> (1, offset)
-        | Some(idx) -> (idx, offset - lines[idx - 1])
-        | None ->
-            let linebreakCount = List.length lines
+        let (lineIdx, colIdx) =
+            match List.tryFindIndexBack (fun x -> x < offset) lines with
+            | Some(idx) ->
+                // The `idx` line break occurs before this line. If tis is the
+                // break at index 0, we're on the line at index 1.
+                (idx + 1, offset - lines[idx] - 1)
+            | None ->
+                // no linebreak occurs before the offset, so we must be on the
+                // first line.
+                (0, offset)
 
-            if linebreakCount = 0 then
-                (1, offset)
-            else
-                (linebreakCount + 1, offset - (List.last lines))
+        (lineIdx + 1, colIdx + 1)
 
     let public offsetToPoint document offset =
         let (line, col) = offsetToLineCol document.LineStarts offset

test/Feersum.Tests/Feersum.Tests.fsproj

@@ -8,6 +8,7 @@
     <ProjectReference Include="..\..\src\Feersum.Core\Feersum.Core.scmproj" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="TextTests.fs" />
     <Compile Include="DiagnosticsTests.fs" />
     <Compile Include="OptionsTests.fs" />
     <Compile Include="SyntaxUtils.fs" />
@@ -32,4 +33,4 @@
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
   </ItemGroup>
-</Project>
+</Project>

test/Feersum.Tests/SyntaxTestsNew.fs

@@ -275,6 +275,6 @@ let ``syntax shim test`` () =
         |> Result.unwrap
 
     Assert.Equal(1L, tree.Location.Start.Line)
-    Assert.Equal(0L, tree.Location.Start.Col)
+    Assert.Equal(1L, tree.Location.Start.Col)
     Assert.Equal(1L, tree.Location.End.Line)
-    Assert.Equal(7L, tree.Location.End.Col)
+    Assert.Equal(8L, tree.Location.End.Col)

test/Feersum.Tests/TextTests.fs

@@ -0,0 +1,32 @@
+module TextTests
+
+open Feersum.CompilerServices.Text
+open Xunit
+
+[<Fact>]
+let ``text document for empty string`` () =
+    let doc = TextDocument.fromParts "hello.scm" ""
+
+    Assert.Equal(TextPoint.FromParts("hello.scm", 1, 1), TextDocument.offsetToPoint doc 0)
+
+[<Fact>]
+let ``text document points`` () =
+    let body =
+        @"(define hello 123)
+(let ((id (lammbda (x) x))) (id (hello 123)))
+
+;; Blank lines and comments
+
+990
+"
+
+    let doc = TextDocument.fromParts "test.scm" body
+
+    let c (line, col) off =
+        Assert.Equal(TextPoint.FromParts("test.scm", line, col), TextDocument.offsetToPoint doc off)
+
+    c (1, 1) 0
+    c (6, 1) (body.LastIndexOf("990"))
+    c (7, 1) (body.LastIndexOf("0") + 2)
+    c (2, 2) (body.IndexOf("let"))
+    c (4, 10) (body.IndexOf("lines and comments"))