Fix a lot of bugs

Author: afreeorange

NOTES.md

@@ -17,6 +17,8 @@ CGO_ENABLED=1 go build --tags "fts5" -o "dist/bock-$(uname)-$(uname -m)" .
 ### TODO
 
 * [x] FIX THE NAVIGATION FFS
+* [ ] FIX THE PATH GENERATION PROBLEM FFS
+* [ ] FIX FOLDER generation
 * [x] Breadcrumbs > Revision
 * [ ] Table of Contents
 * [ ] Recent Changes (Global)
@@ -33,6 +35,9 @@ CGO_ENABLED=1 go build --tags "fts5" -o "dist/bock-$(uname)-$(uname -m)" .
 * [x] 404 Page
 * [ ] Template argument
 * [ ] Revisions argument
+* [ ] Better, less buggy tree
+* [ ] Gist of recursive tree generation!
+- [ ] Fix issue with apostrophes 🤦‍♀️
 * [ ] Use `context` in lieu of `config` struct? What are the dis/advantages?
 * [ ] [Markdown highlight in Raw view](https://www.zupzup.org/go-markdown-syntax-highlight-chroma/)
 * [ ] [Filtering logs with filename is very slow in `go-git`](https://github.com/go-git/go-git/issues/137)```
@@ -328,3 +333,19 @@ func main() {
 	fmt.Println(string(s))
 }
 ```
+
+### Pongo2 Custom Filter Sample
+
+```golang
+var _ = pongo2.RegisterFilter("round", func(in, param *pongo2.Value) (out *pongo2.Value, err *pongo2.Error) {
+	var rounded *pongo2.Value
+
+	if s, err := strconv.ParseFloat(in.String(), 32); err == nil {
+		rounded = pongo2.AsSafeValue(math.Round(s))
+	} else {
+		return pongo2.AsSafeValue("!_COULD_NOT_ROUND_VALUE"), &pongo2.Error{OrigError: err}
+	}
+
+	return rounded, nil
+})
+```

README.md

@@ -17,6 +17,10 @@ git clone https://github.com/afreeorange/bockgo.git
 go run --tags "fts5" . -a /path/to/repo -o /path/to/output -r -j
 ```
 
+## Terminology et al
+
+An "Entity" is either an "Article" (a Markdown file somewhere in your article repository) or a "Folder" (which is exactly what you think it is). A "Revision" is a `git` commit that modifies an Article.
+
 * You can organize your articles into folders.
 * You cannot use `raw`, `revisions`, `random`, and `archive` as folder names.
 * You can place static assets in `__assets` in your article repository. You can reference all assets in there in your Markdown files prefixed with `/assets` (e.g. `__assets/some-file.jpg` → `/assets/some-file.jpg`).
@@ -27,7 +31,7 @@ The command will generate the following:
 * Each article's revision rendered as Raw Source, HTML, and JSON
 * A listing of all revisions for each article (if applicable)
 * Each folder's structure in HTML and JSON ([example](https://wiki.nikhil.io/Food/))
-* An archive page that lets you search your articles
+* An archive page that lets you search your articles thanks to SQLite and [SQL.js](https://github.com/sql-js/sql.js/)
 * A Homepage (if it doesn't exist as `Home.md`) at `/Home`
 * A page that redirects to some random article at `/random`
 * An index page that redirects to `/Home`

VERSION

@@ -1 +1 @@
-4.0.0-beta.18
+4.0.0-beta.19

constants.go

@@ -31,20 +31,16 @@ const (
 )
 
 // Things to ignore when walking the article repository. NOTE: In Golang, only
-// primitive types (like `int`, `string`, etc) can be constants.
-var IGNORED_FOLDERS_REGEX = regexp.MustCompile(strings.Join([]string{
+// primitive types (like `int`, `string`, etc) can be constants. NOTE: Folders
+// beginning with `.` are automatically excluded in the function that uses this
+// pattern.
+var IGNORED_ENTITIES_REGEX = regexp.MustCompile(strings.Join([]string{
 	"__assets",
-	"_assets",
 	"css",
 	"img",
 	"js",
+	"Home.md", // This is dealt with separately
 	"node_modules",
-	// NOTE: Folders beginning with `.` are automatically excluded
-}, "|"))
-
-// Here since we process the Homepage separately
-var IGNORED_FILES_REGEX = regexp.MustCompile(strings.Join([]string{
-	"Lol.md",
 }, "|"))
 
 // We use Goldmark as the Markdown converter. Configure it here.
@@ -72,15 +68,3 @@ var markdown = goldmark.New(
 var templatesContent embed.FS
 var pongoLoader = pongo2.NewFSLoader(templatesContent)
 var templateSet = pongo2.NewSet("template", pongoLoader)
-
-// var _ = pongo2.RegisterFilter("round", func(in, param *pongo2.Value) (out *pongo2.Value, err *pongo2.Error) {
-// 	var rounded *pongo2.Value
-
-// 	if s, err := strconv.ParseFloat(in.String(), 32); err == nil {
-// 		rounded = pongo2.AsSafeValue(math.Round(s))
-// 	} else {
-// 		return pongo2.AsSafeValue("!_COULD_NOT_ROUND_VALUE"), &pongo2.Error{OrigError: err}
-// 	}
-
-// 	return rounded, nil
-// })

helpers.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io/fs"
 	"os"
+	"path"
 	"path/filepath"
 	"sort"
 	"strings"
@@ -153,7 +154,7 @@ func getEntityInfo(config *BockConfig, info fs.FileInfo, path string) *Entity {
 		IsFolder:     info.IsDir(),
 		Modified:     info.ModTime(),
 		Name:         info.Name(),
-		Path:         path,
+		path:         path,
 		RelativePath: makeRelativePath(path, config.articleRoot),
 		SizeInBytes:  info.Size(),
 		Title:        removeExtensionFrom(info.Name()),
@@ -176,7 +177,7 @@ func findChildWithName(children *[]Entity, name string) int {
 	return -1
 }
 
-func makeEntityTree(listOfArticles *[]Entity) []Entity {
+func makeEntityTree(config *BockConfig) []Entity {
 	tree := []Entity{}
 
 	// Bootstrap: create adn append the root entity (a folder)
@@ -189,12 +190,12 @@ func makeEntityTree(listOfArticles *[]Entity) []Entity {
 		SizeInBytes:  0,
 		Title:        "Root",
 		URI:          "/ROOT",
-		Path:         ".",
+		path:         ".",
 	})
 
 	// These loops took me an embarrassingly LONG while to write :/
 
-	for _, article := range *listOfArticles {
+	for _, article := range *config.listOfArticles {
 		pathFragments := strings.Split(article.RelativePath, "/")
 
 		// Use this to build the URI. Reset with each iteration.
@@ -227,8 +228,8 @@ func makeEntityTree(listOfArticles *[]Entity) []Entity {
 						RelativePath: strings.TrimPrefix(uri, "/"),
 						SizeInBytes:  0,
 						Title:        fragment,
-						URI:          uri,
-						Path:         article.Path,
+						URI:          makeURI(uri, config.articleRoot),
+						path:         article.path,
 					})
 				}
 			}
@@ -243,26 +244,80 @@ func makeEntityTree(listOfArticles *[]Entity) []Entity {
 	return tree
 }
 
-func makeListOfArticles(config *BockConfig) ([]Entity, error) {
-	list := []Entity{}
+func uniqueStringsInList(list []string) []string {
+	var uniqueList []string
+	tempMap := make(map[string]bool)
 
-	walkFunction := func(path string, entityInfo os.FileInfo, err error) error {
-		relativePath := makeRelativePath(path, config.articleRoot)
+	for _, e := range list {
+		if _, ok := tempMap[e]; !ok {
+			tempMap[e] = true
+			uniqueList = append(uniqueList, e)
+		}
+	}
+
+	return uniqueList
+}
+
+func makeListOfEntities(config *BockConfig) (
+	listOfArticles []Entity,
+	listOfFolders []string,
+	err error,
+) {
+	walkFunction := func(entityPath string, entityInfo os.FileInfo, walkErr error) error {
+		relativePath := makeRelativePath(entityPath, config.articleRoot)
 
 		isValidArticle := (!entityInfo.IsDir() &&
-			!IGNORED_FILES_REGEX.MatchString(path) &&
+			!IGNORED_ENTITIES_REGEX.MatchString(entityPath) &&
 			!strings.HasPrefix(relativePath, ".") &&
-			filepath.Ext(path) == ".md")
+			filepath.Ext(entityPath) == ".md")
 
 		if isValidArticle {
-			list = append(list, *getEntityInfo(config, entityInfo, path))
+			listOfArticles = append(
+				listOfArticles,
+				*getEntityInfo(config, entityInfo, entityPath),
+			)
+
+			folderPath := path.Dir(entityPath)
+
+			/*
+			   For example,
+
+			   /article/root/sso-react
+			   /article/root/sso-react/build/refresh
+			   /article/root/sso-react/public/refresh
+			   /article/root/sso-react/src/i18n
+
+			   should be
+
+			   /article/root/sso-react
+			   /article/root/sso-react/build
+			   /article/root/sso-react/build/refresh
+			   /article/root/sso-react/public
+			   /article/root/sso-react/public/refresh
+			   /article/root/sso-react/src
+			   /article/root/sso-react/src/i18n
+
+			   That's what we're doing here.
+			*/
+			folderSplits := strings.Split(makeRelativePath(folderPath, config.articleRoot), "/")
+			p := ""
+			if len(folderSplits) > 1 {
+				for _, s := range folderSplits {
+					p += "/" + s
+					listOfFolders = append(listOfFolders, config.articleRoot+p)
+				}
+			} else {
+				listOfFolders = append(listOfFolders, folderPath)
+			}
 		}
 
 		return nil
 	}
 
-	// Make a list of articles and return it
-	// TODO: Sort?
-	err := filepath.Walk(config.articleRoot, walkFunction)
-	return list, err
+	// It strikes me that error-handling in Go is a bit strange... looks like
+	// things can just fall through.
+	err = filepath.Walk(config.articleRoot, walkFunction)
+	listOfFolders = uniqueStringsInList(listOfFolders)
+
+	return listOfArticles, listOfFolders, err
 }

main.go

@@ -17,23 +17,23 @@ import (
 )
 
 func main() {
-	var versionInfo bool
 	var articleRoot string
-	var outputFolder string
 	var generateJSON bool
 	var generateRaw bool
-	var useOnDiskFS bool
+	var generateRevisions bool
 	var minifyOutput bool
-	var createRevisions bool
+	var outputFolder string
+	var useOnDiskFS bool
+	var versionInfo bool
 
-	flag.BoolVar(&versionInfo, "v", false, "Version info")
 	flag.StringVar(&articleRoot, "a", "", "Article root")
-	flag.StringVar(&outputFolder, "o", "", "Output folder")
 	flag.BoolVar(&generateJSON, "j", false, "Create JSON source files")
 	flag.BoolVar(&generateRaw, "r", false, "Create Raw markdown source files")
-	flag.BoolVar(&useOnDiskFS, "d", false, "Use on-disk filesystem to clone article repository (slower; cloned to memory by default)")
+	flag.BoolVar(&generateRevisions, "R", true, "Create article revisions based on git history (default: true)")
 	flag.BoolVar(&minifyOutput, "m", false, "Minify all output (HTML, JS, CSS)")
-	flag.BoolVar(&createRevisions, "R", true, "Create article revisions based on git history (default: true)")
+	flag.StringVar(&outputFolder, "o", "", "Output folder")
+	flag.BoolVar(&useOnDiskFS, "d", false, "Use on-disk filesystem to clone article repository (slower; cloned to memory by default)")
+	flag.BoolVar(&versionInfo, "v", false, "Version info")
 
 	flag.Parse()
 
@@ -100,29 +100,32 @@ func main() {
 		database:       nil,
 		outputFolder:   outputFolder,
 		meta: Meta{
-			Architecture:   runtime.GOARCH,
-			ArticleCount:   0,
-			BuildDate:      time.Now().UTC(),
-			CPUCount:       runtime.NumCPU(),
-			GenerateJSON:   generateJSON,
-			GenerateRaw:    generateRaw,
-			GenerationTime: 0,
-			MemoryInGB:     int(v.Total / (1024 * 1024 * 1024)),
-			Platform:       runtime.GOOS,
-			RevisionCount:  0,
+			Architecture:      runtime.GOARCH,
+			ArticleCount:      0,
+			BuildDate:         time.Now().UTC(),
+			CPUCount:          runtime.NumCPU(),
+			GenerateJSON:      generateJSON,
+			GenerateRaw:       generateRaw,
+			GenerateRevisions: generateRevisions,
+			GenerationTime:    0,
+			MemoryInGB:        int(v.Total / (1024 * 1024 * 1024)),
+			Platform:          runtime.GOOS,
+			RevisionCount:     0,
 		},
 		started:        time.Now(),
 		repository:     repository,
 		workTreeStatus: &status,
 	}
 
-	// Make a flat list of absolute article paths
-	listOfArticles, _ := makeListOfArticles(&config)
+	// Make a flat list of absolute article paths. Use these to build the entity
+	// tree. We do this to prevent unnecessary and empty folders from being
+	// created.
+	listOfArticles, _, _ := makeListOfEntities(&config)
 	config.listOfArticles = &listOfArticles
-
 	fmt.Println("Found", len(*config.listOfArticles), "articles")
 
-	entityTree := makeEntityTree(config.listOfArticles)
+	// Make a tree of entities: articles and folders
+	entityTree := makeEntityTree(&config)
 	config.entityTree = &entityTree
 
 	// Database setup

template/article.njk

@@ -17,5 +17,5 @@
 {% endblock footerElements %}
 
 {% block statistics %}
-  There are {{ meta.ArticleCount }} articles and {{ meta.RevisionCount }} revisions in this wiki. On {{ meta.BuildDate | date:"Monday, 2 January 2006 at 15:04 MST" }}, it took {{ meta.GenerationTimeRounded }} to generate it on a {{ meta.CPUCount }}-core {{ meta.Platform }}/{{ meta.Architecture }} system with {{ meta.MemoryInGB }}GiB RAM
+  There are {{ meta.ArticleCount }} articles and {{ meta.RevisionCount }} revisions in this wiki. It took {{ meta.GenerationTimeRounded }} to generate it on a {{ meta.CPUCount }}-core {{ meta.Platform }}/{{ meta.Architecture }} system with {{ meta.MemoryInGB }}GiB RAM on {{ meta.BuildDate | date:"Monday, 2 January 2006 at 15:04 MST" }}
 {% endblock statistics %}

types.go

@@ -64,14 +64,16 @@ type Entity struct {
 	Title        string    `json:"title"`
 	URI          string    `json:"uri"`
 
-	Path string `json:"path"`
+	// You do NOT want to make this public!
+	path string
 }
 
 type Meta struct {
 	Architecture          string        `json:"architecture"`
 	ArticleCount          int           `json:"articleCount"`
 	BuildDate             time.Time     `json:"buildTime"`
 	CPUCount              int           `json:"cpuCount"`
+	GenerateRevisions     bool          `json:"generateRevisions"`
 	GenerateJSON          bool          `json:"generateJSON"`
 	GenerateRaw           bool          `json:"generateRaw"`
 	GenerationTime        time.Duration `json:"generationTime"`