Initial commit

.editorconfig

@@ -0,0 +1,6 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true

.github/workflows/build.yaml

@@ -0,0 +1,41 @@
+name: Build & Push
+
+on:
+  push:
+    tags:
+      - "*"
+  pull_request:
+    types: [opened, synchronize]
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - uses: actions/setup-go@v5
+      with:
+        go-version-file: 'go.mod'
+
+    - run: go test -v ./...
+
+    - run: go build -ldflags '-s' ./...
+      env:
+        CGO_ENABLED: 0
+
+    - name: Log in to registry
+      if: github.event_name == 'push'
+      run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin
+
+    - name: Build image
+      if: github.event_name == 'push'
+      run: docker build . -t ghcr.io/tipee-sa/gcs-index:$(echo "${{ github.ref }}" | cut -d/ -f3-)
+
+    - name: Push image
+      if: github.event_name == 'push'
+      run: docker push ghcr.io/tipee-sa/gcs-index:$(echo "${{ github.ref }}" | cut -d/ -f3-)

.gitignore

@@ -0,0 +1,2 @@
+# Build output
+/gcs-index

Dockerfile

@@ -0,0 +1,7 @@
+FROM gcr.io/distroless/static:nonroot
+WORKDIR /
+
+COPY gcs-index /gcs-index
+USER nonroot:nonroot
+
+ENTRYPOINT ["/gcs-index"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 tipee
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

go.mod

@@ -0,0 +1,43 @@
+module github.com/tipee-sa/gcs-index
+
+go 1.22.5
+
+require (
+	cloud.google.com/go/storage v1.43.0
+	google.golang.org/api v0.188.0
+)
+
+require (
+	cloud.google.com/go v0.115.0 // indirect
+	cloud.google.com/go/auth v0.7.1 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect
+	cloud.google.com/go/compute/metadata v0.5.0 // indirect
+	cloud.google.com/go/iam v1.1.11 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/s2a-go v0.1.7 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
+	github.com/googleapis/gax-go/v2 v2.12.5 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
+	go.opentelemetry.io/otel v1.28.0 // indirect
+	go.opentelemetry.io/otel/metric v1.28.0 // indirect
+	go.opentelemetry.io/otel/trace v1.28.0 // indirect
+	golang.org/x/crypto v0.25.0 // indirect
+	golang.org/x/net v0.27.0 // indirect
+	golang.org/x/oauth2 v0.21.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
+	golang.org/x/sys v0.22.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	google.golang.org/genproto v0.0.0-20240711142825-46eb208f015d // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240711142825-46eb208f015d // indirect
+	google.golang.org/grpc v1.65.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
+)

main.go

@@ -0,0 +1,269 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	_ "embed"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"os"
+	"os/signal"
+	"slices"
+	"strings"
+	"syscall"
+	"time"
+
+	"cloud.google.com/go/storage"
+	"google.golang.org/api/iterator"
+)
+
+type MountPoint struct {
+	Path   string
+	Bucket string
+	Prefix string
+}
+
+var client *storage.Client
+var mountPoints []MountPoint
+
+//go:embed page.html
+var pageHtml []byte
+
+func main() {
+	prepareMountPoints()
+	slog.Info("initializing", "mountPoints", mountPoints)
+
+	var err error
+	client, err = storage.NewClient(context.Background(), storage.WithJSONReads())
+	if err != nil {
+		slog.Error("failed to create storage client", "err", err)
+		os.Exit(3)
+	}
+
+	server := &http.Server{Addr: fmt.Sprintf(":%s", envOr("PORT", "8080"))}
+	http.HandleFunc("/", handle)
+
+	go func() {
+		slog.Info("server started", "addr", server.Addr)
+		if err := server.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) {
+			slog.Error("server error", "err", err)
+			os.Exit(4)
+		}
+		slog.Warn("server stopped")
+	}()
+
+	// Wait for a signal to stop the server
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
+
+	<-sigChan
+	slog.Warn("shutting down server")
+
+	shutdownCtx, shutdownRelease := context.WithTimeout(context.Background(), 10*time.Second)
+	defer shutdownRelease()
+
+	if err := server.Shutdown(shutdownCtx); err != nil {
+		slog.Error("shutdown error", "err", err)
+		os.Exit(5)
+	}
+	slog.Info("shutdown completed")
+}
+
+func envOr(key, defaultValue string) string {
+	if value, ok := os.LookupEnv(key); ok {
+		return value
+	} else {
+		return defaultValue
+	}
+}
+
+func prepareMountPoints() {
+	if len(os.Args) < 2 {
+		fmt.Fprintf(os.Stderr, "Usage: %s path:bucket:prefix [path:bucket:prefix ...]\n", os.Args[0])
+		os.Exit(1)
+	}
+
+	for _, arg := range os.Args[1:] {
+		mountPointParts := strings.SplitN(arg, ":", 3)
+		if len(mountPointParts) != 3 {
+			slog.Error("invalid mount point", "arg", arg, "reason", "expected 'path:bucket:prefix'")
+			os.Exit(2)
+		}
+
+		// Normalize the path
+		if !strings.HasPrefix(mountPointParts[0], "/") {
+			mountPointParts[0] = "/" + mountPointParts[0]
+		}
+		if !strings.HasSuffix(mountPointParts[0], "/") {
+			mountPointParts[0] += "/"
+		}
+
+		mountPoints = append(mountPoints, MountPoint{
+			Path:   mountPointParts[0],
+			Bucket: mountPointParts[1],
+			Prefix: mountPointParts[2],
+		})
+	}
+
+	slices.SortFunc(mountPoints, func(a, b MountPoint) int {
+		if len(a.Path) != len(b.Path) {
+			return len(b.Path) - len(a.Path)
+		} else {
+			return strings.Compare(a.Path, b.Path)
+		}
+	})
+}
+
+func handle(w http.ResponseWriter, r *http.Request) {
+	slog.Info("request",
+		"path", r.URL.Path,
+		"method", r.Method,
+		"remote", r.RemoteAddr,
+		"forwarded-for", r.Header.Get("X-Forwarded-For"),
+		"user-agent", r.UserAgent())
+
+	if r.Method != http.MethodGet && r.Method != http.MethodHead {
+		slog.Warn("method not allowed", "method", r.Method)
+		w.WriteHeader(http.StatusMethodNotAllowed)
+		return
+	}
+
+	if r.URL.Path == "/favicon.ico" {
+		w.WriteHeader(http.StatusNotFound)
+		return
+	}
+
+	if strings.HasSuffix(r.URL.Path, "/") {
+		handleIndex(w, r)
+	} else {
+		handleObject(w, r)
+	}
+}
+
+func handleIndex(w http.ResponseWriter, r *http.Request) {
+	if r.Method == http.MethodHead {
+		// Directory index always returns 200 OK.
+		return
+	}
+
+	var links []string
+	if r.URL.Path != "/" {
+		links = append(links, "../")
+	}
+
+	links = append(links, linksFromMountPoints(r.URL.Path)...)
+	links = append(links, linksFromStorage(r.Context(), r.URL.Path)...)
+
+	links = slices.Compact(links)
+	slices.Sort(links)
+
+	var output bytes.Buffer
+	for _, link := range links {
+		_, _ = output.WriteString(fmt.Sprintf("<a href=\"%s\">%s</a><br>\n", link, link))
+	}
+
+	w.Write(pageHtml)
+	w.Write(output.Bytes())
+}
+
+func linksFromMountPoints(path string) (links []string) {
+	for _, mountPoint := range mountPoints {
+		if mountPoint.Path != path && strings.HasPrefix(mountPoint.Path, path) {
+			links = append(links, strings.SplitAfterN(strings.TrimPrefix(mountPoint.Path, path), "/", 2)[0])
+		}
+	}
+	return
+}
+
+func linksFromStorage(ctx context.Context, path string) (links []string) {
+	var mountPoint = findMountPoint(path)
+	if mountPoint == nil {
+		return
+	}
+
+	bucket := client.Bucket(mountPoint.Bucket)
+	query := &storage.Query{
+		Prefix:    mountPoint.Prefix + strings.TrimPrefix(path, mountPoint.Path),
+		Delimiter: "/",
+	}
+
+	slog.Debug("listing objects", "bucket", mountPoint.Bucket, "query", query)
+
+	objects := bucket.Objects(ctx, query)
+	for {
+		attrs, err := objects.Next()
+		if err == iterator.Done {
+			break
+		} else if err != nil {
+			slog.Error("failed to list objects", "err", err)
+			break
+		}
+
+		if attrs.Name != "" {
+			if attrs.Name != query.Prefix {
+				links = append(links, strings.TrimPrefix(attrs.Name, query.Prefix))
+			}
+		} else if attrs.Prefix != "" {
+			links = append(links, strings.TrimPrefix(attrs.Prefix, query.Prefix))
+		} else {
+			slog.Warn("unexpected object", "attrs", attrs)
+		}
+	}
+
+	return
+}
+
+func findMountPoint(path string) *MountPoint {
+	for i := len(mountPoints) - 1; i >= 0; i-- {
+		if strings.HasPrefix(path, mountPoints[i].Path) {
+			return &mountPoints[i]
+		}
+	}
+	return nil
+}
+
+func handleObject(w http.ResponseWriter, r *http.Request) {
+	var mountPoint = findMountPoint(r.URL.Path)
+	if mountPoint == nil {
+		w.WriteHeader(http.StatusNotFound)
+		return
+	}
+
+	bucket := client.Bucket(mountPoint.Bucket)
+	obj := bucket.Object(mountPoint.Prefix + strings.TrimPrefix(r.URL.Path, mountPoint.Path))
+
+	reader, err := obj.NewReader(r.Context())
+	if err != nil {
+		slog.Error("failed to read object",
+			"bucket", obj.BucketName(),
+			"object", obj.ObjectName(),
+			"err", err)
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	defer reader.Close()
+
+	setHeaderIfNotEmpty(w, "Content-Length", fmt.Sprintf("%d", reader.Attrs.Size))
+	setHeaderIfNotEmpty(w, "Content-Type", reader.Attrs.ContentType)
+	setHeaderIfNotEmpty(w, "Content-Encoding", reader.Attrs.ContentEncoding)
+	setHeaderIfNotEmpty(w, "Last-Modified", reader.Attrs.LastModified.Format(http.TimeFormat))
+	setHeaderIfNotEmpty(w, "Cache-Control", reader.Attrs.CacheControl)
+
+	if r.Method == http.MethodHead {
+		return
+	}
+
+	if _, err := io.Copy(w, reader); err != nil {
+		slog.Error("failed to write object", "err", err)
+		w.WriteHeader(http.StatusInternalServerError)
+	}
+}
+
+func setHeaderIfNotEmpty(w http.ResponseWriter, key, value string) {
+	if value != "" {
+		w.Header().Set(key, value)
+	}
+}