Merge branch 'main' into gp_metadata

cmd/shared.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"log/slog"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -38,12 +39,14 @@ type SharedFlags struct {
 	JSONLog           bool          // Enable JSON structured log
 	DebugCounters     bool          // Enable CSV action counters per file
 
-	Immich          immich.ImmichInterface // Immich client
-	Log             *slog.Logger           // Logger
-	Jnl             *fileevent.Recorder    // Program's logger
-	LogFile         string                 // Log file name
-	LogWriterCloser io.WriteCloser         // the log writer
-	Banner          ui.Banner
+	Immich             immich.ImmichInterface // Immich client
+	Log                *slog.Logger           // Logger
+	Jnl                *fileevent.Recorder    // Program's logger
+	LogFile            string                 // Log file name
+	LogWriterCloser    io.WriteCloser         // the log writer
+	APITraceWriter     io.WriteCloser         // API tracer
+	APITraceWriterName string
+	Banner             ui.Banner
 }
 
 func (app *SharedFlags) InitSharedFlags() {
@@ -68,7 +71,7 @@ func (app *SharedFlags) SetFlags(fs *flag.FlagSet) {
 	fs.StringVar(&app.LogLevel, "log-level", app.LogLevel, "Log level (DEBUG|INFO|WARN|ERROR), default INFO")
 	fs.StringVar(&app.LogFile, "log-file", app.LogFile, "Write log messages into the file")
 	fs.BoolFunc("log-json", "Output line-delimited JSON file, default FALSE", myflag.BoolFlagFn(&app.JSONLog, app.JSONLog))
-	fs.BoolFunc("api-trace", "enable api call traces", myflag.BoolFlagFn(&app.APITrace, app.APITrace))
+	fs.BoolFunc("api-trace", "enable trace of api calls", myflag.BoolFlagFn(&app.APITrace, app.APITrace))
 	fs.BoolFunc("debug", "enable debug messages", myflag.BoolFlagFn(&app.Debug, app.Debug))
 	fs.StringVar(&app.TimeZone, "time-zone", app.TimeZone, "Override the system time zone")
 	fs.BoolFunc("skip-verify-ssl", "Skip SSL verification", myflag.BoolFlagFn(&app.SkipSSL, app.SkipSSL))
@@ -160,7 +163,18 @@ func (app *SharedFlags) Start(ctx context.Context) error {
 			app.Immich.SetEndPoint(app.API)
 		}
 		if app.APITrace {
-			app.Immich.EnableAppTrace(true)
+			if app.APITraceWriter == nil {
+				err := configuration.MakeDirForFile(app.LogFile)
+				if err != nil {
+					return err
+				}
+				app.APITraceWriterName = strings.TrimSuffix(app.LogFile, filepath.Ext(app.LogFile)) + ".trace.log"
+				app.APITraceWriter, err = os.OpenFile(app.APITraceWriterName, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o664)
+				if err != nil {
+					return err
+				}
+				app.Immich.EnableAppTrace(app.APITraceWriter)
+			}
 		}
 		if app.DeviceUUID != "" {
 			app.Immich.SetDeviceUUID(app.DeviceUUID)

cmd/upload/upload_test.go

@@ -55,7 +55,7 @@ func (c *stubIC) UpdateAsset(ctx context.Context, id string, a *browser.LocalAss
 	return nil, nil
 }
 
-func (c *stubIC) EnableAppTrace(bool) {}
+func (c *stubIC) EnableAppTrace(w io.Writer) {}
 
 func (c *stubIC) GetServerStatistics(ctx context.Context) (immich.ServerStatistics, error) {
 	return immich.ServerStatistics{}, nil

docs/releases.md

@@ -1,11 +1,30 @@
 # Release notes 
 
+- feat: provide a trace of all API calls.
+Use the option `-api-trace` to log all immich calls in a file.
+
+```log
+2024-07-03T08:17:25+02:00 AssetUpload POST http://localhost:2283/api/assets
+   Accept [application/json]
+   Content-Type [multipart/form-data; boundary=1a9ca81d17452313f49073626c0ac04065fc7445efd3fadeffc5704663ed]
+   X-Api-Key [xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx]
+-- Binary body not dumped --
+2024-07-03T08:17:26+02:00 201 Created
+-- response body --
+{
+   "id": "1d839b04-fcf8-4bbb-bfbb-ab873159231b",
+   "duplicate": false
+  }
+-- response body end --
+```
+
 - feat [#332]: Force the use of Google Photos date and GPS
 Immich-go systematically feed immich with GP's date of capture and GPS coordinates. 
 
 This fixes a standing issue with some MP4 files delivered in a takeout file with a wrong date of capture.
 
 ## Release 0.18.2
+
 - fix [#347](https://github.com/simulot/immich-go/issues/347) Denied access to admin only route: /api/job
 
 ## Release 0.18.1

immich/call.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"net/http"
 	"strings"
+	"time"
 )
 
 type TooManyInternalError struct {
@@ -173,8 +174,8 @@ func (sc *serverCall) do(fnRequest requestFunction, opts ...serverResponseOption
 		return sc.Err(req, nil, nil)
 	}
 
-	if sc.ic.APITrace /* && req.Header.Get("Content-Type") == "application/json"*/ {
-		_ = sc.joinError(setTraceJSONRequest()(sc, req))
+	if sc.ic.apiTraceWriter != nil /* && req.Header.Get("Content-Type") == "application/json"*/ {
+		_ = sc.joinError(setTraceRequest()(sc, req))
 	}
 
 	resp, err = sc.ic.client.Do(req)
@@ -236,7 +237,7 @@ func setJSONBody(object any) serverRequestOption {
 	return func(sc *serverCall, req *http.Request) error {
 		b := bytes.NewBuffer(nil)
 		enc := json.NewEncoder(b)
-		if sc.ic.APITrace {
+		if sc.ic.apiTraceWriter != nil {
 			enc.SetIndent("", " ")
 		}
 		err := enc.Encode(object)
@@ -267,6 +268,14 @@ func responseJSON[T any](object *T) serverResponseOption {
 					return nil
 				}
 				err := json.NewDecoder(resp.Body).Decode(object)
+				if sc.ic.apiTraceWriter != nil {
+					fmt.Fprintln(sc.ic.apiTraceWriter, time.Now().Format(time.RFC3339), resp.Status)
+					fmt.Fprintln(sc.ic.apiTraceWriter, "-- response body --")
+					dec := json.NewEncoder(newLimitWriter(sc.ic.apiTraceWriter, 100))
+					dec.SetIndent("", " ")
+					_ = dec.Encode(object)
+					fmt.Fprint(sc.ic.apiTraceWriter, "-- response body end --\n\n")
+				}
 				return err
 			}
 		}

immich/call_test.go

@@ -73,7 +73,7 @@ func TestCall(t *testing.T) {
 				t.Fail()
 				return
 			}
-			ic.EnableAppTrace(true)
+			// ic.EnableAppTrace(true)
 			r := map[string]string{}
 			err = ic.newServerCall(ctx, tst.name).do(tst.requestFn, responseJSON(&r))
 			if tst.expectedErr && err == nil {

immich/client.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"io"
 	"net/http"
 	"os"
 	"strings"
@@ -24,7 +25,7 @@ type ImmichClient struct {
 	DeviceUUID          string        // Device
 	Retries             int           // Number of attempts on 500 errors
 	RetriesDelay        time.Duration // Duration between retries
-	APITrace            bool
+	apiTraceWriter      io.Writer
 	supportedMediaTypes SupportedMedia // Server's list of supported medias
 }
 
@@ -36,8 +37,8 @@ func (ic *ImmichClient) SetDeviceUUID(deviceUUID string) {
 	ic.DeviceUUID = deviceUUID
 }
 
-func (ic *ImmichClient) EnableAppTrace(state bool) {
-	ic.APITrace = state
+func (ic *ImmichClient) EnableAppTrace(w io.Writer) {
+	ic.apiTraceWriter = w
 }
 
 func (ic *ImmichClient) SupportedMedia() SupportedMedia {

immich/immich.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"io"
 	"sync"
 	"time"
 
@@ -15,7 +16,7 @@ import (
 // interface used to mock up the client
 type ImmichInterface interface {
 	SetEndPoint(string)
-	EnableAppTrace(bool)
+	EnableAppTrace(w io.Writer)
 	SetDeviceUUID(string)
 	PingServer(ctx context.Context) error
 	ValidateConnection(ctx context.Context) (User, error)

immich/trace.go

@@ -1,41 +1,93 @@
 package immich
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"net/http"
-	"os"
+	"time"
 )
 
 /*
   To inspect requests or response request, add setTraceJSONRequest or setTraceJSONResponse to the request options
 
 */
 
+type limitWriter struct {
+	W     io.Writer
+	Err   error
+	Lines int
+}
+
+func newLimitWriter(w io.Writer, lines int) *limitWriter {
+	return &limitWriter{W: w, Lines: lines, Err: nil}
+}
+
+func (lw *limitWriter) Write(b []byte) (int, error) {
+	if lw.Lines < 0 {
+		return 0, lw.Err
+	}
+	total := 0
+	for len(b) > 0 && lw.Lines >= 0 && lw.Err == nil {
+		p := bytes.Index(b, []byte{'\n'})
+		var n int
+		if p > 0 {
+			n, lw.Err = lw.W.Write(b[:p+1])
+			b = b[p+1:]
+			lw.Lines--
+		} else {
+			n, lw.Err = lw.W.Write(b)
+		}
+		total += n
+	}
+	if lw.Lines < 0 {
+		_, _ = lw.W.Write([]byte(".... truncated ....\n"))
+	}
+	return total, lw.Err
+}
+
+func (lw *limitWriter) Close() error {
+	if closer, ok := lw.W.(io.Closer); ok {
+		return closer.Close()
+	}
+	return nil
+}
+
 type smartBodyCloser struct {
 	r    io.Reader
 	body io.ReadCloser
+	w    io.Writer
 }
 
 func (sb *smartBodyCloser) Close() error {
-	fmt.Println("\n--- BODY ---")
+	fmt.Fprint(sb.w, "-- request body end --\n\n")
 	return sb.body.Close()
 }
 
 func (sb *smartBodyCloser) Read(b []byte) (int, error) {
 	return sb.r.Read(b)
 }
 
-func setTraceJSONRequest() serverRequestOption {
+func setTraceRequest() serverRequestOption {
 	return func(sc *serverCall, req *http.Request) error {
-		fmt.Println("--------------------")
-		fmt.Println(req.Method, req.URL.String())
+		fmt.Fprintln(sc.ic.apiTraceWriter, time.Now().Format(time.RFC3339), sc.endPoint, req.Method, req.URL.String())
 		for h, v := range req.Header {
-			fmt.Println(h, v)
+			if h == "X-Api-Key" {
+				fmt.Fprintln(sc.ic.apiTraceWriter, "  ", h, []string{"redacted"})
+			} else {
+				fmt.Fprintln(sc.ic.apiTraceWriter, "  ", h, v)
+			}
 		}
-		if req.Body != nil {
-			tr := io.TeeReader(req.Body, os.Stdout)
-			req.Body = &smartBodyCloser{body: req.Body, r: tr}
+		if req.Header.Get("Content-Type") == "application/json" {
+			fmt.Fprintln(sc.ic.apiTraceWriter, "-- request JSON Body --")
+			if req.Body != nil {
+				tr := io.TeeReader(req.Body, newLimitWriter(sc.ic.apiTraceWriter, 100))
+				req.Body = &smartBodyCloser{body: req.Body, r: tr, w: sc.ic.apiTraceWriter}
+			}
+		} else {
+			if req.Body != nil {
+				fmt.Fprintln(sc.ic.apiTraceWriter, "-- Empty body or binary body not dumped --")
+			}
 		}
 		return nil
 	}

main.go

@@ -131,5 +131,8 @@ func Run(ctx context.Context) error {
 		app.Log.Error(err.Error())
 	}
 	fmt.Println("Check the log file: ", app.LogFile)
+	if app.APITraceWriter != nil {
+		fmt.Println("Check the trace file: ", app.APITraceWriterName)
+	}
 	return err
 }

readme.md

@@ -88,6 +88,7 @@ Example: Immich-go check the server's SSL certificate. you can disable this beha
 | `-time-zone=time_zone_name`              | Set the time zone for dates without time zone information                                                                                                                     | the system's time zone                                                                                                                                                                                                 |
 | `-no-ui`                                 | Disable the user interface                                                                                                                                                    | `false`                                                                                                                                                                                                                |
 | `-debug-counters`                        | Enable the generation a CSV beside the log file                                                                                                                               | `false`                                                                                                                                                                                                                  |
+| `-api-trace` | Enable trace of API calls | `false` |
 
 ## Command `upload`