diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..b3c2870 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,21 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +If feasible/relevant, please provide a code snippet (inline or with Go playground) to reproduce the issue. + + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..c89e51c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,10 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +Please use discussions https://github.com/bool64/logz/discussions/categories/ideas to share feature ideas. diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 0000000..ed600c7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,10 @@ +--- +name: Question +about: Any question about features or usage +title: '' +labels: '' +assignees: '' + +--- + +Please use discussions https://github.com/bool64/logz/discussions/categories/q-a to make your question more discoverable by other folks. diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index b6cb223..bd4fb36 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -12,27 +12,43 @@ on: description: 'New Ref' required: true +# Cancel the workflow in progress in newer build is about to start. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + env: GO111MODULE: "on" CACHE_BENCHMARK: "off" # Enables benchmark result reuse between runs, may skew latency results. RUN_BASE_BENCHMARK: "on" # Runs benchmark for PR base in case benchmark result is missing. + GO_VERSION: 1.20.x jobs: bench: - strategy: - matrix: - go-version: [ 1.17.x ] runs-on: ubuntu-latest steps: - - name: Install Go - uses: actions/setup-go@v2 + - name: Install Go stable + if: env.GO_VERSION != 'tip' + uses: actions/setup-go@v4 with: - go-version: ${{ matrix.go-version }} + go-version: ${{ env.GO_VERSION }} + + - name: Install Go tip + if: env.GO_VERSION == 'tip' + run: | + curl -sL https://storage.googleapis.com/go-build-snap/go/linux-amd64/$(git ls-remote https://github.com/golang/go.git HEAD | awk '{print $1;}').tar.gz -o gotip.tar.gz + ls -lah gotip.tar.gz + mkdir -p ~/sdk/gotip + tar -C ~/sdk/gotip -xzf gotip.tar.gz + ~/sdk/gotip/bin/go version + echo "PATH=$HOME/go/bin:$HOME/sdk/gotip/bin/:$PATH" >> $GITHUB_ENV + - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: ref: ${{ (github.event.inputs.new != '') && github.event.inputs.new || github.event.ref }} + - name: Go cache - uses: actions/cache@v2 + uses: actions/cache@v3 with: # In order: # * Module download cache @@ -43,49 +59,58 @@ jobs: key: ${{ runner.os }}-go-cache-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-go-cache + - name: Restore benchstat - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/go/bin/benchstat - key: ${{ runner.os }}-benchstat + key: ${{ runner.os }}-benchstat-legacy + - name: Restore base benchmark result + id: base-benchmark if: env.CACHE_BENCHMARK == 'on' - id: benchmark-base - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: | bench-master.txt bench-main.txt # Use base sha for PR or new commit hash for master/main push in benchmark result key. key: ${{ runner.os }}-bench-${{ (github.event.pull_request.base.sha != github.event.after) && github.event.pull_request.base.sha || github.event.after }} - - name: Checkout base code - if: env.RUN_BASE_BENCHMARK == 'on' && steps.benchmark-base.outputs.cache-hit != 'true' && (github.event.pull_request.base.sha != '' || github.event.inputs.old != '') - uses: actions/checkout@v2 - with: - ref: ${{ (github.event.pull_request.base.sha != '' ) && github.event.pull_request.base.sha || github.event.inputs.old }} - path: __base - - name: Run base benchmark - if: env.RUN_BASE_BENCHMARK == 'on' && steps.benchmark-base.outputs.cache-hit != 'true' && (github.event.pull_request.base.sha != '' || github.event.inputs.old != '') + + - name: Run benchmark + run: | + export REF_NAME=new + make bench + OUTPUT=$(make bench-stat-diff) + echo "${OUTPUT}" + echo "diff<> $GITHUB_OUTPUT && echo "$OUTPUT" >> $GITHUB_OUTPUT && echo "EOF" >> $GITHUB_OUTPUT + OUTPUT=$(make bench-stat) + echo "${OUTPUT}" + echo "result<> $GITHUB_OUTPUT && echo "$OUTPUT" >> $GITHUB_OUTPUT && echo "EOF" >> $GITHUB_OUTPUT + + - name: Run benchmark for base code + if: env.RUN_BASE_BENCHMARK == 'on' && steps.base-benchmark.outputs.cache-hit != 'true' && (github.event.pull_request.base.sha != '' || github.event.inputs.old != '') run: | + git fetch origin master ${{ github.event.pull_request.base.sha }} + HEAD=$(git rev-parse HEAD) + git reset --hard ${{ github.event.pull_request.base.sha }} export REF_NAME=master - cd __base - make | grep bench-run && (BENCH_COUNT=5 make bench-run bench-stat && cp bench-master.txt ../bench-master.txt) || echo "No benchmarks in base" - - name: Benchmark + make bench-run bench-stat + git reset --hard $HEAD + + - name: Benchmark stats id: bench run: | export REF_NAME=new - BENCH_COUNT=5 make bench OUTPUT=$(make bench-stat-diff) - OUTPUT="${OUTPUT//'%'/'%25'}" - OUTPUT="${OUTPUT//$'\n'/'%0A'}" - OUTPUT="${OUTPUT//$'\r'/'%0D'}" - echo "::set-output name=diff::$OUTPUT" + echo "${OUTPUT}" + echo "diff<> $GITHUB_OUTPUT && echo "$OUTPUT" >> $GITHUB_OUTPUT && echo "EOF" >> $GITHUB_OUTPUT OUTPUT=$(make bench-stat) - OUTPUT="${OUTPUT//'%'/'%25'}" - OUTPUT="${OUTPUT//$'\n'/'%0A'}" - OUTPUT="${OUTPUT//$'\r'/'%0D'}" - echo "::set-output name=result::$OUTPUT" - - name: Comment Benchmark Result + echo "${OUTPUT}" + echo "result<> $GITHUB_OUTPUT && echo "$OUTPUT" >> $GITHUB_OUTPUT && echo "EOF" >> $GITHUB_OUTPUT + + - name: Comment benchmark result + continue-on-error: true uses: marocchino/sticky-pull-request-comment@v2 with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/cloc.yml b/.github/workflows/cloc.yml index ed1ddb9..3f1fc19 100644 --- a/.github/workflows/cloc.yml +++ b/.github/workflows/cloc.yml @@ -2,30 +2,37 @@ name: cloc on: pull_request: + +# Cancel the workflow in progress in newer build is about to start. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: cloc: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: path: pr - name: Checkout base code - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.base.sha }} path: base - - name: Count Lines Of Code + - name: Count lines of code id: loc run: | - curl -OL https://github.com/vearutop/sccdiff/releases/download/v1.0.1/linux_amd64.tar.gz && tar xf linux_amd64.tar.gz + curl -sLO https://github.com/vearutop/sccdiff/releases/download/v1.0.3/linux_amd64.tar.gz && tar xf linux_amd64.tar.gz + sccdiff_hash=$(git hash-object ./sccdiff) + [ "$sccdiff_hash" == "ae8a07b687bd3dba60861584efe724351aa7ff63" ] || (echo "::error::unexpected hash for sccdiff, possible tampering: $sccdiff_hash" && exit 1) OUTPUT=$(cd pr && ../sccdiff -basedir ../base) - OUTPUT="${OUTPUT//'%'/'%25'}" - OUTPUT="${OUTPUT//$'\n'/'%0A'}" - OUTPUT="${OUTPUT//$'\r'/'%0D'}" - echo "::set-output name=diff::$OUTPUT" + echo "${OUTPUT}" + echo "diff<> $GITHUB_OUTPUT && echo "$OUTPUT" >> $GITHUB_OUTPUT && echo "EOF" >> $GITHUB_OUTPUT - - name: Comment Code Lines + - name: Comment lines of code + continue-on-error: true uses: marocchino/sticky-pull-request-comment@v2 with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index cf9c148..e651212 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -8,17 +8,26 @@ on: - master - main pull_request: + +# Cancel the workflow in progress in newer build is about to start. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: golangci: name: golangci-lint runs-on: ubuntu-latest steps: + - uses: actions/setup-go@v3 + with: + go-version: 1.20.x - uses: actions/checkout@v2 - name: golangci-lint - uses: golangci/golangci-lint-action@v2.5.2 + uses: golangci/golangci-lint-action@v3.4.0 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.41.1 + version: v1.51.1 # Optional: working directory, useful for monorepos # working-directory: somedir diff --git a/.github/workflows/gorelease.yml b/.github/workflows/gorelease.yml index a733761..41767ba 100644 --- a/.github/workflows/gorelease.yml +++ b/.github/workflows/gorelease.yml @@ -2,43 +2,54 @@ name: gorelease on: pull_request: + +# Cancel the workflow in progress in newer build is about to start. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + env: - GO111MODULE: "on" + GO_VERSION: 1.20.x jobs: gorelease: - strategy: - matrix: - go-version: [ 1.17.x ] runs-on: ubuntu-latest steps: - - name: Install Go - uses: actions/setup-go@v2 + - name: Install Go stable + if: env.GO_VERSION != 'tip' + uses: actions/setup-go@v4 with: - go-version: ${{ matrix.go-version }} + go-version: ${{ env.GO_VERSION }} + - name: Install Go tip + if: env.GO_VERSION == 'tip' + run: | + curl -sL https://storage.googleapis.com/go-build-snap/go/linux-amd64/$(git ls-remote https://github.com/golang/go.git HEAD | awk '{print $1;}').tar.gz -o gotip.tar.gz + ls -lah gotip.tar.gz + mkdir -p ~/sdk/gotip + tar -C ~/sdk/gotip -xzf gotip.tar.gz + ~/sdk/gotip/bin/go version + echo "PATH=$HOME/go/bin:$HOME/sdk/gotip/bin/:$PATH" >> $GITHUB_ENV - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Gorelease cache - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: | ~/go/bin/gorelease - key: ${{ runner.os }}-gorelease + key: ${{ runner.os }}-gorelease-generic - name: Gorelease id: gorelease run: | test -e ~/go/bin/gorelease || go install golang.org/x/exp/cmd/gorelease@latest - OUTPUT=$(gorelease || exit 0) - OUTPUT="${OUTPUT//'%'/'%25'}" - OUTPUT="${OUTPUT//$'\n'/'%0A'}" - OUTPUT="${OUTPUT//$'\r'/'%0D'}" - echo "::set-output name=report::$OUTPUT" - - name: Comment Report + OUTPUT=$(gorelease 2>&1 || exit 0) + echo "${OUTPUT}" + echo "report<> $GITHUB_OUTPUT && echo "$OUTPUT" >> $GITHUB_OUTPUT && echo "EOF" >> $GITHUB_OUTPUT + - name: Comment report + continue-on-error: true uses: marocchino/sticky-pull-request-comment@v2 with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} header: gorelease message: | - ### API Changes + ### Go API Changes
             ${{ steps.gorelease.outputs.report }}
diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml
index 79ed0f6..2817dcc 100644
--- a/.github/workflows/test-unit.yml
+++ b/.github/workflows/test-unit.yml
@@ -6,24 +6,45 @@ on:
       - master
       - main
   pull_request:
+
+# Cancel the workflow in progress in newer build is about to start.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 env:
   GO111MODULE: "on"
   RUN_BASE_COVERAGE: "on"  # Runs test for PR base in case base test coverage is missing.
+  COV_GO_VERSION: 1.20.x   # Version of Go to collect coverage
+  TARGET_DELTA_COV: 90     # Target coverage of changed lines, in percents
 jobs:
   test:
     strategy:
       matrix:
-        go-version: [ 1.13.x, 1.14.x, 1.15.x, 1.16.x, 1.17.x ]
+        go-version: [ 1.13.x, 1.19.x, 1.20.x ]
     runs-on: ubuntu-latest
     steps:
-      - name: Install Go
-        uses: actions/setup-go@v2
+      - name: Install Go stable
+        if: matrix.go-version != 'tip'
+        uses: actions/setup-go@v4
         with:
           go-version: ${{ matrix.go-version }}
+
+      - name: Install Go tip
+        if: matrix.go-version == 'tip'
+        run: |
+          curl -sL https://storage.googleapis.com/go-build-snap/go/linux-amd64/$(git ls-remote https://github.com/golang/go.git HEAD | awk '{print $1;}').tar.gz -o gotip.tar.gz
+          ls -lah gotip.tar.gz
+          mkdir -p ~/sdk/gotip
+          tar -C ~/sdk/gotip -xzf gotip.tar.gz
+          ~/sdk/gotip/bin/go version
+          echo "PATH=$HOME/go/bin:$HOME/sdk/gotip/bin/:$PATH" >> $GITHUB_ENV
+
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+
       - name: Go cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           # In order:
           # * Module download cache
@@ -34,42 +55,55 @@ jobs:
           key: ${{ runner.os }}-go-cache-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-cache
+
       - name: Restore base test coverage
-        if: matrix.go-version == '1.17.x'
+        id: base-coverage
+        if: matrix.go-version == env.COV_GO_VERSION && github.event.pull_request.base.sha != ''
         uses: actions/cache@v2
         with:
           path: |
             unit-base.txt
           # Use base sha for PR or new commit hash for master/main push in test result key.
           key: ${{ runner.os }}-unit-test-coverage-${{ (github.event.pull_request.base.sha != github.event.after) && github.event.pull_request.base.sha || github.event.after }}
-      - name: Checkout base code
-        if: matrix.go-version == '1.17.x' && env.RUN_BASE_COVERAGE == 'on' && steps.benchmark-base.outputs.cache-hit != 'true' && github.event.pull_request.base.sha != ''
-        uses: actions/checkout@v2
-        with:
-          ref: ${{ github.event.pull_request.base.sha }}
-          path: __base
+
       - name: Run test for base code
-        if: matrix.go-version == '1.17.x' && env.RUN_BASE_COVERAGE == 'on' && steps.benchmark-base.outputs.cache-hit != 'true' && github.event.pull_request.base.sha != ''
+        if: matrix.go-version == env.COV_GO_VERSION && env.RUN_BASE_COVERAGE == 'on' && steps.base-coverage.outputs.cache-hit != 'true' && github.event.pull_request.base.sha != ''
         run: |
-          cd __base
-          make | grep test-unit && (make test-unit && go tool cover -func=./unit.coverprofile | sed -e 's/.go:[0-9]*:\t/.go\t/g' | sed -e 's/\t\t*/\t/g'  > ../unit-base.txt) || echo "No test-unit in base"
+          git fetch origin master ${{ github.event.pull_request.base.sha }}
+          HEAD=$(git rev-parse HEAD)
+          git reset --hard ${{ github.event.pull_request.base.sha }}
+          (make test-unit && go tool cover -func=./unit.coverprofile > unit-base.txt) || echo "No test-unit in base"
+          git reset --hard $HEAD
+
       - name: Test
         id: test
         run: |
           make test-unit
-          go tool cover -func=./unit.coverprofile | sed -e 's/.go:[0-9]*:\t/.go\t/g' | sed -e 's/\t\t*/\t/g'  > unit.txt
-          OUTPUT=$(test -e unit-base.txt && (diff unit-base.txt unit.txt || exit 0) || cat unit.txt)
-          OUTPUT="${OUTPUT//'%'/'%25'}"
-          OUTPUT="${OUTPUT//$'\n'/'%0A'}"
-          OUTPUT="${OUTPUT//$'\r'/'%0D'}"
+          go tool cover -func=./unit.coverprofile > unit.txt
           TOTAL=$(grep 'total:' unit.txt)
-          echo "::set-output name=diff::$OUTPUT"
-          echo "::set-output name=total::$TOTAL"
-      - name: Store base coverage
-        if: ${{ github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' }}
-        run: cp unit.txt unit-base.txt
-      - name: Comment Test Coverage
-        if: matrix.go-version == '1.17.x'
+          echo "${TOTAL}"
+          echo "total=$TOTAL" >> $GITHUB_OUTPUT
+
+      - name: Annotate missing test coverage
+        id: annotate
+        if: matrix.go-version == env.COV_GO_VERSION && github.event.pull_request.base.sha != ''
+        run: |
+          curl -sLO https://github.com/vearutop/gocovdiff/releases/download/v1.3.6/linux_amd64.tar.gz && tar xf linux_amd64.tar.gz
+          gocovdiff_hash=$(git hash-object ./gocovdiff)
+          [ "$gocovdiff_hash" == "8e507e0d671d4d6dfb3612309b72b163492f28eb" ] || (echo "::error::unexpected hash for gocovdiff, possible tampering: $gocovdiff_hash" && exit 1)
+          git fetch origin master ${{ github.event.pull_request.base.sha }}
+          REP=$(./gocovdiff -cov unit.coverprofile -gha-annotations gha-unit.txt -delta-cov-file delta-cov-unit.txt -target-delta-cov ${TARGET_DELTA_COV})
+          echo "${REP}"
+          cat gha-unit.txt
+          DIFF=$(test -e unit-base.txt && ./gocovdiff -func-cov unit.txt -func-base-cov unit-base.txt || echo "Missing base coverage file")
+          TOTAL=$(cat delta-cov-unit.txt)
+          echo "rep<> $GITHUB_OUTPUT && echo "$REP" >> $GITHUB_OUTPUT && echo "EOF" >> $GITHUB_OUTPUT
+          echo "diff<> $GITHUB_OUTPUT && echo "$DIFF" >> $GITHUB_OUTPUT && echo "EOF" >> $GITHUB_OUTPUT
+          echo "total<> $GITHUB_OUTPUT && echo "$TOTAL" >> $GITHUB_OUTPUT && echo "EOF" >> $GITHUB_OUTPUT
+
+      - name: Comment test coverage
+        continue-on-error: true
+        if: matrix.go-version == env.COV_GO_VERSION && github.event.pull_request.base.sha != ''
         uses: marocchino/sticky-pull-request-comment@v2
         with:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -77,15 +111,25 @@ jobs:
           message: |
             ### Unit Test Coverage
             ${{ steps.test.outputs.total }}
+            ${{ steps.annotate.outputs.total }}
+            
Coverage of changed lines + + ${{ steps.annotate.outputs.rep }} + +
+
Coverage diff with base branch - ```diff - ${{ steps.test.outputs.diff }} - ``` + ${{ steps.annotate.outputs.diff }} +
+ - name: Store base coverage + if: ${{ github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' }} + run: cp unit.txt unit-base.txt + - name: Upload code coverage - if: matrix.go-version == '1.17.x' + if: matrix.go-version == env.COV_GO_VERSION uses: codecov/codecov-action@v1 with: file: ./unit.coverprofile diff --git a/.golangci.yml b/.golangci.yml index c203410..feb9910 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -20,6 +20,7 @@ linters-settings: linters: enable-all: true disable: + - containedctx - lll - maligned - gochecknoglobals @@ -33,6 +34,18 @@ linters: - scopelint # deprecated - ifshort # too many false positives - golint # deprecated + - varnamelen + - tagliatelle + - errname + - ireturn + - exhaustruct + - nonamedreturns + - nosnakecase + - structcheck + - varcheck + - deadcode + - testableexamples + - dupword issues: exclude-use-default: false @@ -44,5 +57,13 @@ issues: - noctx - funlen - dupl + - structcheck + - unused + - unparam + - nosnakecase path: "_test.go" + - linters: + - errcheck # Error checking omitted for brevity. + - gosec + path: "example_" diff --git a/Makefile b/Makefile index d60ee4b..053e982 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -#GOLANGCI_LINT_VERSION := "v1.41.1" # Optional configuration to pinpoint golangci-lint version. +#GOLANGCI_LINT_VERSION := "v1.51.1" # Optional configuration to pinpoint golangci-lint version. # The head of Makefile determines location of dev-go to include standard targets. GO ?= go @@ -23,7 +23,7 @@ ifeq ($(DEVGO_PATH),) DEVGO_PATH := $(shell GO111MODULE=on $(GO) list ${modVendor} -f '{{.Dir}}' -m github.com/bool64/dev) ifeq ($(DEVGO_PATH),) $(info Module github.com/bool64/dev not found, downloading.) - DEVGO_PATH := $(shell export GO111MODULE=on && $(GO) mod tidy && $(GO) list -f '{{.Dir}}' -m github.com/bool64/dev) + DEVGO_PATH := $(shell export GO111MODULE=on && $(GO) get github.com/bool64/dev && $(GO) list -f '{{.Dir}}' -m github.com/bool64/dev) endif endif diff --git a/README.md b/README.md index 0e726fb..9eab3ab 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ They are useful for last mile observability of logs. * Adapter for [`go.uber.org/zap`](./zzap). * Adapter for [`github.com/bool64/ctxd`](./ctxz). * HTTP handler to serve aggregated messages. +* Best effort [filtering](https://pkg.go.dev/github.com/vearutop/lograte/filter#Dynamic) of dynamic parts of messages. ![Screenshot](./_examples/screenshot.png) @@ -30,6 +31,7 @@ zc := zap.NewDevelopmentConfig() zz, lo := zzap.NewOption(logz.Config{ MaxCardinality: 5, MaxSamples: 10, + FilterMessage: true, }) l, err := zc.Build(zz) @@ -41,6 +43,9 @@ l.Debug("starting example") l.Sugar().Infow("sample info", "one", 1, "two", 2) l.Error("unexpected end of the world") +l.Warn("please be careful with asf872dfse732") +l.Warn("please be careful with sdf890sdf0w9d") // These messages will be grouped together with help of filtering. + l.Info("starting server at http://localhost:6060/") err = http.ListenAndServe("0.0.0.0:6060", logzpage.Handler(lo...)) if err != nil { diff --git a/_examples/screenshot.png b/_examples/screenshot.png index fb15d85..eb6b553 100644 Binary files a/_examples/screenshot.png and b/_examples/screenshot.png differ diff --git a/_examples/zzap/go.mod b/_examples/zzap/go.mod index b744ed0..84f6378 100644 --- a/_examples/zzap/go.mod +++ b/_examples/zzap/go.mod @@ -1,11 +1,18 @@ module zapage -go 1.11 +go 1.17 replace github.com/bool64/logz => ./../.. require ( github.com/bool64/logz v0.0.0-00010101000000-000000000000 github.com/drhodes/golorem v0.0.0-20160418191928-ecccc744c2d9 - go.uber.org/zap v1.16.0 + go.uber.org/zap v1.24.0 +) + +require ( + github.com/vearutop/dynhist-go v1.0.0 // indirect + github.com/vearutop/lograte v1.0.5 // indirect + go.uber.org/atomic v1.10.0 // indirect + go.uber.org/multierr v1.10.0 // indirect ) diff --git a/_examples/zzap/go.sum b/_examples/zzap/go.sum index c43669f..e4b40d9 100644 --- a/_examples/zzap/go.sum +++ b/_examples/zzap/go.sum @@ -1,68 +1,79 @@ -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/bool64/dev v0.1.13 h1:edfquat4DBM1FBVMtm+z4Ksf1Du/S+qiAjslmE3xdIM= -github.com/bool64/dev v0.1.13/go.mod h1:pn52JC52uSgpazChx9CeXyG+S3sW2V36HHoLNBbscdg= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/bool64/ctxd v1.2.1/go.mod h1:ZG6QkeGVLTiUl2mxPpyHmFhDzFZCyocr9hluBV3LYuc= +github.com/bool64/dev v0.2.20/go.mod h1:iJbh1y/HkunEPhgebWRNcs8wfGq7sjvJ6W5iabL8ACg= +github.com/bool64/dev v0.2.24/go.mod h1:iJbh1y/HkunEPhgebWRNcs8wfGq7sjvJ6W5iabL8ACg= +github.com/bool64/dev v0.2.27 h1:mFT+B74mFVgUeUmm/EbfM6ELPA55lEXBjQ/AOHCwCOc= +github.com/bool64/dev v0.2.27/go.mod h1:iJbh1y/HkunEPhgebWRNcs8wfGq7sjvJ6W5iabL8ACg= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/drhodes/golorem v0.0.0-20160418191928-ecccc744c2d9 h1:EQOZw/LCQ0SM4sNez3EhUf9gQalQrLrs4mPtmQa+d58= github.com/drhodes/golorem v0.0.0-20160418191928-ecccc744c2d9/go.mod h1:NsKVpF4h4j13Vm6Cx7Kf0V03aJKjfaStvm5rvK4+FyQ= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/swaggest/usecase v1.2.0/go.mod h1:oc5+QoAxG3Et5Gl9lRXgEOm00l4VN9gdVQSMIa5EeLY= github.com/vearutop/dynhist-go v1.0.0 h1:SLYGcQMzmIoWnuqtF9lV8ULUBC735A56M5nL2CNth4c= github.com/vearutop/dynhist-go v1.0.0/go.mod h1:7Cgyu5Ww8FwdB+Y+zawRz9cQT5oXAxw294L9lQ+JI/k= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +github.com/vearutop/lograte v1.0.5 h1:H3XAhqjggYt0oyJ8AqzxHClKpJW0odmu3mYYMeLcvkQ= +github.com/vearutop/lograte v1.0.5/go.mod h1:hN7Zo1dpCKPVEEkxPM3zKtZHpkxtOip0cSITUZKZAxg= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= -go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/_examples/zzap/main.go b/_examples/zzap/main.go index c145264..7ea0ccc 100644 --- a/_examples/zzap/main.go +++ b/_examples/zzap/main.go @@ -3,6 +3,7 @@ package main import ( "math/rand" "net/http" + "strconv" "time" "github.com/bool64/logz" @@ -17,6 +18,7 @@ func main() { zz, lo := zzap.NewOption(logz.Config{ MaxCardinality: 5, MaxSamples: 10, + FilterMessage: true, }) l, err := zc.Build(zz) @@ -53,7 +55,7 @@ func main() { keysAndValues = append(keysAndValues, lorem.Word(3, 6), lorem.Word(3, 6)) } - lw.Sugar().Warnw(msg, keysAndValues...) + lw.Sugar().Warnw(msg+" Dynamic"+strconv.Itoa(i)+".", keysAndValues...) time.Sleep(time.Duration(1e9 * rand.Float64())) } @@ -61,7 +63,7 @@ func main() { } l.Info("starting server at http://localhost:6060/") - err = http.ListenAndServe("0.0.0.0:6060", logzpage.Handler(lo...)) + err = http.ListenAndServe("localhost:6060", logzpage.Handler(lo...)) if err != nil { l.Fatal(err.Error()) } diff --git a/go.mod b/go.mod index eabc790..4a9d7a2 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,20 @@ module github.com/bool64/logz -go 1.13 +go 1.17 require ( - github.com/bool64/ctxd v1.0.0 - github.com/bool64/dev v0.1.41 - github.com/stretchr/testify v1.7.0 + github.com/bool64/ctxd v1.2.1 + github.com/bool64/dev v0.2.27 + github.com/stretchr/testify v1.8.2 github.com/vearutop/dynhist-go v1.0.0 - go.uber.org/zap v1.19.0 + github.com/vearutop/lograte v1.0.5 + go.uber.org/zap v1.16.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + go.uber.org/atomic v1.6.0 // indirect + go.uber.org/multierr v1.5.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 36fd1d5..3032f0c 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,17 @@ -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/bool64/ctxd v1.0.0 h1:Btvo6BU5FulG7H2V9m5Il/2vcqT0nxe86AiEpbG08/I= -github.com/bool64/ctxd v1.0.0/go.mod h1:+rjDVFNOJeO+xlvMqQfG0p53CzuRB7FhPSo5nWSkpQ0= -github.com/bool64/dev v0.1.25/go.mod h1:cTHiTDNc8EewrQPy3p1obNilpMpdmlUesDkFTF2zRWU= -github.com/bool64/dev v0.1.28/go.mod h1:cTHiTDNc8EewrQPy3p1obNilpMpdmlUesDkFTF2zRWU= -github.com/bool64/dev v0.1.41 h1:L554LCQZc3d7mtcdPUgDbSrCVbr48/30zgu0VuC/FTA= -github.com/bool64/dev v0.1.41/go.mod h1:cTHiTDNc8EewrQPy3p1obNilpMpdmlUesDkFTF2zRWU= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/bool64/ctxd v1.2.1 h1:hARFteq0zdn4bwfmxLhak3fXFuvtJVKDH2X29VV/2ls= +github.com/bool64/ctxd v1.2.1/go.mod h1:ZG6QkeGVLTiUl2mxPpyHmFhDzFZCyocr9hluBV3LYuc= +github.com/bool64/dev v0.2.20/go.mod h1:iJbh1y/HkunEPhgebWRNcs8wfGq7sjvJ6W5iabL8ACg= +github.com/bool64/dev v0.2.24/go.mod h1:iJbh1y/HkunEPhgebWRNcs8wfGq7sjvJ6W5iabL8ACg= +github.com/bool64/dev v0.2.27 h1:mFT+B74mFVgUeUmm/EbfM6ELPA55lEXBjQ/AOHCwCOc= +github.com/bool64/dev v0.2.27/go.mod h1:iJbh1y/HkunEPhgebWRNcs8wfGq7sjvJ6W5iabL8ACg= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -18,43 +21,58 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/swaggest/usecase v0.1.5 h1:xMDWXnYGysVaF2f3ZnmDsn2FlZ8fd3FJD+O+8wl4aNQ= -github.com/swaggest/usecase v0.1.5/go.mod h1:uubX4ZbjQK1Bnl0xX9hOYpb/IUiSoVKk/yQImawbNMU= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/swaggest/usecase v1.2.0 h1:cHVFqxIbHfyTXp02JmWXk+ZADaSa87UZP+b3qL5Nz90= +github.com/swaggest/usecase v1.2.0/go.mod h1:oc5+QoAxG3Et5Gl9lRXgEOm00l4VN9gdVQSMIa5EeLY= github.com/vearutop/dynhist-go v1.0.0 h1:SLYGcQMzmIoWnuqtF9lV8ULUBC735A56M5nL2CNth4c= github.com/vearutop/dynhist-go v1.0.0/go.mod h1:7Cgyu5Ww8FwdB+Y+zawRz9cQT5oXAxw294L9lQ+JI/k= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= -go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +github.com/vearutop/lograte v1.0.5 h1:H3XAhqjggYt0oyJ8AqzxHClKpJW0odmu3mYYMeLcvkQ= +github.com/vearutop/lograte v1.0.5/go.mod h1:hN7Zo1dpCKPVEEkxPM3zKtZHpkxtOip0cSITUZKZAxg= +go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= +go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11 h1:Yq9t9jnGoR+dBuitxdo9l6Q7xh/zOyNnYUtDKaQ3x0E= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/logzpage/handler.go b/logzpage/handler.go index 15e12fa..f40f0ae 100644 --- a/logzpage/handler.go +++ b/logzpage/handler.go @@ -23,7 +23,7 @@ type tplData struct { } // Handler creates HTTP handler to expose entries from observers. -func Handler(observers ...*logz.Observer) http.Handler { // nolint:funlen // This template is lengthy. +func Handler(observers ...*logz.Observer) http.Handler { //nolint:funlen // This template is lengthy. // language=GoTemplate tpl := `{{- /*gotype: github.com/bool64/logz/logzpage.tplData*/ -}} @@ -146,7 +146,7 @@ func Handler(observers ...*logz.Observer) http.Handler { // nolint:funlen // Thi t, err := template.New("Logz").Funcs(template.FuncMap{ "marshal": marshal, "histogram": func(buckets []logz.Bucket) template.HTML { - return template.HTML(Histogram(buckets)) // nolint:gosec // Data is well-formed. + return template.HTML(Histogram(buckets)) //nolint:gosec // Data is well-formed. }, "time": func(t time.Time) string { return t.Format(time.RFC3339) @@ -207,7 +207,7 @@ func Handler(observers ...*logz.Observer) http.Handler { // nolint:funlen // Thi func marshal(v interface{}) template.JS { if bb, ok := v.([]byte); ok { - return template.JS(bb) // nolint:gosec // Data is well-formed. + return template.JS(bb) //nolint:gosec // Data is well-formed. } b := bytes.Buffer{} @@ -217,10 +217,10 @@ func marshal(v interface{}) template.JS { err := enc.Encode(v) if err != nil { - return template.JS(err.Error()) // nolint:gosec // Data is well-formed. + return template.JS(err.Error()) //nolint:gosec // Data is well-formed. } - return template.JS(b.Bytes()) // nolint:gosec // Data is well-formed. + return template.JS(b.Bytes()) //nolint:gosec // Data is well-formed. } // Histogram renders distribution using HTML elements. diff --git a/observer.go b/observer.go index 2d642e1..45738bf 100644 --- a/observer.go +++ b/observer.go @@ -7,6 +7,7 @@ import ( "time" "github.com/vearutop/dynhist-go" + "github.com/vearutop/lograte/filter" ) // Config defines observer configuration. @@ -23,7 +24,7 @@ type Config struct { // Default 10. MaxSamples uint32 - // SamplingInterval is the minimum amount of time needed to pass from a last sample collection in particular message family. + // SamplingInterval is the minimum amount of time needed to pass from a last Sample collection in particular message family. // Messages that are observed too quickly after last sampling are counted, but not sampled. // Default 1ms. SamplingInterval time.Duration @@ -35,6 +36,13 @@ type Config struct { // DistRetentionPeriod is maximum age of bucket. Use -1 for unlimited. // Default one week (168 hours). DistRetentionPeriod time.Duration + + // FilterMessage can reduce cardinality by filtering dynamic parts of messages. + // It uses github.com/vearutop/lograte/filter.Dynamic + // See https://pkg.go.dev/github.com/vearutop/lograte/filter#Dynamic. + // This option is not needed if you already have messages without dynamic interpolated values. + // This option worsen performance, so use it only if you need it. + FilterMessage bool } // Observer keeps track of messages. @@ -54,7 +62,7 @@ type Observer struct { type entry struct { msg string - samples chan sample + samples chan Sample count uint64 first int64 latest int64 @@ -62,13 +70,14 @@ type entry struct { distRetentionPeriod int64 } -type sample struct { +// Sample is a single sample of a message. +type Sample struct { Msg string `json:"msg"` Data interface{} `json:"data"` Time time.Time `json:"time"` } -func (en *entry) push(now int64, sample sample) { +func (en *entry) push(now int64, sample Sample) { cnt := atomic.AddUint64(&en.count, 1) if en.distribution != nil { @@ -89,7 +98,7 @@ func (en *entry) push(now int64, sample sample) { atomic.StoreInt64(&en.latest, now) - // Push new sample. + // Push new Sample. <-en.samples en.samples <- sample } @@ -121,10 +130,10 @@ func (l *Observer) initialize() { } l.other = &entry{ - samples: make(chan sample, l.maxSamples), + samples: make(chan Sample, l.maxSamples), } for i := uint32(0); i < l.maxSamples; i++ { - l.other.samples <- sample{} + l.other.samples <- Sample{} } if l.distResolution > 0 { @@ -141,12 +150,16 @@ func (l *Observer) ObserveMessage(msg string, data interface{}) { tn := time.Now() now := tn.UnixNano() / l.samplingInterval - s := sample{ + s := Sample{ Msg: msg, Data: data, Time: tn, } + if l.Config.FilterMessage { + msg = string(filter.Dynamic([]byte(msg), 200)) + } + if e, ok := l.entries.Load(msg); ok { e.(*entry).push(now, s) @@ -158,7 +171,7 @@ func (l *Observer) ObserveMessage(msg string, data interface{}) { msg: msg, first: now, count: 0, - samples: make(chan sample, l.maxSamples), + samples: make(chan Sample, l.maxSamples), } if l.distResolution > 0 { @@ -169,7 +182,7 @@ func (l *Observer) ObserveMessage(msg string, data interface{}) { } for i := uint32(0); i < l.maxSamples; i++ { - e.samples <- sample{} + e.samples <- Sample{} } l.entries.Store(msg, &e) atomic.AddUint32(&l.count, 1) @@ -208,7 +221,7 @@ func (l *Observer) exportEntry(en *entry, withSamples bool) Entry { } if withSamples { - e.Samples = make([]interface{}, 0, l.maxSamples) + e.Samples = make([]Sample, 0, l.maxSamples) for i := int(l.maxSamples) - 1; i >= 0; i-- { sample := <-en.samples @@ -227,7 +240,7 @@ func (l *Observer) exportEntry(en *entry, withSamples bool) Entry { type Entry struct { Message string Count uint64 - Samples []interface{} + Samples []Sample First time.Time Last time.Time diff --git a/observer_test.go b/observer_test.go index 999c8b2..8ed34c7 100644 --- a/observer_test.go +++ b/observer_test.go @@ -42,6 +42,39 @@ func TestObserver_ObserveMessage(t *testing.T) { assert.NotEmpty(t, entry.Samples) } +func TestObserver_ObserveMessage_filter(t *testing.T) { + o := logz.Observer{} + o.Config.FilterMessage = true + + o.ObserveMessage("test foo123", 123) + o.ObserveMessage("test bar456", 456) + o.ObserveMessage("another test", 789) + + entries := o.GetEntries() + + // Order of entries may be random as they are retrieved from a map. + sort.Slice(entries, func(i, j int) bool { + return entries[i].Message > entries[j].Message + }) + + assert.Equal(t, "test X", entries[0].Message) + assert.Equal(t, uint64(2), entries[0].Count) + assert.NotEmpty(t, entries[0].First) + assert.NotEmpty(t, entries[0].Last) + + assert.Equal(t, "another test", entries[1].Message) + assert.Equal(t, uint64(1), entries[1].Count) + assert.NotEmpty(t, entries[1].First) + assert.NotEmpty(t, entries[1].Last) + + entry := o.Find("test X") + assert.Equal(t, "test X", entry.Message) + assert.Equal(t, uint64(2), entry.Count) + assert.NotEmpty(t, entry.First) + assert.NotEmpty(t, entry.Last) + assert.NotEmpty(t, entry.Samples) +} + func BenchmarkObserver_ObserveMessage(b *testing.B) { o := logz.Observer{} wg := sync.WaitGroup{} @@ -64,3 +97,27 @@ func BenchmarkObserver_ObserveMessage(b *testing.B) { wg.Wait() } + +func BenchmarkObserver_ObserveMessage_filter(b *testing.B) { + o := logz.Observer{} + o.Config.FilterMessage = true + wg := sync.WaitGroup{} + concurrency := 50 + + b.ReportAllocs() + + for i := 0; i < concurrency; i++ { + msg := "foo bar message" + strconv.Itoa(i) + + wg.Add(1) + + go func() { + for i := 0; i < b.N/concurrency; i++ { + o.ObserveMessage(msg, i) + } + wg.Done() + }() + } + + wg.Wait() +}