Use journald app logs in opentelemetry collector (#896)

mattermost · Mar 5, 2025 · 316edc9 · 316edc9
1 parent 701a1b4
commit 316edc9
Show file tree

Hide file tree

Showing 7 changed files with 106 additions and 100 deletions.
diff --git a/deployment/terraform/assets/bindata.go b/deployment/terraform/assets/bindata.go
diff --git a/deployment/terraform/assets/provisioners/agent.sh b/deployment/terraform/assets/provisioners/agent.sh
@@ -3,26 +3,28 @@
 set -euo pipefail
 
 # Wait for boot to be finished (e.g. networking to be up).
-while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done
+while [ ! -f /var/lib/cloud/instance/boot-finished ]; do
+	echo 'Waiting for cloud-init...'
+	sleep 1
+done
 
 # Retry loop (up to 3 times)
 n=0
-until [ "$n" -ge 3 ]
-do
-        # Note: commands below are expected to be either idempotent or generally safe to be run more than once.
-        echo "Attempt ${n}"
-        sudo apt-get -y update && \
-        sudo apt-get install -y prometheus-node-exporter && \
-        sudo apt-get install -y numactl linux-tools-aws && \
-        # Install OpenTelemetry collector, using ubuntu user to avoid permission issues
-        wget https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.110.0/otelcol-contrib_0.110.0_linux_amd64.deb && \
-        sudo dpkg -i otelcol-contrib_0.110.0_linux_amd64.deb && \
-        sudo sed -i 's/User=.*/User=ubuntu/g' /lib/systemd/system/otelcol-contrib.service && \
-        sudo sed -i 's/Group=.*/Group=ubuntu/g' /lib/systemd/system/otelcol-contrib.service && \
-        sudo systemctl daemon-reload && sudo systemctl restart otelcol-contrib && \
-        exit 0
-   n=$((n+1)) 
-   sleep 2
+until [ "$n" -ge 3 ]; do
+	# Note: commands below are expected to be either idempotent or generally safe to be run more than once.
+	echo "Attempt ${n}"
+	sudo apt-get -y update &&
+		sudo apt-get install -y prometheus-node-exporter &&
+		sudo apt-get install -y numactl linux-tools-aws &&
+		# Install OpenTelemetry collector, using ubuntu user to avoid permission issues
+		wget https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.120.0/otelcol-contrib_0.120.0_linux_amd64.deb &&
+		sudo dpkg -i otelcol-contrib_0.120.0_linux_amd64.deb &&
+		sudo sed -i 's/User=.*/User=ubuntu/g' /lib/systemd/system/otelcol-contrib.service &&
+		sudo sed -i 's/Group=.*/Group=ubuntu/g' /lib/systemd/system/otelcol-contrib.service &&
+		sudo systemctl daemon-reload && sudo systemctl restart otelcol-contrib &&
+		exit 0
+	n=$((n + 1))
+	sleep 2
 done
 
 echo 'All retry attempts have failed, exiting' && exit 1
diff --git a/deployment/terraform/assets/provisioners/app.sh b/deployment/terraform/assets/provisioners/app.sh
@@ -3,34 +3,36 @@
 set -euo pipefail
 
 # Wait for boot to be finished (e.g. networking to be up).
-while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done
+while [ ! -f /var/lib/cloud/instance/boot-finished ]; do
+	echo 'Waiting for cloud-init...'
+	sleep 1
+done
 
 # Retry loop (up to 3 times)
 n=0
-until [ "$n" -ge 3 ]
-do
-      # Note: commands below are expected to be either idempotent or generally safe to be run more than once.
-      echo "Attempt ${n}"
-      echo 'tcp_bbr' | sudo tee -a /etc/modules && \
-      sudo modprobe tcp_bbr && \
-      wget -qO - https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | sudo tee /usr/share/keyrings/postgres-archive-keyring.gpg && \
-      sudo sh -c 'echo "deb [signed-by=/usr/share/keyrings/postgres-archive-keyring.gpg] http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' && \
-      sudo apt-get -y update && \
-      sudo apt-get install -y mysql-client-8.0 && \
-      sudo apt-get install -y postgresql-client-14 && \
-      sudo apt-get install -y prometheus-node-exporter && \
-      sudo apt-get install -y numactl linux-tools-aws && \
-      wget https://github.com/streamer45/netpeek/releases/download/v0.1.4/netpeek-v0.1.4 && \
-      sudo mv netpeek-v* /usr/local/bin/netpeek && sudo chmod +x /usr/local/bin/netpeek && \
-      # Install OpenTelemetry collector, using ubuntu user to avoid permission issues
-      wget https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.110.0/otelcol-contrib_0.110.0_linux_amd64.deb && \
-      sudo dpkg -i otelcol-contrib_0.110.0_linux_amd64.deb && \
-      sudo sed -i 's/User=.*/User=ubuntu/g' /lib/systemd/system/otelcol-contrib.service && \
-      sudo sed -i 's/Group=.*/Group=ubuntu/g' /lib/systemd/system/otelcol-contrib.service && \
-      sudo systemctl daemon-reload && sudo systemctl restart otelcol-contrib && \
-      exit 0
-   n=$((n+1)) 
-   sleep 2
+until [ "$n" -ge 3 ]; do
+	# Note: commands below are expected to be either idempotent or generally safe to be run more than once.
+	echo "Attempt ${n}"
+	echo 'tcp_bbr' | sudo tee -a /etc/modules &&
+		sudo modprobe tcp_bbr &&
+		wget -qO - https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | sudo tee /usr/share/keyrings/postgres-archive-keyring.gpg &&
+		sudo sh -c 'echo "deb [signed-by=/usr/share/keyrings/postgres-archive-keyring.gpg] http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' &&
+		sudo apt-get -y update &&
+		sudo apt-get install -y mysql-client-8.0 &&
+		sudo apt-get install -y postgresql-client-14 &&
+		sudo apt-get install -y prometheus-node-exporter &&
+		sudo apt-get install -y numactl linux-tools-aws &&
+		wget https://github.com/streamer45/netpeek/releases/download/v0.1.4/netpeek-v0.1.4 &&
+		sudo mv netpeek-v* /usr/local/bin/netpeek && sudo chmod +x /usr/local/bin/netpeek &&
+		# Install OpenTelemetry collector, using ubuntu user to avoid permission issues
+		wget https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.120.0/otelcol-contrib_0.120.0_linux_amd64.deb &&
+		sudo dpkg -i otelcol-contrib_0.120.0_linux_amd64.deb &&
+		sudo sed -i 's/User=.*/User=ubuntu/g' /lib/systemd/system/otelcol-contrib.service &&
+		sudo sed -i 's/Group=.*/Group=ubuntu/g' /lib/systemd/system/otelcol-contrib.service &&
+		sudo systemctl daemon-reload && sudo systemctl restart otelcol-contrib &&
+		exit 0
+	n=$((n + 1))
+	sleep 2
 done
 
 echo 'All retry attempts have failed, exiting' && exit 1
diff --git a/deployment/terraform/assets/provisioners/proxy.sh b/deployment/terraform/assets/provisioners/proxy.sh
@@ -3,39 +3,41 @@
 set -euo pipefail
 
 # Wait for boot to be finished (e.g. networking to be up).
-while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo 'Waiting for cloud-init...'; sleep 1; done
+while [ ! -f /var/lib/cloud/instance/boot-finished ]; do
+	echo 'Waiting for cloud-init...'
+	sleep 1
+done
 
 # Retry loop (up to 3 times)
 n=0
-until [ "$n" -ge 3 ]
-do
-      # Note: commands below are expected to be either idempotent or generally safe to be run more than once.
-      echo "Attempt ${n}"
-      echo 'tcp_bbr' | sudo tee -a /etc/modules && \
-      sudo modprobe tcp_bbr && \
-      wget -qO - https://nginx.org/keys/nginx_signing.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/nginx.gpg && \
-      sudo sh -c 'echo "deb [arch=amd64] http://nginx.org/packages/mainline/ubuntu/ $(lsb_release -cs) nginx" > /etc/apt/sources.list.d/nginx.list' && \
-      sudo sh -c 'echo "deb-src http://nginx.org/packages/mainline/ubuntu/ $(lsb_release -cs) nginx" >> /etc/apt/sources.list.d/nginx.list' && \
-      sudo apt-get -y update && \
-      sudo apt-get install -y nginx && \
-      sudo apt-get install -y prometheus-node-exporter && \
-      sudo apt-get install -y numactl linux-tools-aws && \
-      sudo systemctl daemon-reload && \
-      sudo systemctl enable nginx && \
-      sudo mkdir -p /etc/nginx/snippets && \
-      sudo mkdir -p /etc/nginx/sites-available && \
-      sudo mkdir -p /etc/nginx/sites-enabled && \
-      sudo rm -f /etc/nginx/sites-enabled/default && \
-      sudo ln -fs /etc/nginx/sites-available/mattermost /etc/nginx/sites-enabled/mattermost && \
-      # Install OpenTelemetry collector, using ubuntu user to avoid permission issues
-      wget https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.110.0/otelcol-contrib_0.110.0_linux_amd64.deb && \
-      sudo dpkg -i otelcol-contrib_0.110.0_linux_amd64.deb && \
-      sudo sed -i 's/User=.*/User=ubuntu/g' /lib/systemd/system/otelcol-contrib.service && \
-      sudo sed -i 's/Group=.*/Group=ubuntu/g' /lib/systemd/system/otelcol-contrib.service && \
-      sudo systemctl daemon-reload && sudo systemctl restart otelcol-contrib && \
-      exit 0
-   n=$((n+1)) 
-   sleep 2
+until [ "$n" -ge 3 ]; do
+	# Note: commands below are expected to be either idempotent or generally safe to be run more than once.
+	echo "Attempt ${n}"
+	echo 'tcp_bbr' | sudo tee -a /etc/modules &&
+		sudo modprobe tcp_bbr &&
+		wget -qO - https://nginx.org/keys/nginx_signing.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/nginx.gpg &&
+		sudo sh -c 'echo "deb [arch=amd64] http://nginx.org/packages/mainline/ubuntu/ $(lsb_release -cs) nginx" > /etc/apt/sources.list.d/nginx.list' &&
+		sudo sh -c 'echo "deb-src http://nginx.org/packages/mainline/ubuntu/ $(lsb_release -cs) nginx" >> /etc/apt/sources.list.d/nginx.list' &&
+		sudo apt-get -y update &&
+		sudo apt-get install -y nginx &&
+		sudo apt-get install -y prometheus-node-exporter &&
+		sudo apt-get install -y numactl linux-tools-aws &&
+		sudo systemctl daemon-reload &&
+		sudo systemctl enable nginx &&
+		sudo mkdir -p /etc/nginx/snippets &&
+		sudo mkdir -p /etc/nginx/sites-available &&
+		sudo mkdir -p /etc/nginx/sites-enabled &&
+		sudo rm -f /etc/nginx/sites-enabled/default &&
+		sudo ln -fs /etc/nginx/sites-available/mattermost /etc/nginx/sites-enabled/mattermost &&
+		# Install OpenTelemetry collector, using ubuntu user to avoid permission issues
+		wget https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.120.0/otelcol-contrib_0.120.0_linux_amd64.deb &&
+		sudo dpkg -i otelcol-contrib_0.120.0_linux_amd64.deb &&
+		sudo sed -i 's/User=.*/User=ubuntu/g' /lib/systemd/system/otelcol-contrib.service &&
+		sudo sed -i 's/Group=.*/Group=ubuntu/g' /lib/systemd/system/otelcol-contrib.service &&
+		sudo systemctl daemon-reload && sudo systemctl restart otelcol-contrib &&
+		exit 0
+	n=$((n + 1))
+	sleep 2
 done
 
 echo 'All retry attempts have failed, exiting' && exit 1
diff --git a/deployment/terraform/create.go b/deployment/terraform/create.go
@@ -268,7 +268,6 @@ func (t *Terraform) Create(extAgent *ssh.ExtAgent, initData bool) error {
 		if t.output.HasElasticSearch() {
 			mlog.Info("Setting up Elasticsearch")
 			err := t.setupElasticSearchServer(extAgent, t.output.Instances[0].GetConnectionIP())
-
 			if err != nil {
 				errorsChan <- fmt.Errorf("unable to setup Elasticsearch server: %w", err)
 				return
@@ -604,7 +603,6 @@ func (t *Terraform) setupElasticSearchServer(extAgent *ssh.ExtAgent, ip string)
 		if !strings.HasPrefix(i, ".") {
 			snapshotIndices = append(snapshotIndices, i)
 		}
-
 	}
 	mlog.Debug("Indices in the snapshot to be restored", mlog.Array("snapshot indices", snapshotIndices))
 
@@ -1014,13 +1012,13 @@ func (t *Terraform) updateAppConfig(siteURL string, sshc *ssh.Client, jobServerE
 	}
 
 	cfg.LogSettings.EnableConsole = model.NewPointer(true)
-	cfg.LogSettings.ConsoleLevel = model.NewPointer("ERROR")
+	cfg.LogSettings.ConsoleLevel = model.NewPointer("WARN")
 	cfg.LogSettings.EnableFile = model.NewPointer(true)
 	cfg.LogSettings.FileLevel = model.NewPointer("WARN")
 	cfg.LogSettings.EnableSentry = model.NewPointer(false)
 
 	cfg.NotificationLogSettings.EnableConsole = model.NewPointer(true)
-	cfg.NotificationLogSettings.ConsoleLevel = model.NewPointer("ERROR")
+	cfg.NotificationLogSettings.ConsoleLevel = model.NewPointer("WARN")
 	cfg.NotificationLogSettings.EnableFile = model.NewPointer(true)
 	cfg.NotificationLogSettings.FileLevel = model.NewPointer("WARN")
 

diff --git a/deployment/terraform/strings.go b/deployment/terraform/strings.go
@@ -549,7 +549,12 @@ RestartSec=1
 WantedBy=multi-user.target
 `
 
-const otelcolOperatorAppAgent = `
+const otelcolOperatorApp = `
+      - type: move
+        from: body.MESSAGE
+        to: body`
+
+const otelcolOperatorAgent = `
       - type: json_parser
         timestamp:
           parse_from: attributes.timestamp
@@ -583,7 +588,9 @@ receivers:
         endpoint: 0.0.0.0:4318
 {{range .Receivers}}
   {{.Name}}:
+    {{- if .IncludeFiles }}
     include: [ {{.IncludeFiles}} ]
+    {{- end }}
     resource:
       service.name: "{{.ServiceName}}"
       service.instance.id: "{{.ServiceInstanceId}}"
@@ -616,15 +623,15 @@ func renderAgentOtelcolConfig(instanceName string, metricsIP string) (string, er
 		IncludeFiles:      "/home/ubuntu/mattermost-load-test-ng/ltagent.log",
 		ServiceName:       "agent",
 		ServiceInstanceId: instanceName,
-		Operator:          otelcolOperatorAppAgent,
+		Operator:          otelcolOperatorAgent,
 	}
 
 	coordinatorReceiver := otelcolReceiver{
 		Name:              "filelog/coordinator",
 		IncludeFiles:      "/home/ubuntu/mattermost-load-test-ng/ltcoordinator.log",
 		ServiceName:       "coordinator",
 		ServiceInstanceId: instanceName,
-		Operator:          otelcolOperatorAppAgent,
+		Operator:          otelcolOperatorAgent,
 	}
 
 	otelcolConfig, err := fillConfigTemplate(otelcolConfigTmpl, map[string]any{
@@ -668,11 +675,10 @@ func renderProxyOtelcolConfig(instanceName string, metricsIP string) (string, er
 
 func renderAppOtelcolConfig(instanceName string, metricsIP string) (string, error) {
 	appReceiver := otelcolReceiver{
-		Name:              "filelog/app",
-		IncludeFiles:      "/opt/mattermost/logs/mattermost.log",
+		Name:              "journald/app",
 		ServiceName:       "app",
 		ServiceInstanceId: instanceName,
-		Operator:          otelcolOperatorAppAgent,
+		Operator:          otelcolOperatorApp,
 	}
 
 	otelcolConfig, err := fillConfigTemplate(otelcolConfigTmpl, map[string]any{

diff --git a/deployment/terraform/strings_test.go b/deployment/terraform/strings_test.go
@@ -62,18 +62,14 @@ receivers:
       http:
         endpoint: 0.0.0.0:4318
 
-  filelog/app:
-    include: [ /opt/mattermost/logs/mattermost.log ]
+  journald/app:
     resource:
       service.name: "app"
       service.instance.id: "instance-name"
     operators:
-      - type: json_parser
-        timestamp:
-          parse_from: attributes.timestamp
-          layout: '%Y-%m-%d %H:%M:%S.%L Z'
-        severity:
-          parse_from: attributes.level
+      - type: move
+        from: body.MESSAGE
+        to: body
 
 exporters:
   otlphttp/logs:
@@ -84,7 +80,7 @@ exporters:
 service:
   pipelines:
     logs:
-      receivers: [filelog/app,]
+      receivers: [journald/app,]
       exporters: [otlphttp/logs]
 `