Merge remote-tracking branch 'origin/main' into pkoutsovasilis/root_l…

…inux_integrations

# Conflicts:
#	packages/auditd_manager/changelog.yml
#	packages/fim/changelog.yml
#	packages/system_audit/changelog.yml

.buildkite/pipeline.serverless.yml

@@ -3,7 +3,8 @@
 env:
   SETUP_GVM_VERSION: "v0.5.1"
   LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
-  DOCKER_COMPOSE_VERSION: "v2.17.2"
+  DOCKER_COMPOSE_VERSION: "v2.24.1"
+  DOCKER_VERSION: "false"
   KIND_VERSION: 'v0.20.0'
   K8S_VERSION: 'v1.29.0'
   YQ_VERSION: 'v4.35.2'

.buildkite/pipeline.yml

@@ -3,7 +3,8 @@
 env:
   SETUP_GVM_VERSION: "v0.5.1"
   LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
-  DOCKER_COMPOSE_VERSION: "v2.17.2"
+  DOCKER_COMPOSE_VERSION: "v2.24.1"
+  DOCKER_VERSION: "false"
   KIND_VERSION: 'v0.20.0'
   K8S_VERSION: 'v1.29.0'
   YQ_VERSION: 'v4.35.2'

.buildkite/scripts/common.sh

@@ -135,14 +135,51 @@ with_mage() {
     mage --version
 }
 
-with_docker_compose() {
+with_docker() {
+    echo "--- Setting up the Docker environment..."
+    echo "Current docker client version:"
+    docker version -f json  | jq -r '.Client.Version'
+    echo "Current docekr server version:"
+    docker version -f json  | jq -r '.Server.Version'
+
+    if [[ "${DOCKER_VERSION:-"false"}" == "false" ]]; then
+        echo "Skip docker installation"
+        return
+    fi
+    local ubuntu_version
+    local ubuntu_codename
+    local architecture
+    ubuntu_version="$(lsb_release -rs)" # 20.04
+    ubuntu_codename="$(lsb_release -sc)" # focal
+    architecture=$(dpkg --print-architecture)
+    local debian_version="5:${DOCKER_VERSION}-1~ubuntu.${ubuntu_version}~${ubuntu_codename}"
+
+    sudo sudo mkdir -p /etc/apt/keyrings
+    if [ ! -f /etc/apt/keyrings/docker.gpg ]; then
+        curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+    fi
+    echo "deb [arch=${architecture} signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu ${ubuntu_codename} stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+    sudo apt-get update
+    sudo DEBIAN_FRONTEND=noninteractive apt-get install --allow-downgrades -y "docker-ce=${debian_version}"
+    sudo DEBIAN_FRONTEND=noninteractive apt-get install --allow-downgrades -y "docker-ce-cli=${debian_version}"
+    sudo systemctl start docker
+}
+
+with_docker_compose_plugin() {
+    echo "--- Setting up the Docker compose plugin environment..."
+    if [[ "${DOCKER_COMPOSE_VERSION:-"false"}" == "false" ]]; then
+        echo "Skip docker compose installation (plugin)"
+        return
+    fi
     create_bin_folder
     check_platform_architecture
 
-    echo "--- Setting up the Docker-compose environment..."
-    retry 5 curl -sSL -o "${BIN_FOLDER}/docker-compose" "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-${platform_type_lowercase}-${hw_type}"
-    chmod +x "${BIN_FOLDER}/docker-compose"
-    docker-compose version
+    local DOCKER_CONFIG="$HOME/.docker/cli-plugins"
+    mkdir -p "$DOCKER_CONFIG"
+
+    retry 5 curl -SL -o ${DOCKER_CONFIG}/docker-compose "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-${platform_type_lowercase}-${hw_type}"
+    chmod +x ${DOCKER_CONFIG}/docker-compose
+    docker compose version
 }
 
 with_kubernetes() {

.buildkite/scripts/test_integrations_with_serverless.sh

@@ -39,7 +39,8 @@ add_bin_path
 
 with_yq
 with_mage
-with_docker_compose
+with_docker
+with_docker_compose_plugin
 with_kubernetes
 
 use_elastic_package

.buildkite/scripts/test_one_package.sh

@@ -28,7 +28,8 @@ add_bin_path
 
 with_yq
 with_mage
-with_docker_compose
+with_docker
+with_docker_compose_plugin
 with_kubernetes
 
 use_elastic_package

docs/tips_for_building_integrations.md

@@ -40,7 +40,7 @@ $ ./elastic-package help
 
 4. Set the proper package owner (either Github team or personal account)
 
-   Good candidates for a team: `elastic/integrations`, `elastic/security-external-integrations`
+   Good candidates for a team: `elastic/integrations`, `elastic/security-service-integrations`
 
    Update the `.github/CODEOWNERS` file accordingly.
 

packages/1password/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.26.1"
+  changes:
+    - description: Changed owners
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/8943
 - version: "1.26.0"
   changes:
     - description: Limit request tracer log count to five.

packages/1password/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.0"
 name: 1password
 title: "1Password"
-version: "1.26.0"
+version: "1.26.1"
 description: Collect logs from 1Password with Elastic Agent.
 type: integration
 categories:
@@ -85,5 +85,5 @@ policy_templates:
             multi: false
             default: false
 owner:
-  github: elastic/security-external-integrations
+  github: elastic/security-service-integrations
   type: partner

packages/activemq/_dev/build/docs/README.md

@@ -1,25 +1,104 @@
 # ActiveMQ Integration
 
-This integration periodically fetches metrics from [ActiveMQ](https://activemq.apache.org/) servers. It can parse broker, queue and topic.
-System logs and Audit logs are also collected using this integration.
+## Overview
+
+Apache [ActiveMQ](https://activemq.apache.org) is the most popular open-source, multi-protocol, Java-based message broker. It supports industry-standard protocols, facilitating client choices across various languages and platforms, including JavaScript, C, C++, Python, .Net, and more. ActiveMQ enables seamless integration of multi-platform applications through the widely used AMQP protocol and allows efficient message exchange between web applications using STOMP over WebSockets. Additionally, it supports IoT device management via MQTT and provides flexibility to accommodate any messaging use case, supporting both existing JMS infrastructure and beyond.
+
+Use the ActiveMQ integration to:
+
+- Collect logs related to the audit and ActiveMQ instance and collect metrics related to the broker, queue and topic.
+- Create visualizations to monitor, measure and analyze the usage trend and key data, and derive business insights.
+- Create alerts to reduce the MTTD and also the MTTR by referencing relevant logs when troubleshooting an issue.
+
+## Data streams
+
+The ActiveMQ integration collects logs and metrics data.
+
+Logs help you keep a record of events that happen on your machine. The `Log` data streams collected by ActiveMQ integration are `audit` and `log` so that users can keep track of the username, audit threads, messages, name of the caller issuing the logging requests, logging event etc.
+
+Metrics give you insight into the statistics of the ActiveMQ. The `Metric` data streams collected by the ActiveMQ integration are `broker`, `queue` and `topic` so that the user can monitor and troubleshoot the performance of the ActiveMQ instance.
+
+Data streams:
+- `audit`: Collects information related to the username, audit threads and messages.
+- `broker`: Collects information related to the statistics of enqueued and dequeued messages, consumers, producers and memory usage (broker, store, temp).
+- `log`: Collects information related to the startup and shutdown of the ActiveMQ application server, the deployment of new applications, or the failure of one or more subsystems.
+- `queue`: Collects information related to the statistics of queue name and size, exchanged messages and number of producers and consumers.
+- `topic`: Collects information related to the statistics of exchanged messages, consumers, producers and memory usage.
+
+Note:
+- Users can monitor and see the log inside the ingested documents for ActiveMQ in the `logs-*` index pattern from `Discover`, and for metrics, the index pattern is `metrics-*`.
 
 ## Compatibility
 
-The ActiveMQ datasets were tested with ActiveMQ 5.17.1 or higher (independent from operating system).
+This integration has been tested against ActiveMQ 5.17.1 (independent from the operating system).
+
+## Prerequisites
+
+You need Elasticsearch to store and search your data and Kibana to visualize and manage it.
+You can use our hosted Elasticsearch Service on Elastic Cloud, which is recommended or self-manage the Elastic Stack on your hardware.
+
+## Setup
+
+For step-by-step instructions on how to set up an integration, see the [Getting Started](https://www.elastic.co/guide/en/welcome-to-elastic/current/getting-started-observability.html) guide.
+
+## Supported Log Formats
+
+Here are the supported log format for the Audit logs and ActiveMQ logs in the ActiveMQ instance,
+
+### Audit Logs
+
+```
+%-5p | %m | %t%n
+```
+
+Here is the breakdown of the pattern:
+
+- %-5p: This part represents the log level left-aligned with a width of 5 characters. The - signifies left alignment.
+
+- %m: This part represents the log message.
+
+- %t%n: This part represents the thread name (%t) followed by a newline (%n).
+
+### ActiveMQ Logs
+
+```
+%d | %-5p | %m | %c | %t%n%throwable{full}
+```
+
+Here is the breakdown of the pattern:
+- %d: This part represents the date and time of the log event in the ISO8601 format.
+
+- %-5p: This part represents the log level left-aligned with a width of 5 characters. The - signifies left alignment.
+
+- %m: This part represents the log message.
+
+- %c: This part represents the logger category (class name).
+
+- %t%n: This part represents the thread name (%t) followed by a newline (%n).
+
+- %throwable{full}: This part represents the full stack trace if an exception is attached to the log entry.
+
+## Validation
+
+After the integration is successfully configured, clicking on the Assets tab of the ActiveMQ Integration should display a list of available dashboards. Click on the dashboard available for your configured data stream. It should be populated with the required data.
+
+## Troubleshooting
+
+If `host.ip` appears conflicted under the `log-*` or `metrics-*` data view, this issue can be resolved by [reindexing](https://www.elastic.co/guide/en/elasticsearch/reference/current/use-a-data-stream.html#reindex-with-a-data-stream) the indices of the `Audit`, `Log`, `Broker`, `Queue` and `Topic` data streams.
 
 ## Logs
 
 ### ActiveMQ Logs
 
-Collects the ActiveMQ System logs.
+These logs are System logs of ActiveMQ.
 
 {{event "log"}}
 
 {{fields "log"}}
 
 ### Audit Logs
 
-Audit logs collects the ActiveMQ Audit logs.
+In secured environments, it is required to log every user management action. ActiveMQ implements audit logging, which means that every management action made through JMX or Web Console management interface is logged and available for later inspection.
 
 {{event "audit"}}
 
@@ -29,23 +108,23 @@ Audit logs collects the ActiveMQ Audit logs.
 
 ### Broker Metrics
 
-The server broker stream collects data from the ActiveMQ broker module. 
+ActiveMQ brokers serve as implementations of the Java Messaging Service (JMS), a Java specification facilitating the seamless exchange of data between applications. Metrics provide insights into statistics such as enqueued and dequeued messages, as well as details on consumers, producers, and memory usage (broker, store, temp).
 
 {{event "broker"}}
 
 {{fields "broker"}}
 
 ### Queue Metrics
 
-The server queue stream collects data from the ActiveMQ queue module.
+Queues are FIFO (first-in, first-out) pipelines of messages produced and consumed by brokers and clients. Producers create messages and push them onto these queues. Then, those messages are polled and collected by consumer applications, one message at a time. Metrics show statistics of exchanged messages, consumers, producers and memory usage.
 
 {{event "queue"}}
 
 {{fields "queue"}}
 
 ### Topic Metrics
 
-The server topic stream collects data from the ActiveMQ topic module.
+Topics are subscription-based message broadcast channels. When a producing application sends a message, multiple recipients who are 'subscribed' to that topic receive a broadcast of the message. Metrics show statistics of exchanged messages, consumers, producers and memory usage.
 
 {{event "topic"}}
 

packages/activemq/changelog.yml

@@ -1,4 +1,19 @@
 # newer versions go on top
+- version: 0.16.0
+  changes:
+    - description: Update README to use documentation guidelines.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/8926
+- version: 0.15.0
+  changes:
+    - description: Add metric_type and support tooltip for period.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/8928
+- version: 0.14.2
+  changes:
+    - description: Resolve host.ip field conflict.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/8927
 - version: 0.14.1
   changes:
     - description: Add null and ignore_missing check to handle event.original field.

packages/activemq/data_stream/broker/fields/ecs.yml

@@ -47,3 +47,5 @@
 - external: ecs
   name: host.name
   dimension: true
+- external: ecs
+  name: host.ip

packages/activemq/data_stream/broker/fields/fields.yml

@@ -52,4 +52,5 @@
       description: Broker name.
     - name: producers.count
       type: long
+      metric_type: gauge
       description: Number of message producers active on destinations on the broker.

packages/activemq/data_stream/broker/manifest.yml

@@ -10,6 +10,7 @@ streams:
         required: true
         show_user: true
         default: 10s
+        description: The period for fetching metrics, e.g., every 1s, 1m, 1h
       - name: tags
         type: text
         title: Tags

packages/activemq/data_stream/log/fields/ecs.yml

@@ -18,3 +18,5 @@
   name: message
 - external: ecs
   name: tags
+- external: ecs
+  name: host.ip

packages/activemq/data_stream/queue/fields/ecs.yml

@@ -47,3 +47,5 @@
 - external: ecs
   name: host.name
   dimension: true
+- external: ecs
+  name: host.ip

packages/activemq/data_stream/queue/fields/fields.yml

@@ -3,6 +3,7 @@
   fields:
     - name: consumers.count
       type: long
+      metric_type: gauge
       description: Number of consumers subscribed to this destination.
     - name: mbean
       type: keyword
@@ -63,10 +64,12 @@
       type: keyword
       # Reason to add as a dimension field : Multiple queue name can exist
       dimension: true
-      description: Queue name
+      description: Queue name.
     - name: producers.count
       type: long
+      metric_type: gauge
       description: Number of producers attached to this destination.
     - name: size
       type: long
-      description: Queue size
+      metric_type: gauge
+      description: Queue size.

packages/activemq/data_stream/queue/manifest.yml

@@ -10,6 +10,7 @@ streams:
         required: true
         show_user: true
         default: 10s
+        description: The period for fetching metrics, e.g., every 1s, 1m, 1h
       - name: tags
         type: text
         title: Tags

packages/activemq/data_stream/topic/fields/ecs.yml

@@ -47,3 +47,5 @@
 - external: ecs
   name: host.name
   dimension: true
+- external: ecs
+  name: host.ip

packages/activemq/data_stream/topic/fields/fields.yml

@@ -3,6 +3,7 @@
   fields:
     - name: consumers.count
       type: long
+      metric_type: gauge
       description: Number of consumers subscribed to this destination.
     - name: mbean
       type: keyword

packages/activemq/data_stream/topic/manifest.yml

@@ -10,6 +10,7 @@ streams:
         required: true
         show_user: true
         default: 10s
+        description: The period for fetching metrics, e.g., every 1s, 1m, 1h
       - name: tags
         type: text
         title: Tags