Syncing recent changes.

docker-compose.testing.yaml

@@ -0,0 +1,13 @@
+
+services:
+  grr-admin-ui:
+    image: ghcr.io/google/grr:testing
+
+  grr-fleetspeak-frontend:
+    image: ghcr.io/google/grr:testing
+
+  grr-worker:
+    image: ghcr.io/google/grr:testing
+
+  grr-client:
+    image: ghcr.io/google/grr:testing

docker-compose.yaml

@@ -0,0 +1,172 @@
+services:
+  db:
+    image: mysql:8.2
+    env_file: docker_config_files/mysql/.env
+    container_name: grr-db
+    hostname: mysql-host
+    command: [
+      --max_allowed_packet=40M,
+      --log_bin_trust_function_creators=1,
+      --innodb_redo_log_capacity=167772160,
+      --innodb_log_file_size=2500M
+    ]
+    restart: always
+    volumes:
+      - ./docker_config_files/mysql/init.sh:/docker-entrypoint-initdb.d/init.sh
+      - db_data:/var/lib/mysql:rw
+    ports:
+      - "3306:3306"
+    expose:
+      - "3306"
+    networks:
+      - server-network
+    healthcheck:
+      test: ["CMD", "mysqladmin", "ping", "-h", "localhost"]
+      timeout: 5s
+      retries: 10
+
+  grr-admin-ui:
+    image: ghcr.io/google/grr:docker-compose-testing
+    container_name: grr-admin-ui
+    hostname: admin-ui
+    restart: always
+    depends_on:
+      db:
+        condition: service_healthy
+    volumes:
+      - ./docker_config_files/server:/configs/
+    ports:
+      - "8000:8000"
+    expose:
+      - "8000"
+    networks:
+      - server-network
+    command:
+      - -component
+      - admin_ui
+      - -config
+      - /configs/grr.server.yaml
+      - --verbose
+
+  grr-fleetspeak-frontend:
+    image: ghcr.io/google/grr:docker-compose-testing
+    container_name: grr-fleetspeak-frontend
+    hostname: grr-fleetspeak-frontend
+    depends_on:
+      db:
+        condition: service_healthy
+    volumes:
+      - ./docker_config_files/server/:/configs/
+    expose:
+      - "11111"
+    restart: always
+    networks:
+      - server-network
+    command:
+      - -component
+      - frontend
+      - -config
+      - /configs/grr.server.yaml
+      - --verbose
+
+  fleetspeak-admin:
+    image: ghcr.io/google/fleetspeak:cl-601031487
+    container_name: fleetspeak-admin
+    hostname: fleetspeak-admin
+    depends_on:
+      db:
+        condition: service_healthy
+    networks:
+      - server-network
+    expose:
+      - "4444"
+    volumes:
+      - ./docker_config_files/server/:/configs/
+    entrypoint: [
+      "server",
+      "-components_config",
+      "/configs/textservices/admin.components.config",
+      "-services_config",
+      "/configs/grr_frontend.service",
+      "-alsologtostderr",
+      "-v",
+      "1000"
+    ]
+
+  fleetspeak-frontend:
+    image: ghcr.io/google/fleetspeak:cl-601031487
+    container_name: fleetspeak-frontend
+    hostname: fleetspeak-frontend
+    depends_on:
+      db:
+        condition: service_healthy
+    networks:
+      - server-network
+    expose:
+      - "4443"
+      - "10000"
+    volumes:
+      - ./docker_config_files/server/:/configs/
+    entrypoint: [
+      "server",
+      "-components_config",
+      "/configs/textservices/frontend.components.config",
+      "-services_config",
+      "/configs/grr_frontend.service",
+      "-alsologtostderr",
+      "-v",
+      "1000"
+    ]
+
+  grr-worker:
+    image: ghcr.io/google/grr:docker-compose-testing
+    container_name: grr-worker
+    volumes:
+      - ./docker_config_files/server/:/configs/
+    hostname: grr-worker
+    depends_on:
+      db:
+        condition: service_healthy
+    restart: always
+    networks:
+      - server-network
+    command:
+      - -component
+      - worker
+      - -config
+      - /configs/grr.server.yaml
+      - --verbose
+
+  grr-client:
+    image: ghcr.io/google/grr:docker-compose-testing
+    container_name: grr-client
+    restart: always
+    depends_on:
+      - db
+      - fleetspeak-frontend
+    volumes:
+      - ./docker_config_files/client/:/configs/
+      # Mount the client_installers folder, to preserve
+      # the repacked templates across restarts.
+      - client_installers:/client_installers
+    networks:
+      - server-network
+    entrypoint: [
+      "/bin/bash",
+      "-c",
+      "/configs/repack_install_client.sh && fleetspeak-client -config /configs/client.config"
+    ]
+    healthcheck:
+      test: |
+        if [[ "$(ps aux | grep grr_response_client.client | grep -v grep | wc -l)" == "0" ]]; then
+          echo "Healthckeck: GRR client process not running"
+          exit 1
+        fi
+      timeout: 10s
+      retries: 10
+
+volumes:
+  db_data:
+  client_installers:
+networks:
+  server-network:

docker_config_files/client/client.config

@@ -0,0 +1,11 @@
+server: "fleetspeak-frontend:4443"
+#  .-.
+# (o.o) WARNING: Publicly stored key. For testing only.
+#  |=|           NEVER reuse in production.
+trusted_certs: "-----BEGIN CERTIFICATE-----\nMIIBhjCCASygAwIBAgIQbZTIkKIjOwVDH5kZDEwz+zAKBggqhkjOPQQDAjAjMSEw\nHwYDVQQDExhGbGVldHNwZWFrIEZsZWV0c3BlYWsgQ0EwHhcNMjQwMTEyMTQ1MTU0\nWhcNMzQwMTA5MTQ1MTU0WjAjMSEwHwYDVQQDExhGbGVldHNwZWFrIEZsZWV0c3Bl\nYWsgQ0EwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAARcKcmCDpGj32sDzRUxBO9E\n9eNg92wGHYYbqHJ5DxqQWVyU8lmE7pPyrZAhVvAAIWQN5pL/MwGRDncOhAciseFW\no0IwQDAOBgNVHQ8BAf8EBAMCAoQwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU\nWl3keEC1M5wmeN/+sUTqrtOVgpIwCgYIKoZIzj0EAwIDSAAwRQIgGMUGaqhSEt4Q\n4SkeTjeU2lr4UpO5wCTRJ80SVENoZUICIQDL31xpZF25HQroy9ApHYuxn8C7oUES\n2RvOjey+9sHQzg==\n-----END CERTIFICATE-----\n"
+client_label: ""
+filesystem_handler: {
+  configuration_directory:"/configs/"
+  state_file:"/tmp/fleetspeak-client.state"
+}
+streaming:true

docker_config_files/client/create_fake_user.sh

@@ -0,0 +1,9 @@
+
+# End-to-end test require a user on a client, which e.g. sets the
+# home directory for file collection.
+
+useradd -m testuser &&
+echo "[7] [01234] [ts/3] [testuser] [pts/3       ] [100.100.10.10       ] [100.100.10.10  ] [Thu Jan 01 00:00:00 1970 UTC]" > wtmp.txt && \
+    utmpdump /var/log/wtmp >> wtmp.txt && \
+    utmpdump --reverse < wtmp.txt > /var/log/wtmp && \
+    utmpdump /var/log/wtmp

docker_config_files/client/grr.client.yaml

@@ -0,0 +1,32 @@
+Client.fleetspeak_enabled: true
+ClientBuilder.fleetspeak_bundled: true
+ClientBuilder.template_dir: /client_templates
+Client.server_urls:
+- fleetspeak-frontend
+Client.foreman_check_frequency: 10  # seconds
+
+Logging.verbose: true
+Logging.engines: file,stderr
+Logging.path: /tmp/grr-client
+Logging.filename: /tmp/grr-client/grr-client.log
+
+#  .-.
+# (o.o) WARNING: Publicly stored key. For testing only.
+#  |=|           NEVER reuse in production.
+Client.executable_signing_public_key: |
+  -----BEGIN PUBLIC KEY-----
+  MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx6YQNUwITzi7l+biDnwv
+  n63Rg3vbfPZexL/0O1XzQw1Z7mFp3uHtnSrkgDmqYIDXwxDXvn8Ck+k8dYt8SZCc
+  Jq4Jd/YkJXaUiM2E/2Y+Gv33ioVaN7QRyVBGRldK7X6a9Z8tEBE8jF3mlzlO2Z16
+  ZCgMLD1I6ZJpHfQFcDGJP7idHY1TVHJ7j9YG8PObi2k9r5E9UBg6DcFD3Rqg5CP/
+  OUtE56B7VW3y8q49c8pw+ZfiQaXd11xMLuMOX9Brlsp/RqFC6wvM1RJc9oR08Bq8
+  je7ZmTVuwGEUR8snL2eqPqhM1UAvelbEF4IVG9E7A043Fhh7qVPxVGqKSkgfwXS0
+  0QIDAQAB
+  -----END PUBLIC KEY-----
+
+Target:Linux:
+  ClientBuilder.fleetspeak_client_config: /configs/client.config
+Target:Windows:
+  ClientBuilder.fleetspeak_client_config: /configs/client.config
+Target:Darwin:
+  ClientBuilder.fleetspeak_client_config: /configs/client.config

docker_config_files/client/repack_install_client.sh

@@ -0,0 +1,38 @@
+#! /bin/bash
+
+# GRR client docker compose initialization script.
+# This script is run when the client is started in the
+# docker-compose stack. It repacks the client using the
+# provided configuration files and installs the resulting
+# debian package if no installers or fleetspeak-client
+# binary are found.
+#
+# This script assumes the client-config files
+# (docker_config_files/client) to be mounted at /configs.
+
+# Template dir is initializes when building the image via
+# the github actions, which also builds the templates.
+TEMPLATE_DIR="/client_templates"
+INSTALLERS_DIR="/client_installers"
+
+
+if [[ -z "$(ls -A ${INSTALLERS_DIR})" ]]
+    then
+        echo "** Repacking clients."
+        grr_client_build repack_multiple \
+            --templates ${TEMPLATE_DIR}/*/*.zip \
+            --repack_configs /configs/grr.client.yaml \
+            --output_dir ${INSTALLERS_DIR}
+    else
+        echo "** Found existing client installers dir, skipping repacking."
+fi
+
+if ! command -v fleetspeak-client &> /dev/null
+    then
+        echo "**Installing Client from debian package."
+        dpkg -i  ${INSTALLERS_DIR}/grr.client/*.deb
+    else
+        echo "** Found fleetspeak-client binary, skipping install."
+fi
+
+echo "** Completed client setup."

docker_config_files/client/textservices/grr_client.service

@@ -0,0 +1,11 @@
+name: "GRR"
+factory: "Daemon"
+config: {
+  [type.googleapis.com/fleetspeak.daemonservice.Config]: {
+    argv: "python"
+    argv: "-m"
+    argv: "grr_response_client.client"
+    argv: "--secondary_configs"
+    argv: "/configs/grr.client.yaml"
+  }
+}

docker_config_files/mysql/.env

@@ -0,0 +1,12 @@
+ADMIN_PASSWORD='admin'
+
+MYSQL_ROOT_PASSWORD='root'
+MYSQL_ROOT_HOST='%'
+
+FLEETSPEAK_DB='fleetspeak'
+FLEETSPEAK_DB_USER='fleetspeak-user'
+FLEETSPEAK_DB_PASSWORD='fleetspeak-password'
+
+GRR_DB='grr'
+GRR_DB_USER='grru'
+GRR_DB_PASSWORD='grrp'

docker_config_files/mysql/init.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -ex
+
+echo "** Creating default DB for GRR and fleetspeak"
+
+mysql -u root -p"$MYSQL_ROOT_PASSWORD" --execute \
+"CREATE USER'$GRR_DB_USER'@'$MYSQL_ROOT_HOST' IDENTIFIED BY '$GRR_DB_PASSWORD';
+CREATE DATABASE $GRR_DB;
+GRANT ALL ON $GRR_DB.* TO '$GRR_DB_USER'@'$MYSQL_ROOT_HOST';
+CREATE USER '$FLEETSPEAK_DB_USER'@'$MYSQL_ROOT_HOST' IDENTIFIED BY '$FLEETSPEAK_DB_PASSWORD';
+CREATE DATABASE $FLEETSPEAK_DB;
+GRANT ALL ON $FLEETSPEAK_DB.* TO '$FLEETSPEAK_DB_USER'@'$MYSQL_ROOT_HOST';
+FLUSH PRIVILEGES;"
+
+echo "** Finished creating DBs and users"

docker_config_files/server/grr.server.yaml

@@ -0,0 +1,67 @@
+AdminUI.csrf_secret_key: KPK,_0a_xY&DTeiaokEdsH1uXGobNIhfrr67BTSLlPPv64_UE0nyn8QsD6
+  nwNZ-C87mwVLkdrc77AKdoz12hxzmYXsBTT1bC#d7
+AdminUI.url: http://admin-ui:8000
+AdminUI.bind: 0.0.0.0
+AdminUI.use_precompiled_js: true
+
+Server.initialized: true
+Server.fleetspeak_enabled: true
+Server.fleetspeak_server: fleetspeak-admin:4444
+Server.fleetspeak_message_listen_address: grr-fleetspeak-frontend:11111
+
+API.DefaultRouter: ApiCallRouterWithoutChecks
+
+Mysql.host: mysql-host
+Mysql.port: 3306
+Mysql.database_name: fleetspeak
+Mysql.database_password: fleetspeak-password
+Mysql.database_username: fleetspeak-user
+Mysql.database: grr
+Mysql.password: grrp
+Mysql.username: grru
+
+Blobstore.implementation: DbBlobStore
+Database.implementation: MysqlDB
+
+#  .-.
+# (o.o) WARNING: Publicly stored key. For testing only.
+#  |=|           NEVER reuse in production.
+Client.executable_signing_public_key: |
+  -----BEGIN PUBLIC KEY-----
+  MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx6YQNUwITzi7l+biDnwv
+  n63Rg3vbfPZexL/0O1XzQw1Z7mFp3uHtnSrkgDmqYIDXwxDXvn8Ck+k8dYt8SZCc
+  Jq4Jd/YkJXaUiM2E/2Y+Gv33ioVaN7QRyVBGRldK7X6a9Z8tEBE8jF3mlzlO2Z16
+  ZCgMLD1I6ZJpHfQFcDGJP7idHY1TVHJ7j9YG8PObi2k9r5E9UBg6DcFD3Rqg5CP/
+  OUtE56B7VW3y8q49c8pw+ZfiQaXd11xMLuMOX9Brlsp/RqFC6wvM1RJc9oR08Bq8
+  je7ZmTVuwGEUR8snL2eqPqhM1UAvelbEF4IVG9E7A043Fhh7qVPxVGqKSkgfwXS0
+  0QIDAQAB
+  -----END PUBLIC KEY-----
+
+PrivateKeys.executable_signing_private_key: |
+  -----BEGIN RSA PRIVATE KEY-----
+  MIIEpAIBAAKCAQEAx6YQNUwITzi7l+biDnwvn63Rg3vbfPZexL/0O1XzQw1Z7mFp
+  3uHtnSrkgDmqYIDXwxDXvn8Ck+k8dYt8SZCcJq4Jd/YkJXaUiM2E/2Y+Gv33ioVa
+  N7QRyVBGRldK7X6a9Z8tEBE8jF3mlzlO2Z16ZCgMLD1I6ZJpHfQFcDGJP7idHY1T
+  VHJ7j9YG8PObi2k9r5E9UBg6DcFD3Rqg5CP/OUtE56B7VW3y8q49c8pw+ZfiQaXd
+  11xMLuMOX9Brlsp/RqFC6wvM1RJc9oR08Bq8je7ZmTVuwGEUR8snL2eqPqhM1UAv
+  elbEF4IVG9E7A043Fhh7qVPxVGqKSkgfwXS00QIDAQABAoIBAQCi51KEWoTRN4aC
+  PMcpcJVfYnH5Kj/+5/yN596957T1elhuFRhQ3+KFgrEuG191HMxxAzY23uXYkNBf
+  TTBdylxPh2R8eOAnnWk3cxLZXrDAT4gDhCoIF6sHq7Obw7CEtvB0CKy5VockNZ5o
+  uD8pe8CZJsA//MWYqHmTEkC5ugG2dlde7FcYHsqVU7NlGHhz5UqPpzrgvdTfnWwj
+  GOd2zL+BuUKbs8ZIVGEDbgtr8ILNN9MMK8nDioIB29SMWP/Jfb2Z7HSRkn2HK7Jf
+  bkv/eTJlOJnAlB5BbDDvQ8vUPgk0j0cMjcapoyoENGmbsgSvydG2O7RyBnkeGmud
+  vEExNZHBAoGBAPgGmD3A07pTYGzd7RytJJZ1u+so4IlWPg2Jp9p0WmP6D6vbB2dl
+  1lIdtzII5hh/wbd2FNZJ5X2iV93gQsffRBGeOJ8b5No91q/EdmCZpFGu7LJQqWVO
+  1+Nft/xW6Kkog811KwYNgQpE241ZRCGoD/KzZpOfb9n+EW+hVYbjOfiZAoGBAM4R
+  S56AFXKHIoZQOgX1drsWr6DKDH8Za7BNsGT1nDi1ROmNZxzx8I9avF4ZSwUMmiXR
+  AXMY69CjqFFwTtWhrZ8UHhl5x7zWAffQdof4jKtdCJ8G4CyYDCZ31Cbi7Gfo4tUP
+  FmLmN59o3l69887y1vgyFnDevSGuCzJ9hJ1LSij5AoGAGKjvMhSd+ISZrblS/erp
+  HFyQVo015fHBMa9iFQJEinQuYrPgRJOHf5qcwEjKN91b8VW4NKYcPyWI/vJxMVYt
+  emL01jz7wAct9UPfUTN1dvmhZwlGDmCMbnrx3BD4CPmSQTdJE8z76311JtSdRYtk
+  KolTxZGwmUf9i8/KpSKqfOECgYB8Kj23TpQdw0FRTwv3RTV6e6vtpXEsMGQMAnPU
+  EY5FOSxB0hscfMeniVPRG0pxy2sieDJ4aL7Go6YrFBHcdaQJI3UTgqaQqR7cdHbH
+  bUNNiixErj7rf95qW2+w0rEB13i+Sm4Bv5gqbGT5D1nWC8ruGDgfYIbzwUwr6ye6
+  I4CW+QKBgQC9xKPizqJoi375rDeLVSc/bN3fidyj+Ti87YQa9sDSyXxSF2uk2HUF
+  xCjMJcqyIOhPSze9wpip6edj8p6N3pvKEMLdFrRJR9Gkv/V9+kJffJbLwyH6Ta/x
+  v89V954580cna0V/lZYpZM/DDdhVv3hCaGIm+uAHA1mYtxzBBTKX3Q==
+  -----END RSA PRIVATE KEY-----