diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9af6a1cac..5be6b86c0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -182,49 +182,6 @@ jobs: path: gcs_upload_dir/ retention-days: 1 - build-push-docker-image: - env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - runs-on: ubuntu-22.04 - needs: - - build-centos - - build-ubuntu - - build-osx - - build-windows - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - name: Download installers from GitHub artifacts - id: download - uses: actions/download-artifact@v4 - with: - path: ./_artifacts - pattern: '*installer*' - - name: Login to GitHub Container registry - if: ${{ github.event_name == 'push' }} - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for Docker - if: ${{ github.event_name == 'push' }} - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - - name: Build and push Docker image - if: ${{ github.event_name == 'push' }} - uses: docker/build-push-action@v5 - with: - context: . - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - build-push-docker-base-image: env: REGISTRY: ghcr.io @@ -276,7 +233,7 @@ jobs: id-token: 'write' runs-on: ubuntu-22.04 needs: - - build-push-docker + - docker-compose-test steps: - uses: actions/checkout@v3 - name: Download installers from GitHub artifacts @@ -318,3 +275,38 @@ jobs: destination: ${{ env.GCS_BUCKET_OPENAPI }} # Omit `path` (e.g. /home/runner/deploy/) in final GCS path. parent: false + + docker-compose-test: + if: ${{ github.event_name == 'push' }} + permissions: + contents: 'read' + id-token: 'write' + runs-on: ubuntu-22.04 + needs: + - build-centos + - build-ubuntu + - build-osx + - build-windows + steps: + - uses: actions/checkout@v3 + - name: Start docker-compose stack + shell: bash + run: | + docker-compose pull --include-deps + docker-compose up -d + - name: Test + shell: bash + run: | + docker build -f ./Dockerfile . -t grr-testing + docker run -d \ + --add-host=host.docker.internal:host-gateway \ + -v $(pwd):/ws \ + -w /ws \ + --entrypoint appveyor/e2e_tests/run_docker_compose_e2e_test.sh \ + grr-testing \ + $(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' grr-linux-client) + - name: Stop docker-compose stack + if: always() + shell: bash + run: | + docker-compose down --volumes diff --git a/appveyor/e2e_tests/docker_compose_client_collection_test.py b/appveyor/e2e_tests/docker_compose_client_collection_test.py new file mode 100644 index 000000000..655a912e3 --- /dev/null +++ b/appveyor/e2e_tests/docker_compose_client_collection_test.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +import time + +from grr_api_client import api + + +grrapi = api.InitHttp(api_endpoint="http://localhost:8000", + auth=("admin", "root")) + +search_result = grrapi.SearchClients() + +result = {} +for client in search_result: + client_id = client.client_id + client_last_seen_at = client.data.last_seen_at + result[client_id] = client_last_seen_at + +assert len(result) == 1 + + +flow_args = grrapi.types.CreateFlowArgs("FileFinder") +flow_args.ClearField("paths") +flow_args.paths.append("/client_templates/*") +flow_args.action.action_type = flow_args.action.DOWNLOAD + +hunt_runner_args = grrapi.types.CreateHuntRunnerArgs() + +hunt = grrapi.CreateHunt(flow_name="FileFinder", flow_args=flow_args, + hunt_runner_args=hunt_runner_args) +hunt = hunt.Start() + +# Wait until results are available. +time.sleep(20) + +found_files = set([f.payload.stat_entry.pathspec.path for f in hunt.ListResults()]) + +assert len(found_files) == 4 +assert found_files == { + '/client_templates/windows-installers', + '/client_templates/osx-installers', + '/client_templates/centos-installers', + '/client_templates/ubuntu-installers' + } diff --git a/appveyor/e2e_tests/run_docker_compose_e2e_test.sh b/appveyor/e2e_tests/run_docker_compose_e2e_test.sh new file mode 100755 index 000000000..29b5cb783 --- /dev/null +++ b/appveyor/e2e_tests/run_docker_compose_e2e_test.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# +# Runs the e2e test in the docker-compose stack. + +set -ex + +readonly GRR_ADMIN_PASS="root" + +readonly CLIENT_IP=${1} + +readonly FLAKY_TESTS_ARR=(\ + TestCheckRunner.runTest \ +) +# Convert array to string (comma-separated). +readonly FLAKY_TESTS="$(IFS=,;echo "${FLAKY_TESTS_ARR[*]}")" + +function fatal() { + >&2 echo "Error: ${1}" + exit 1 +} + +# Install the grr tests +cd /usr/src/grr && pip install -e grr/test && cd - + +# grr_config_updater add_user ${GRR_ADMIN_USER} \ +# --password ${GRR_ADMIN_PASS} \ +# --secondary_configs docker_config_files/server/grr.server.yaml + +grr_end_to_end_tests --verbose \ + --secondary_configs docker_config_files/testing/grr.testing.yaml \ + --api_endpoint "http://host.docker.internal:8000" \ + --api_user "${GRR_ADMIN_USER}" \ + --api_password "${GRR_ADMIN_PASS}" \ + --client_ip "${CLIENT_IP}" \ + --noupload_test_binaries \ + --flow_timeout_secs 240 \ + --flow_results_sla_secs 60 \ + --skip_tests "${FLAKY_TESTS}" \ + 2>&1 | tee e2e.log + +if [[ ! -z "$(cat e2e.log | grep -F '[ FAIL ]')" ]]; then + fatal 'End-to-end tests failed.' +fi + +if [[ -z "$(cat e2e.log | grep -F '[ PASS ]')" ]]; then + fatal "Expected to find at least one passing test in the test log. It is possible no tests actually ran." +fi + + + + diff --git a/docker-compose.yaml b/docker-compose.yaml index f39e2d51b..3561f7a4c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -26,7 +26,9 @@ services: retries: 10 grr-admin-ui: - image: ghcr.io/google/grr:grr-github-actions-docker + build: + dockerfile: Dockerfile + context: . container_name: grr-admin-ui hostname: admin-ui restart: always @@ -36,7 +38,7 @@ services: volumes: - ./docker_config_files/server:/configs/ ports: - - "5555:8000" + - "8000:8000" expose: - "8000" networks: @@ -51,7 +53,9 @@ services: - --verbose grr-fleetspeak-frontend: - image: ghcr.io/google/grr:grr-github-actions-docker + build: + dockerfile: Dockerfile + context: . container_name: grr-fleetspeak-frontend hostname: grr-fleetspeak-frontend depends_on: @@ -125,7 +129,9 @@ services: ] grr-worker: - image: ghcr.io/google/grr:grr-github-actions-docker + build: + dockerfile: Dockerfile + context: . container_name: grr-worker volumes: - ./docker_config_files/server/:/configs/ diff --git a/docker_config_files/testing/grr.testing.yaml b/docker_config_files/testing/grr.testing.yaml new file mode 100644 index 000000000..8b42a1e50 --- /dev/null +++ b/docker_config_files/testing/grr.testing.yaml @@ -0,0 +1,60 @@ +AdminUI.csrf_secret_key: KPK,_0a_xY&DTeiaokEdsH1uXGobNIhfrr67BTSLlPPv64_UE0nyn8QsD6 + nwNZ-C87mwVLkdrc77AKdoz12hxzmYXsBTT1bC#d7 +AdminUI.url: http://admin-ui:8000 +AdminUI.bind: 0.0.0.0 +AdminUI.use_precompiled_js: true + +Server.initialized: true +Server.fleetspeak_enabled: true +Server.fleetspeak_server: fleetspeak-admin:4444 +Server.fleetspeak_message_listen_address: grr-fleetspeak-frontend:11111 + +API.DefaultRouter: ApiCallRouterWithoutChecks + +Mysql.host: host.docker.internal +Mysql.port: 3306 +Mysql.database_name: fleetspeak +Mysql.database_password: fleetspeak-password +Mysql.database_username: fleetspeak-user +Mysql.database: grr +Mysql.password: grrp +Mysql.username: grru + +Blobstore.implementation: DbBlobStore +Database.implementation: MysqlDB + +Client.fleetspeak_enabled: true # TODO remove? +ClientBuilder.fleetspeak_bundled: true # TODO remove? + +Logging.domain: admin-ui +Monitoring.alert_email: grr-monitoring@admin-ui +Monitoring.emergency_access_email: grr-emergency@admin-ui + +PrivateKeys.executable_signing_private_key: | + -----BEGIN RSA PRIVATE KEY----- + MIIEpAIBAAKCAQEAx6YQNUwITzi7l+biDnwvn63Rg3vbfPZexL/0O1XzQw1Z7mFp + 3uHtnSrkgDmqYIDXwxDXvn8Ck+k8dYt8SZCcJq4Jd/YkJXaUiM2E/2Y+Gv33ioVa + N7QRyVBGRldK7X6a9Z8tEBE8jF3mlzlO2Z16ZCgMLD1I6ZJpHfQFcDGJP7idHY1T + VHJ7j9YG8PObi2k9r5E9UBg6DcFD3Rqg5CP/OUtE56B7VW3y8q49c8pw+ZfiQaXd + 11xMLuMOX9Brlsp/RqFC6wvM1RJc9oR08Bq8je7ZmTVuwGEUR8snL2eqPqhM1UAv + elbEF4IVG9E7A043Fhh7qVPxVGqKSkgfwXS00QIDAQABAoIBAQCi51KEWoTRN4aC + PMcpcJVfYnH5Kj/+5/yN596957T1elhuFRhQ3+KFgrEuG191HMxxAzY23uXYkNBf + TTBdylxPh2R8eOAnnWk3cxLZXrDAT4gDhCoIF6sHq7Obw7CEtvB0CKy5VockNZ5o + uD8pe8CZJsA//MWYqHmTEkC5ugG2dlde7FcYHsqVU7NlGHhz5UqPpzrgvdTfnWwj + GOd2zL+BuUKbs8ZIVGEDbgtr8ILNN9MMK8nDioIB29SMWP/Jfb2Z7HSRkn2HK7Jf + bkv/eTJlOJnAlB5BbDDvQ8vUPgk0j0cMjcapoyoENGmbsgSvydG2O7RyBnkeGmud + vEExNZHBAoGBAPgGmD3A07pTYGzd7RytJJZ1u+so4IlWPg2Jp9p0WmP6D6vbB2dl + 1lIdtzII5hh/wbd2FNZJ5X2iV93gQsffRBGeOJ8b5No91q/EdmCZpFGu7LJQqWVO + 1+Nft/xW6Kkog811KwYNgQpE241ZRCGoD/KzZpOfb9n+EW+hVYbjOfiZAoGBAM4R + S56AFXKHIoZQOgX1drsWr6DKDH8Za7BNsGT1nDi1ROmNZxzx8I9avF4ZSwUMmiXR + AXMY69CjqFFwTtWhrZ8UHhl5x7zWAffQdof4jKtdCJ8G4CyYDCZ31Cbi7Gfo4tUP + FmLmN59o3l69887y1vgyFnDevSGuCzJ9hJ1LSij5AoGAGKjvMhSd+ISZrblS/erp + HFyQVo015fHBMa9iFQJEinQuYrPgRJOHf5qcwEjKN91b8VW4NKYcPyWI/vJxMVYt + emL01jz7wAct9UPfUTN1dvmhZwlGDmCMbnrx3BD4CPmSQTdJE8z76311JtSdRYtk + KolTxZGwmUf9i8/KpSKqfOECgYB8Kj23TpQdw0FRTwv3RTV6e6vtpXEsMGQMAnPU + EY5FOSxB0hscfMeniVPRG0pxy2sieDJ4aL7Go6YrFBHcdaQJI3UTgqaQqR7cdHbH + bUNNiixErj7rf95qW2+w0rEB13i+Sm4Bv5gqbGT5D1nWC8ruGDgfYIbzwUwr6ye6 + I4CW+QKBgQC9xKPizqJoi375rDeLVSc/bN3fidyj+Ti87YQa9sDSyXxSF2uk2HUF + xCjMJcqyIOhPSze9wpip6edj8p6N3pvKEMLdFrRJR9Gkv/V9+kJffJbLwyH6Ta/x + v89V954580cna0V/lZYpZM/DDdhVv3hCaGIm+uAHA1mYtxzBBTKX3Q== + -----END RSA PRIVATE KEY----- diff --git a/grr/test/grr_response_test/end_to_end_tests/runner.py b/grr/test/grr_response_test/end_to_end_tests/runner.py index d6aec98c4..9210e070a 100644 --- a/grr/test/grr_response_test/end_to_end_tests/runner.py +++ b/grr/test/grr_response_test/end_to_end_tests/runner.py @@ -259,6 +259,34 @@ def Retry(): return Retry() + def SearchClientByID(self, client_ip): + """Searches for a client with a given IP via the GRR API and return its id. + + Args: + client_ip: Client's IP. + + Returns: + The IP of the client. + """ + start_time = time.time() + + def DeadlineExceeded(): + return time.time() - start_time > self._api_retry_deadline_secs + + @retry.When( + requests.ConnectionError, + lambda _: not DeadlineExceeded(), + opts=retry.Opts( + attempts=sys.maxsize, # Limited by deadline. + init_delay=datetime.timedelta(seconds=self._api_retry_period_secs), + ), + ) + def Retry(): + clients = list(self._grr_api.SearchClients(f"ip:{client_ip}")) + return clients[0].client_id if clients else None + + return Retry() + def _GetApplicableTests(self, client): """Returns all e2e test methods that should be run against the client.""" applicable_tests = {} diff --git a/grr/test/grr_response_test/run_end_to_end_tests.py b/grr/test/grr_response_test/run_end_to_end_tests.py index 17a315543..f310e4d4a 100644 --- a/grr/test/grr_response_test/run_end_to_end_tests.py +++ b/grr/test/grr_response_test/run_end_to_end_tests.py @@ -27,6 +27,10 @@ _CLIENT_ID = flags.DEFINE_string("client_id", "", "Id for client to run tests against.") +_CLIENT_IP = flags.DEFINE_string( + "client_ip", "", "IP address of a client to run tests against." +) + _RUN_ONLY_TESTS = flags.DEFINE_list( "run_only_tests", [], "(Optional) comma-separated list of tests to run (skipping all others).") @@ -84,7 +88,14 @@ def main(argv): upload_test_binaries=_UPLOAD_TEST_BINARIES.value) test_runner.Initialize() - results, _ = test_runner.RunTestsAgainstClient(_CLIENT_ID.value) + client_id = _CLIENT_ID.value + if _CLIENT_IP.value and not _CLIENT_ID.value: + client_id = test_runner.SearchClientByID(_CLIENT_IP.value) + + if not client_id: + sys.exit(1) + + results, _ = test_runner.RunTestsAgainstClient(client_id) # Exit with a non-0 error code if one of the tests failed. for r in results.values(): if r.errors or r.failures: