diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 8f124fea4ae8..84f725981fe8 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -5,6 +5,3 @@ env:
 steps:
   - label: "Generate dynamic pipeline"
     command: ".buildkite/scripts/generate_pipeline.sh"
-    agents:
-      provider: "gcp"
-      image: "family/platform-ingest-beats-ubuntu-2204"
diff --git a/.buildkite/pytest.ini b/.buildkite/pytest.ini
new file mode 100644
index 000000000000..3eff7473d9f0
--- /dev/null
+++ b/.buildkite/pytest.ini
@@ -0,0 +1,11 @@
+[pytest]
+junit_family=xunit1
+
+addopts = --strict-markers
+markers =
+    load: Load tests
+    tag(name): Tag tests with Go-like semantics
+
+# Ignore setup and teardown for the timeout
+#timeout_func_only = True
+
diff --git a/.buildkite/scripts/generate_pipeline.sh b/.buildkite/scripts/generate_pipeline.sh
index 0d9703e9660d..c5627aaf0073 100755
--- a/.buildkite/scripts/generate_pipeline.sh
+++ b/.buildkite/scripts/generate_pipeline.sh
@@ -2,7 +2,12 @@
 set -euo pipefail
 
 echo "~~~ Install dependencies"
-pip3 install --quiet "ruamel.yaml<0.18.0"
+python3 -mpip install --quiet "ruamel.yaml<0.18.0"
+# temporary solution until we have this into a base container
+curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o /usr/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
+chmod a+x /usr/bin/yq
+
+.buildkite/scripts/run_dynamic_pipeline_tests.sh
 
 echo "+++ Run pipeline generator in dry-run mode"
 python3 .buildkite/pipeline.py | yq .
diff --git a/.buildkite/scripts/run_dynamic_pipeline_tests.sh b/.buildkite/scripts/run_dynamic_pipeline_tests.sh
new file mode 100755
index 000000000000..8eb72d7a96b1
--- /dev/null
+++ b/.buildkite/scripts/run_dynamic_pipeline_tests.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# Run tests for the dynamic pipeline generator only if it's a PR and related files have been changed
+# this will allow us to fail fast, if e.g. a PR has broken the generator
+
+set -euo pipefail
+
+are_paths_changed() {
+  local patterns=("${@}")
+  local changelist=()
+  for pattern in "${patterns[@]}"; do
+    changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern"))
+    if [ "${#changed_files[@]}" -gt 0 ]; then
+      changelist+=("${changed_files[@]}")
+    fi
+  done
+
+  if [ "${#changelist[@]}" -gt 0 ]; then
+    echo "Files changed:"
+    echo "${changelist[*]}"
+    return 0
+  else
+    echo "No files changed within specified changeset:"
+    echo "${patterns[*]}"
+    return 1
+  fi
+}
+
+pipeline_generator_changeset=(
+  "^.buildkite/pipeline.py"
+  "^*/buildkite.yml"
+)
+
+if ! are_paths_changed "${pipeline_generator_changeset[@]}" || [[ "${BUILDKITE_PULL_REQUEST}" == "false" ]]; then
+  echo "~~~ Skipping pipeline generator tests"
+  exit
+fi
+
+echo "~~~ Running pipeline generator tests"
+
+python3 -mpip install --quiet "pytest"
+pushd .buildkite
+pytest .
+popd
diff --git a/.buildkite/test_pipeline.py b/.buildkite/test_pipeline.py
new file mode 100644
index 000000000000..75fd949ccc8e
--- /dev/null
+++ b/.buildkite/test_pipeline.py
@@ -0,0 +1,71 @@
+import os
+
+import pytest
+import pipeline
+
+
+@pytest.fixture
+def ubuntu2204_aws_agent():
+    return {
+        "command": "fake-cmd",
+        "platform": "platform-ingest-beats-ubuntu-2204-aarch64",
+        "provider": "aws"
+    }
+
+
+@pytest.fixture()
+def fake_simple_group():
+    return {
+        "unitTest": {
+            "command": "fake-cmd",
+            "platform": "family/platform-ingest-beats-ubuntu-2204",
+        },
+        "integrationTest": {
+            "command": "fake-integration",
+            "platform": "family/platform-ingest-beats-ubuntu-2204",
+            "env": {
+                "FOO": "BAR",
+            },
+        },
+    }
+
+
+def test_fetch_stage(ubuntu2204_aws_agent):
+    step = pipeline.fetch_stage("test", ubuntu2204_aws_agent, "fake", "fake-category")
+    assert step.create_entity() == {
+        "label": "fake test",
+        "command": ["cd fake", "fake-cmd"],
+        "notify": [
+            {
+                "github_commit_status": {
+                    "context": "Fake: test",
+                }
+            }
+        ],
+        "agents": {
+            "provider": "aws",
+            "imagePrefix": "platform-ingest-beats-ubuntu-2204-aarch64",
+            "instanceType": "t4g.large",
+        },
+        "artifact_paths": [
+            "fake/build/*.xml",
+            "fake/build/*.json",
+        ],
+    }
+
+
+def test_fetch_group(fake_simple_group):
+    group = pipeline.fetch_group(fake_simple_group, "fake-project", "testing")
+    assert len(group.steps) == 2
+    for step in group.steps:
+        assert "testing" == step.category
+        assert "gcp" == step.agent.provider
+
+    assert group.steps[1].env.get("FOO") == "BAR"
+
+
+def test_is_pr():
+    os.environ["BUILDKITE_PULL_REQUEST"] = "1234"
+    assert pipeline.is_pr() is True
+    os.environ["BUILDKITE_PULL_REQUEST"] = "false"
+    assert pipeline.is_pr() is False