Dataplane: Add gRPC health check

config/dataplane/dataplane.yaml

@@ -28,3 +28,25 @@ spec:
         - name: RUST_LOG
           value: debug
         imagePullPolicy: IfNotPresent
+        # The gRPC API has a slow startup time, so this probe helps to provide some
+        # grace while starting up to avoid unnecessary kills.
+        #
+        # TODO: When we complete https://github.com/kubernetes-sigs/blixt/issues/173
+        # if we decide that we intend to keep the gRPC API around long term, we should
+        # take some time to see if we can clean up and improve the start time overall.
+        startupProbe:
+          grpc:
+            port: 9874
+          failureThreshold: 30
+          periodSeconds: 10
+        livenessProbe:
+          grpc:
+            port: 9874
+          initialDelaySeconds: 5
+          periodSeconds: 5
+        readinessProbe:
+          grpc:
+            port: 9874
+          initialDelaySeconds: 5
+          periodSeconds: 5
+

dataplane/api-server/Cargo.toml

@@ -7,6 +7,7 @@ publish = false
 [dependencies]
 prost = "0.11.9"
 tonic = "0.9.2"
+tonic-health = "0.9.2"
 anyhow = "1"
 log = "0.4"
 aya = { version = ">=0.11", features=["async_tokio"] }

dataplane/api-server/src/lib.rs

@@ -24,9 +24,12 @@ pub async fn start(
     gateway_indexes_map: HashMap<MapData, BackendKey, u16>,
     tcp_conns_map: HashMap<MapData, ClientKey, LoadBalancerMapping>,
 ) -> Result<(), Error> {
+    let (_, health_service) = tonic_health::server::health_reporter();
+
     let server = server::BackendService::new(backends_map, gateway_indexes_map, tcp_conns_map);
     // TODO: mTLS https://github.com/Kong/blixt/issues/50
     Server::builder()
+        .add_service(health_service)
         .add_service(BackendsServer::new(server))
         .serve(SocketAddrV4::new(addr, port).into())
         .await?;

test/integration/suite_test.go

@@ -36,6 +36,7 @@ import (
 	"sigs.k8s.io/gateway-api/pkg/client/clientset/versioned"
 
 	testutils "github.com/kubernetes-sigs/blixt/internal/test/utils"
+	"github.com/kubernetes-sigs/blixt/pkg/vars"
 )
 
 var (
@@ -181,6 +182,9 @@ func TestMain(m *testing.M) {
 	fmt.Println("INFO: waiting for Blixt component readiness")
 	exitOnErr(testutils.WaitForBlixtReadiness(ctx, env))
 
+	fmt.Println("INFO: waiting for Dataplane readiness")
+	exitOnErr(waitForDataplaneReadiness(ctx, env))
+
 	exit := m.Run()
 
 	exitOnErr(runCleanup(mainCleanupKey))
@@ -309,3 +313,50 @@ func waitForBpfdConfigDelete(ctx context.Context, env environments.Environment)
 		}
 	}
 }
+
+func waitForDataplaneReadiness(ctx context.Context, env environments.Environment) error {
+	for {
+		select {
+		case <-ctx.Done():
+			if err := ctx.Err(); err != nil {
+				return fmt.Errorf("context completed while waiting for dataplane readiness, and an error occurred: %w", err)
+			}
+			return fmt.Errorf("context completed while waiting for dataplane readiness")
+		default:
+			dataplanes, err := env.Cluster().Client().CoreV1().Pods(vars.DefaultNamespace).
+				List(ctx, metav1.ListOptions{LabelSelector: fmt.Sprintf("component=%s", vars.DefaultDataPlaneComponentLabel)})
+			if err != nil {
+				return fmt.Errorf("failed to fetch dataplane pod list: %w", err)
+			}
+
+			if len(dataplanes.Items) == 0 {
+				return fmt.Errorf("no dataplanes found in namespace %s", vars.DefaultNamespace)
+			}
+
+			// Check whether dataplane is set readiness probe.
+			for _, container := range dataplanes.Items[0].Spec.Containers {
+				if container.Name == "dataplane" && container.ReadinessProbe == nil {
+					return fmt.Errorf("found a dataplane container which doesn't have readiness probe")
+				}
+			}
+
+			// Check all pods' readiness
+			var ready int
+			for _, pod := range dataplanes.Items {
+				for _, status := range pod.Status.ContainerStatuses {
+					if status.Name == "dataplane" && status.Ready {
+						ready++
+					}
+				}
+			}
+
+			if ready != len(dataplanes.Items) {
+				fmt.Printf("%d dataplanes not yet ready\n", len(dataplanes.Items)-ready)
+				time.Sleep(time.Second) // small rest from hitting the API over and over again
+				break
+			}
+
+			return nil
+		}
+	}
+}