dev-env: wait until the lb exclusion is effective

We saw a flake where the first test fails, probably because the lb
exclusion is not effective yet. So here we:

- move the exclusion from the e2e command to the dev-env command, so
  it will work when testing / tinkering too
- active wait until all the labels are removed

Signed-off-by: Federico Paolinelli <fpaoline@redhat.com>

.github/workflows/ci.yaml

@@ -326,6 +326,10 @@ jobs:
           sed -i 's/native/frr/g' bin/metallb-operator.yaml
           kubectl apply -f bin/metallb-operator.yaml
 
+      - name: Remove the exclude from external lb label from the master node
+        run: |
+          env "PATH=$PATH" inv remove-lb-exclusion-from-nodes
+
       - name: Ensure MetalLB operator is ready
         run: |
           COUNT=0

tasks.py

@@ -463,6 +463,10 @@ def dev_env(ctx, architecture="amd64", name="kind", protocol=None, frr_volume_di
 
             run("{} apply -f {}".format(kubectl_path, manifest_file), echo=True)
 
+    # Kind puts the remove exclusions annotation on the master node while
+    # the e2e tests expect master to be serviceable, so we remove the annotations
+    remove_lb_exclusion_from_nodes(ctx)
+
     if protocol == "bgp":
         print("Configuring MetalLB with a BGP test environment")
         bgp_dev_env(ip_family, frr_volume_dir)
@@ -845,14 +849,8 @@ def e2etest(ctx, name="kind", export=None, kubeconfig=None, system_namespaces="k
     for ns in namespaces:
         run("{} -n {} wait --for=condition=Ready --all pods --timeout 300s".format(kubectl_path, ns), hide=True)
 
-    """
-       the control-plane node will be labeled "node.kubernetes.io/exclude-from-external-load-balancers" by default 
-       when the cluster is created. and when https://github.com/metallb/metallb/pull/2073 was merged in, which will 
-       affect the currently e2e tests. In order to code minimal changes, we should remove the label here.
-    """
-    nodes = dont_exclude_from_lb(name)
-
     if node_nics == "kind":
+        nodes = run("{} get nodes -o jsonpath={{.items[*].metadata.name}}".format(kubectl_path), hide=True).stdout.strip().split()
         node_nics = _get_node_nics(nodes[0])
 
     if local_nics == "kind":
@@ -889,12 +887,19 @@ def e2etest(ctx, name="kind", export=None, kubeconfig=None, system_namespaces="k
     if testrun.failed:
         raise Exit(message="E2E tests failed", code=testrun.return_code)
 
-def dont_exclude_from_lb(name):
-    nodes = run("kind get nodes --name {name}".format(name=name)).stdout.strip().split("\n")
+@task
+def remove_lb_exclusion_from_nodes(ctx):
+    _fetch_kubectl()
+    nodes = run("{} get nodes -o jsonpath={{.items[*].metadata.name}}".format(kubectl_path), hide=True).stdout.strip().split()
     for node in nodes:
         run("{} label nodes {} node.kubernetes.io/exclude-from-external-load-balancers-".format(kubectl_path,node), hide=True)
-
-    return nodes
+
+    for iter in range(1, 11):
+        res = run("{} get nodes -l node.kubernetes.io/exclude-from-external-load-balancers -o jsonpath={{.items}}".format(kubectl_path), hide=True).stdout
+        if res == "[]":
+            return
+        time.sleep(1)
+    raise Exception("not able to remove lb exclusions", res)
 
 @task
 def bumplicense(ctx):