chore(ci): avoid dep on confluent-kafka 2.8.1

[hsheth2]

See confluentinc/confluent-kafka-python#1927

Checklist

• The PR conforms to DataHub's Contributing Guideline (particularly Commit Message Format)
• Links to related issues (if applicable)
• Tests for the changes have been added/updated (if applicable)
• Docs related to the changes have been added/updated (if applicable). If a new feature has been added a Usage Guide has been added for the same.
• For any breaking change/potential downtime/deprecation/big changes an entry has been made in Updating DataHub

[codecov]

❌ 5 Tests Failed:

Tests completed	Failed	Passed	Skipped
2243	5	2238	55

View the top 3 failed test(s) by shortest run time

tests.integration.kafka.test_kafka::test_kafka_ingest[kafka]

Stack Traces | 0.001s run time

docker_compose_runner = <function docker_compose_runner.<locals>.run at 0x7f3f7e479730>
test_resources_dir = PosixPath('.../tests/integration/kafka')

    @pytest.fixture(scope="module")
    def mock_kafka_service(docker_compose_runner, test_resources_dir):
        with docker_compose_runner(
            test_resources_dir / "docker-compose.yml", "kafka", cleanup=False
        ) as docker_services:
            wait_for_port(docker_services, "test_zookeeper", 52181, timeout=120)
    
        # Running docker compose twice, since the broker sometimes fails to come up on the first try.
        with docker_compose_runner(
            test_resources_dir / "docker-compose.yml", "kafka"
        ) as docker_services:
>           wait_for_port(docker_services, "test_broker", 29092, timeout=120)

.../integration/kafka/test_kafka.py:36: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.../datahub/testing/docker_utils.py:36: in wait_for_port
    docker_services.wait_until_responsive(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = Services(_docker_compose=DockerComposeExecutor(_compose_command='docker compose', _compose_files=[PosixPath('/home/run...adata.../integration/kafka/docker-compose.yml')], _compose_project_name='pytest4825-kafka'), _services={})
check = <function wait_for_port.<locals>.<lambda> at 0x7f3fa4a83550>
timeout = 120, pause = 0.5, clock = <built-in function perf_counter>

    def wait_until_responsive(
        self,
        check: Any,
        timeout: float,
        pause: float,
        clock: Any = timeit.default_timer,
    ) -> None:
        """Wait until a service is responsive."""
    
        ref = clock()
        now = ref
        while (now - ref) < timeout:
            if check():
                return
            time.sleep(pause)
            now = clock()
    
>       raise Exception("Timeout reached while waiting on service!")
E       Exception: Timeout reached while waiting on service!

venv/lib/python3.8.../site-packages/pytest_docker/plugin.py:121: Exception

tests.integration.kafka.test_kafka::test_kafka_oauth_callback

Stack Traces | 0.001s run time

docker_compose_runner = <function docker_compose_runner.<locals>.run at 0x7f3f7e479730>
test_resources_dir = PosixPath('.../tests/integration/kafka')

    @pytest.fixture(scope="module")
    def mock_kafka_service(docker_compose_runner, test_resources_dir):
        with docker_compose_runner(
            test_resources_dir / "docker-compose.yml", "kafka", cleanup=False
        ) as docker_services:
            wait_for_port(docker_services, "test_zookeeper", 52181, timeout=120)
    
        # Running docker compose twice, since the broker sometimes fails to come up on the first try.
        with docker_compose_runner(
            test_resources_dir / "docker-compose.yml", "kafka"
        ) as docker_services:
>           wait_for_port(docker_services, "test_broker", 29092, timeout=120)

.../integration/kafka/test_kafka.py:36: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.../datahub/testing/docker_utils.py:36: in wait_for_port
    docker_services.wait_until_responsive(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = Services(_docker_compose=DockerComposeExecutor(_compose_command='docker compose', _compose_files=[PosixPath('/home/run...adata.../integration/kafka/docker-compose.yml')], _compose_project_name='pytest4825-kafka'), _services={})
check = <function wait_for_port.<locals>.<lambda> at 0x7f3fa4a83550>
timeout = 120, pause = 0.5, clock = <built-in function perf_counter>

    def wait_until_responsive(
        self,
        check: Any,
        timeout: float,
        pause: float,
        clock: Any = timeit.default_timer,
    ) -> None:
        """Wait until a service is responsive."""
    
        ref = clock()
        now = ref
        while (now - ref) < timeout:
            if check():
                return
            time.sleep(pause)
            now = clock()
    
>       raise Exception("Timeout reached while waiting on service!")
E       Exception: Timeout reached while waiting on service!

venv/lib/python3.8.../site-packages/pytest_docker/plugin.py:121: Exception

tests.integration.kafka.test_kafka::test_kafka_test_connection[config_dict0-True]

Stack Traces | 0.001s run time

docker_compose_runner = <function docker_compose_runner.<locals>.run at 0x7f3f7e479730>
test_resources_dir = PosixPath('.../tests/integration/kafka')

    @pytest.fixture(scope="module")
    def mock_kafka_service(docker_compose_runner, test_resources_dir):
        with docker_compose_runner(
            test_resources_dir / "docker-compose.yml", "kafka", cleanup=False
        ) as docker_services:
            wait_for_port(docker_services, "test_zookeeper", 52181, timeout=120)
    
        # Running docker compose twice, since the broker sometimes fails to come up on the first try.
        with docker_compose_runner(
            test_resources_dir / "docker-compose.yml", "kafka"
        ) as docker_services:
>           wait_for_port(docker_services, "test_broker", 29092, timeout=120)

.../integration/kafka/test_kafka.py:36: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
.../datahub/testing/docker_utils.py:36: in wait_for_port
    docker_services.wait_until_responsive(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = Services(_docker_compose=DockerComposeExecutor(_compose_command='docker compose', _compose_files=[PosixPath('/home/run...adata.../integration/kafka/docker-compose.yml')], _compose_project_name='pytest4825-kafka'), _services={})
check = <function wait_for_port.<locals>.<lambda> at 0x7f3fa4a83550>
timeout = 120, pause = 0.5, clock = <built-in function perf_counter>

    def wait_until_responsive(
        self,
        check: Any,
        timeout: float,
        pause: float,
        clock: Any = timeit.default_timer,
    ) -> None:
        """Wait until a service is responsive."""
    
        ref = clock()
        now = ref
        while (now - ref) < timeout:
            if check():
                return
            time.sleep(pause)
            now = clock()
    
>       raise Exception("Timeout reached while waiting on service!")
E       Exception: Timeout reached while waiting on service!

venv/lib/python3.8.../site-packages/pytest_docker/plugin.py:121: Exception

To view more test analytics, go to the Test Analytics Dashboard
_{📋 Got 3 mins? Take this short survey to help us improve Test Analytics.}

[hsheth2]

Merging despite CI pending to expedite this fix.