-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathdocker-compose.yml
143 lines (132 loc) · 4.65 KB
/
docker-compose.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
---
version: '3.2'
networks:
cnvr-kafka:
driver: bridge
services:
zookeeper:
image: confluentinc/cp-zookeeper
hostname: zookeeper
networks:
- cnvr-kafka
container_name: cnvr_zookeeper
ports:
- "2181:2181"
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
broker:
image: confluentinc/cp-enterprise-kafka
hostname: broker
networks:
- cnvr-kafka
container_name: cnvr_kafka_broker
volumes:
- "./data/kdata:/var/lib/kafka/data"
depends_on:
- zookeeper
ports:
- "9092:9092"
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://broker:9092'
KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker:9092
CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zookeeper:2181
CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
CONFLUENT_METRICS_ENABLE: 'true'
CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'
schema_registry:
image: confluentinc/cp-schema-registry
hostname: schema_registry
networks:
- cnvr-kafka
container_name: cnvr_kafka_registry
depends_on:
- zookeeper
- broker
ports:
- "8081:8081"
environment:
SCHEMA_REGISTRY_HOST_NAME: schema_registry
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
connect:
image: confluentinc/cp-kafka-connect
hostname: connect
networks:
- cnvr-kafka
container_name: cnvr_kafka_connect
depends_on:
- zookeeper
- broker
- schema_registry
ports:
- "8083:8083"
environment:
CONNECT_BOOTSTRAP_SERVERS: 'broker:9092'
CONNECT_REST_ADVERTISED_HOST_NAME: connect
CONNECT_REST_PORT: 8083
CONNECT_GROUP_ID: compose-connect-group
CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema_registry:8081'
CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema_registry:8081'
CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
CONNECT_ZOOKEEPER_CONNECT: 'zookeeper:2181'
CONNECT_PLUGIN_PATH: '/usr/share/java/'
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:6.2.4
hostname: elasticsearch
networks:
- cnvr-kafka
container_name: cnvr_elasticsearch
volumes:
- "./data/esdata:/usr/share/elasticsearch/data"
ports:
- 9200:9200
environment:
- discovery.type=single-node
- cluster.name=kafkastreams
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- "xpack.security.enabled=false"
kibana:
depends_on:
- elasticsearch
image: docker.elastic.co/kibana/kibana:6.2.4
hostname: kibana
networks:
- cnvr-kafka
container_name: cnvr_kibana
ports:
- 5601:5601
environment:
SERVER_NAME: kibana.corp.valueclick.com
ELASTICSEARCH_URL: http://elasticsearch:9200
XPACK_SECURITY_ENABLED: 'false'
jupyter:
depends_on:
- elasticsearch
- kibana
- broker
image: cnvr/jupyter
hostname: jupyter
networks:
- cnvr-kafka
container_name: cnvr_notebook
volumes:
- "./src/jupyter:/cnvr/jupyter"
- "./src/spark-avro/target/scala-2.11:/cnvr/spark-avro"
ports:
- 8888:8888
- 4040-4050:4040-4050