push
This commit is contained in:
66
kafka-stack/README.md
Executable file
66
kafka-stack/README.md
Executable file
@@ -0,0 +1,66 @@
|
||||
Qualtrics Messaging Platform Docker Quick Start Locally
|
||||
===================================
|
||||
|
||||
Usage
|
||||
-----
|
||||
|
||||
Clone this repository:
|
||||
```bash
|
||||
git clone ssh://git@gitlab-app.eng.qops.net:10022/core/qmp/qmp-docker.git
|
||||
```
|
||||
Move into directory and set required environment variables:
|
||||
```bash
|
||||
cd qmp-docker/
|
||||
export KAFKA_CREATE_TOPICS="my-topic-1:20:1;my-topic-2:20:1"
|
||||
export SCHEMAS_LOCATION="<absolute path to .avsc schema files>"
|
||||
```
|
||||
|
||||
*KAFKA_CREATE_TOPICS is a list of topics separated by a ';'. Each topic consists of three fields separated by ':',
|
||||
topic name, number of partitions, number of replicas. Since this is for running locally, we are only starting one
|
||||
broker therefore the number of replicas is required to be 1. By default, QMP creates topics with 20 partitions which
|
||||
directly maps to the maximum number of consumers that can consume from this topic concurrently.*
|
||||
|
||||
*SCHEMAS_LOCATION is the path to the directory containing your avro schemas. The schema registry will automatically
|
||||
load in the schemas at this location on start up.*
|
||||
|
||||
**NOTE:** The qmp.envelope.schema_id.avsc schema file is required for qmp. If you do not have this file, copy it from
|
||||
[here](https://gitlab-app.eng.qops.net/core/qmp/qmp-schema/blob/master/test/qmp/qmp.envelope.schema_id.v1.avsc).
|
||||
|
||||
To run specific QMP image versions, export the following:
|
||||
```text
|
||||
export ZOOKEEPER_VERSION="<version of core zookeeper image>"
|
||||
export KAFKA_VERSION="<version of core kafka image>"
|
||||
export SCHEMA_REGISTRY_VERSION="<version of core schema registry image>"
|
||||
export SCHEMA_REGISTRY_UI_VERSION="<version of core schema registry ui image>"
|
||||
export KAFKA_MANAGER_VERSION="<version of core kafka manager image>"
|
||||
```
|
||||
|
||||
Start the QMP images in order:
|
||||
```bash
|
||||
docker-compose up
|
||||
```
|
||||
|
||||
This will start the following images on the following ports:
|
||||
* Zookeeper: 2181
|
||||
* Kafka: 9092
|
||||
* Schema Registry: 8081
|
||||
* Schema Registry UI: 8000
|
||||
* Kafka Manager: 9000
|
||||
|
||||
To add the cluster in kafka manager, navigate to [localhost:9000](http://localhost:9000). If you are prompted for a
|
||||
username and password, simply input *admin* for the username and *password* for the password. Click on the Cluster
|
||||
dropdown and select *Add Cluster*. Input *local* for the cluster name, and *core-zookeeper:2181/qmp/local* for cluster
|
||||
zookeeper hosts. These are the only required fields, scroll to the bottom and hit *Save*.
|
||||
|
||||
The Schema Registry UI can be accessed at [localhost:8000](http://localhost:8000).
|
||||
|
||||
To tear down QMP locally, control+c the process, wait for graceful teardown, then execute:
|
||||
```bash
|
||||
docker-compose down
|
||||
```
|
||||
|
||||
It's recommended to add the following entry to your `/etc/hosts` file:
|
||||
```text
|
||||
127.0.0.1 core-kafka
|
||||
```
|
||||
This way you can access the kafka broker from the container name `core-kafka`.
|
||||
128
kafka-stack/docker-compose.yml
Executable file
128
kafka-stack/docker-compose.yml
Executable file
@@ -0,0 +1,128 @@
|
||||
version: '2.3'
|
||||
|
||||
services:
|
||||
zookeeper:
|
||||
image: "registry-app.eng.qops.net:5001/core-platform/zookeeper:${ZOOKEEPER_VERSION:-0.1.4}"
|
||||
container_name: "core-zookeeper"
|
||||
healthcheck:
|
||||
test: ["CMD", "echo", "ruok", "|", "nc", "core-zookeeper", "2181"]
|
||||
interval: 5s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
ports:
|
||||
- "2181:2181"
|
||||
environment:
|
||||
- "ZOO_ENSEMBLE_HOST_LIST=zk1"
|
||||
|
||||
kafka:
|
||||
image: "registry-app.eng.qops.net:5001/core-platform/kafka:${KAFKA_VERSION:-0.2.0}"
|
||||
container_name: "core-kafka"
|
||||
depends_on:
|
||||
zookeeper:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "grep", "KafkaServer id=", "/opt/qualtrics/kafka/logs/server.log"]
|
||||
interval: 5s
|
||||
timeout: 10s
|
||||
retries: 12
|
||||
ports:
|
||||
- "9092:9092"
|
||||
- "8092:8092"
|
||||
environment:
|
||||
- "KAFKA_CONFIG_PORT=9092"
|
||||
- "KAFKA_CONFIG_BROKER_ID=1"
|
||||
- "KAFKA_HTTP_HEALTH_PORT=8092"
|
||||
- "KAFKA_CONFIG_ZOOKEEPER_CONNECT=core-zookeeper:2181"
|
||||
- "KAFKA_CONFIG_HOST_NAME=core-kafka"
|
||||
- "KAFKA_ZOO_CHROOT=kafka/local"
|
||||
- "KAFKA_CREATE_TOPICS=${KAFKA_CREATE_TOPICS}"
|
||||
|
||||
schema-registry:
|
||||
image: "registry-app.eng.qops.net:5001/core-platform/schema-registry:${SCHEMA_REGISTRY_VERSION:-0.2.1}"
|
||||
container_name: "qmp-schema-registry"
|
||||
depends_on:
|
||||
zookeeper:
|
||||
condition: service_healthy
|
||||
kafka:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://127.0.0.1:8081"]
|
||||
interval: 5s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
ports:
|
||||
- "8081:8081"
|
||||
environment:
|
||||
- "SCHEMA_REGISTRY_PORT=8081"
|
||||
- "SCHEMA_REGISTRY_CONFIG_KAFKASTORE_CONNECTION_URL=core-zookeeper:2181"
|
||||
- "SCHEMA_REGISTRY_CONFIG_KAFKASTORE_TOPIC_REPLICATION_FACTOR=1"
|
||||
- "SCHEMA_REGISTRY_CONFIG_DEBUG=true"
|
||||
- "SCHEMA_REGISTRY_CONFIG_ACCESS_CONTROL_ALLOW_METHODS=GET,POST,PUT,OPTIONS"
|
||||
- "SCHEMA_REGISTRY_CONFIG_ACCESS_CONTROL_ALLOW_ORIGIN=*"
|
||||
- "SCHEMA_REGISTRY_ZOO_CHROOT=kafka/local"
|
||||
|
||||
schema-registry-ui:
|
||||
image: "registry-app.eng.qops.net:5001/core-platform/schema-registry-ui:${SCHEMA_REGISTRY_UI_VERSION:-0.1.0}"
|
||||
container_name: "qmp-schema-registry-ui"
|
||||
depends_on:
|
||||
schema-registry:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
- "8000:8000"
|
||||
|
||||
kafka-manager:
|
||||
image: "registry-app.eng.qops.net:5001/core-platform/kafka-manager:${KAFKA_MANAGER_VERSION:-0.1.5}"
|
||||
container_name: "core-kafka-manager"
|
||||
depends_on:
|
||||
zookeeper:
|
||||
condition: service_healthy
|
||||
kafka:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
- "9000:9000"
|
||||
environment:
|
||||
- "ZK_HOSTS=core-zookeeper:2181/kafka/local"
|
||||
- "KAFKA_MANAGER_AUTH_ENABLED=false"
|
||||
- "APPLICATION_SECRET=letmein"
|
||||
|
||||
qmp-tool:
|
||||
image: "registry-app.eng.qops.net:5001/core-platform/qmp-tool:${QMP_TOOL_VERSION:-1.1.0}"
|
||||
container_name: "qmp-tool"
|
||||
depends_on:
|
||||
schema-registry:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
- "SCHEMA_REGISTRY_URL=qmp-schema-registry:8081"
|
||||
- "COMMAND=schema-management"
|
||||
volumes:
|
||||
- "${SCHEMAS_LOCATION:-./}:/opt/qualtrics/qmp/data"
|
||||
|
||||
kafka-rest-proxy:
|
||||
image: "confluentinc/cp-kafka-rest"
|
||||
container_name: "kafka-rest-proxy"
|
||||
depends_on:
|
||||
kafka:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://127.0.0.1:8082"]
|
||||
interval: 5s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
ports:
|
||||
- "8082:8082"
|
||||
environment:
|
||||
- "KAFKA_REST_ZOOKEEPER_CONNECT=${ZK_CONNECT:-core-zookeeper:2181/kafka/local}"
|
||||
- "KAFKA_REST_SCHEMA_REGISTRY_URL=${AVSC_CONNECT:-qmp-schema-registry:8081}"
|
||||
- "KAFKA_REST_HOST_NAME=localhost"
|
||||
- "KAFKA_REST_LISTENERS=http://0.0.0.0:8082"
|
||||
|
||||
kafka-topics-ui:
|
||||
image: "landoop/kafka-topics-ui"
|
||||
container_name: "kafka-topics-ui"
|
||||
depends_on:
|
||||
kafka-rest-proxy:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
- "8083:8000"
|
||||
environment:
|
||||
- "KAFKA_REST_PROXY_URL=http://kafka-rest-proxy:8082"
|
||||
24
kafka-stack/isolation-service.key-rotation.v1.avsc
Executable file
24
kafka-stack/isolation-service.key-rotation.v1.avsc
Executable file
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"namespace": "qmp.envelope",
|
||||
"type": "record",
|
||||
"name": "envelope",
|
||||
"fields": [
|
||||
{"name": "schema_id", "type": "string"},
|
||||
{"name": "message_id", "type": "string"},
|
||||
{"name": "timestamp", "type": "long"},
|
||||
{"name": "writer_host", "type": "string"},
|
||||
{"name": "writer_app", "type": "string"},
|
||||
{"name": "trace_id", "type": ["null", "string"], "default": null},
|
||||
{"name": "organization_id", "type": ["null", "string"], "default": null},
|
||||
{"name": "payload", "type": {
|
||||
"type": "record",
|
||||
"name": "payload",
|
||||
"fields": [
|
||||
{
|
||||
"name": "brandId",
|
||||
"type": "string"
|
||||
}
|
||||
]
|
||||
}}
|
||||
]
|
||||
}
|
||||
90
kafka-stack/old-docker-compose.yml
Executable file
90
kafka-stack/old-docker-compose.yml
Executable file
@@ -0,0 +1,90 @@
|
||||
version: '2.3'
|
||||
|
||||
services:
|
||||
zookeeper:
|
||||
image: "registry-app.eng.qops.net:5001/core-platform/zookeeper:${ZOOKEEPER_VERSION:-0.1.2}"
|
||||
container_name: "core-zookeeper"
|
||||
healthcheck:
|
||||
test: ["CMD", "echo", "ruok", "|", "nc", "core-zookeeper", "2181"]
|
||||
interval: 5s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
ports:
|
||||
- "2181:2181"
|
||||
environment:
|
||||
- "ZOO_ENSEMBLE_HOST_LIST=zk1"
|
||||
- "ZOO_HOST_MIN_MEMORY=0"
|
||||
|
||||
kafka:
|
||||
image: "registry-app.eng.qops.net:5001/core-platform/kafka:${KAFKA_VERSION:-0.1.4}"
|
||||
container_name: "core-kafka"
|
||||
depends_on:
|
||||
zookeeper:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "grep", "KafkaServer id=", "/opt/qualtrics/kafka/logs/server.log"]
|
||||
interval: 5s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
ports:
|
||||
- "9092:9092"
|
||||
environment:
|
||||
- "KAFKA_CONFIG_PORT=9092"
|
||||
- "KAFKA_CONFIG_ZOOKEEPER_CONNECT=core-zookeeper:2181"
|
||||
- "KAFKA_CONFIG_HOST_NAME=core-kafka"
|
||||
- "KAFKA_ZOO_CHROOT=qmp/local"
|
||||
- "KAFKA_CREATE_TOPICS=${KAFKA_CREATE_TOPICS}"
|
||||
- "JMX_PORT=${JMX_PORT:-9999}"
|
||||
|
||||
schema-registry:
|
||||
image: "registry-app.eng.qops.net:5001/core-platform/schema-registry:${SCHEMA_REGISTRY_VERSION:-0.2.0}"
|
||||
container_name: "qmp-schema-registry"
|
||||
depends_on:
|
||||
zookeeper:
|
||||
condition: service_healthy
|
||||
kafka:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://127.0.0.1:8081/config"]
|
||||
interval: 5s
|
||||
timeout: 10s
|
||||
retries: 5
|
||||
ports:
|
||||
- "8081:8081"
|
||||
environment:
|
||||
- "SCHEMA_REGISTRY_PORT=8081"
|
||||
- "SCHEMA_REGISTRY_CONFIG_KAFKASTORE_CONNECTION_URL=core-zookeeper:2181"
|
||||
- "SCHEMA_REGISTRY_CONFIG_KAFKASTORE_TOPIC_REPLICATION_FACTOR=1"
|
||||
- "SCHEMA_REGISTRY_CONFIG_DEBUG=true"
|
||||
- "SCHEMA_REGISTRY_CONFIG_ACCESS_CONTROL_ALLOW_METHODS=GET,POST,PUT,OPTIONS"
|
||||
- "SCHEMA_REGISTRY_CONFIG_ACCESS_CONTROL_ALLOW_ORIGIN=*"
|
||||
- "SCHEMA_REGISTRY_ZOO_CHROOT=qmp/local"
|
||||
- "SCHEMA_REGISTRY_SCHEMA_LOCATION=/opt/qualtrics/schemas"
|
||||
volumes:
|
||||
- "${SCHEMAS_LOCATION}:/opt/qualtrics/schemas"
|
||||
|
||||
schema-registry-ui:
|
||||
image: "registry-app.eng.qops.net:5001/core-platform/schema-registry-ui:${SCHEMA_REGISTRY_UI_VERSION:-0.1.0}"
|
||||
container_name: "qmp-schema-registry-ui"
|
||||
depends_on:
|
||||
schema-registry:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
- "8000:8000"
|
||||
|
||||
kafka-manager:
|
||||
image: "registry-app.eng.qops.net:5001/core-platform/kafka-manager:${KAFKA_MANAGER_VERSION:-0.1.1}"
|
||||
container_name: "core-kafka-manager"
|
||||
depends_on:
|
||||
zookeeper:
|
||||
condition: service_healthy
|
||||
kafka:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
- "9000:9000"
|
||||
environment:
|
||||
- "ZK_HOSTS=core-zookeeper:2181"
|
||||
- "KAFKA_MANAGER_AUTH_ENABLED=true"
|
||||
- "KAFKA_MANAGER_USERNAME=admin"
|
||||
- "KAFKA_MANAGER_PASSWORD=password"
|
||||
- "APPLICATION_SECRET=letmein"
|
||||
111
kafka-stack/qmp.configuration.v1.avsc
Executable file
111
kafka-stack/qmp.configuration.v1.avsc
Executable file
@@ -0,0 +1,111 @@
|
||||
[
|
||||
{
|
||||
"type": "record",
|
||||
"name": "tc_topic_field",
|
||||
"fields": [
|
||||
{"name": "max_message_bytes", "type": ["null", "int"], "default": null},
|
||||
{"name": "retention_bytes", "type": ["null", "long"], "default": null},
|
||||
{"name": "retention_ms", "type": ["null", "long"], "default": null},
|
||||
{"name": "segment_ms", "type": ["null", "long"], "default": null},
|
||||
{"name": "min_insync_replicas", "type": ["null", "int"], "default": null},
|
||||
{"name": "cleanup_policy", "type": ["null", "string"], "default": null}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "record",
|
||||
"name": "tc_producer_field",
|
||||
"fields": [
|
||||
{"name": "max_request_size", "type": ["null", "int"], "default": null},
|
||||
{"name": "buffer_memory", "type": ["null", "long"], "default": null},
|
||||
{"name": "batch_size", "type": ["null", "int"], "default": null},
|
||||
{"name": "linger_ms", "type": ["null", "long"], "default": null}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "record",
|
||||
"name": "tc_consumer_field",
|
||||
"fields": [
|
||||
{"name": "fetch_max_bytes", "type": ["null", "int"], "default": null}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "record",
|
||||
"name": "qpl_producer_field",
|
||||
"fields": [
|
||||
{"name": "acks", "type": ["null", "string"], "default": null},
|
||||
{"name": "retries", "type": ["null", "int"], "default": null},
|
||||
{"name": "max_in_flight_requests_per_connection", "type": ["null", "int"], "default": null}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "record",
|
||||
"name": "qcl_consumer_field",
|
||||
"fields": [
|
||||
{"name": "enable_auto_commit", "type": "boolean", "default": false},
|
||||
{"name": "max_poll_records", "type": ["null", "int"], "default": null},
|
||||
{"name": "auto_offset_reset", "type": ["null", "string"], "default": null}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "record",
|
||||
"name": "qcl_qmp_field",
|
||||
"fields": [
|
||||
{"name": "qmp_max_lag_ms", "type": ["null", "long"], "default": null},
|
||||
{"name": "qmp_record_retries", "type": ["null", "int"], "default": null},
|
||||
{"name": "qmp_poll_timeout", "type": ["null", "long"], "default": null},
|
||||
{"name": "qmp_sleep_time", "type": ["null", "long"], "default": null}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "record",
|
||||
"name": "topic_class",
|
||||
"fields": [
|
||||
{"name": "topic", "type": "tc_topic_field"},
|
||||
{"name": "producer", "type": "tc_producer_field"},
|
||||
{"name": "consumer", "type": "tc_consumer_field"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "record",
|
||||
"name": "producer",
|
||||
"fields": [
|
||||
{"name": "producer", "type": "qpl_producer_field"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "record",
|
||||
"name": "consumer",
|
||||
"fields": [
|
||||
{"name": "consumer", "type": "qcl_consumer_field"},
|
||||
{"name": "qmp", "type": "qcl_qmp_field"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"namespace": "qmp.envelope",
|
||||
"type": "record",
|
||||
"name": "envelope",
|
||||
"fields": [
|
||||
{"name": "schema_id", "type": "string"},
|
||||
{"name": "message_id", "type": "string"},
|
||||
{"name": "timestamp", "type": "long"},
|
||||
{"name": "writer_host", "type": "string"},
|
||||
{"name": "writer_app", "type": "string"},
|
||||
{"name": "trace_id", "type": ["null", "string"], "default": null},
|
||||
{"name": "organization_id", "type": ["null", "string"], "default": null},
|
||||
{"name": "payload", "type": {
|
||||
"type": "record",
|
||||
"name": "payload",
|
||||
"fields": [
|
||||
{"name": "A", "type": ["null", "topic_class"], "default": null},
|
||||
{"name": "B", "type": ["null", "topic_class"], "default": null},
|
||||
{"name": "C", "type": ["null", "topic_class"], "default": null},
|
||||
{"name": "bulk_producer", "type": ["null", "producer"], "default": null},
|
||||
{"name": "durable_producer", "type": ["null", "producer"], "default": null},
|
||||
{"name": "bulk_consumer", "type": ["null", "consumer"], "default": null},
|
||||
{"name": "background_job_consumer", "type": ["null", "consumer"], "default": null},
|
||||
{"name": "idempotent_background_job_consumer", "type": ["null", "consumer"], "default": null},
|
||||
{"name": "topic_map", "type": ["null", {"type": "map", "values": "string"}], "default": null}
|
||||
]
|
||||
}}
|
||||
]
|
||||
}]
|
||||
8
kafka-stack/qmp.envelope.schema_id.v1.avsc
Executable file
8
kafka-stack/qmp.envelope.schema_id.v1.avsc
Executable file
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"namespace": "qmp.unittest",
|
||||
"type": "record",
|
||||
"name": "envelope",
|
||||
"fields": [
|
||||
{"name": "schema_id", "type": "string"}
|
||||
]
|
||||
}
|
||||
8
kafka-stack/qmp.envelope.v1.avsc
Executable file
8
kafka-stack/qmp.envelope.v1.avsc
Executable file
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"namespace": "qmp.envelope",
|
||||
"type": "record",
|
||||
"name": "envelope",
|
||||
"fields": [
|
||||
{"name": "schema_id", "type": "string"}
|
||||
]
|
||||
}
|
||||
21
kafka-stack/qmp.payload.v1.avsc
Executable file
21
kafka-stack/qmp.payload.v1.avsc
Executable file
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"namespace": "qmp.unittest",
|
||||
"type": "record",
|
||||
"name": "envelope",
|
||||
"fields": [
|
||||
{"name": "schema_id", "type": "string"},
|
||||
{"name": "message_id", "type": "string"},
|
||||
{"name": "timestamp", "type": "long"},
|
||||
{"name": "writer_host", "type": "string"},
|
||||
{"name": "writer_app", "type": "string"},
|
||||
{"name": "trace_id", "type": ["null", "string"], "default": null},
|
||||
{"name": "organization_id", "type": ["null", "string"], "default": null},
|
||||
{"name": "payload", "type": {
|
||||
"type": "record",
|
||||
"name": "payload",
|
||||
"fields": [
|
||||
{"name": "body1", "type": ["null", "string"], "default": null}
|
||||
]
|
||||
}}
|
||||
]
|
||||
}
|
||||
74
kafka-stack/start.sh
Executable file
74
kafka-stack/start.sh
Executable file
@@ -0,0 +1,74 @@
|
||||
#! /bin/bash
|
||||
|
||||
function variables {
|
||||
echo "Setting environment variables..."
|
||||
export KAFKA_CREATE_TOPICS="${KAFKA_CREATE_TOPICS:-my-topic-1:20:1;my-topic-2:20:1;testtopic:5:1;qmp.state.config:1:1;qmp.state.throttle:1:1;isolation-service.key.rotation:20:1}"
|
||||
export SCHEMAS_LOCATION="${SCHEMAS_LOCATION:-$(dirname "${BASH_SOURCE[0]}")}"
|
||||
export SCHEMA_REGISTRY_URL="${SCHEMA_REGISTRY_URL:-localhost:8081}"
|
||||
export QPL_BULK_BOOTSTRAP_SERVERS="${QPL_BULK_BOOTSTRAP_SERVERS:-core-kafka:9092}"
|
||||
export QPL_DURABLE_BOOTSTRAP_SERVERS="${QPL_DURABLE_BOOTSTRAP_SERVERS:-$QPL_BULK_BOOTSTRAP_SERVERS}"
|
||||
export QCL_BULK_BOOTSTRAP_SERVERS="${QCL_BULK_BOOTSTRAP_SERVERS:-$QPL_BULK_BOOTSTRAP_SERVERS}"
|
||||
export QCL_RECORD_STATE_BOOTSTRAP_SERVERS="${QCL_RECORD_STATE_BOOTSTRAP_SERVERS:-$QPL_BULK_BOOTSTRAP_SERVERS}"
|
||||
export QCL_BACKGROUND_JOB_BOOTSTRAP_SERVERS="${QCL_BACKGROUND_JOB_BOOTSTRAP_SERVERS:-$QPL_BULK_BOOTSTRAP_SERVERS}"
|
||||
export QCL_IDEMPOTENT_BACKGROUND_JOB_BOOTSTRAP_SERVERS="${QCL_IDEMPOTENT_BACKGROUND_JOB_BOOTSTRAP_SERVERS:-$QPL_BULK_BOOTSTRAP_SERVERS}"
|
||||
export JMX_PORT="${JMX_PORT:-9999}"
|
||||
}
|
||||
|
||||
function start {
|
||||
if [ "$(cat /etc/hosts | grep core-kafka)" == "" ]; then
|
||||
echo "WARNING: Need 127.0.0.1 core-kafka in /etc/hosts"
|
||||
fi
|
||||
variables
|
||||
echo "Starting docker..."
|
||||
docker-compose up
|
||||
}
|
||||
|
||||
function stop {
|
||||
echo "Stopping docker..."
|
||||
docker-compose down
|
||||
}
|
||||
|
||||
function initCluster {
|
||||
echo "Initializing cluster at localhost:9000"
|
||||
res="$(curl -X POST \
|
||||
-F 'name=local' \
|
||||
-F 'zkHosts=core-zookeeper%3A2181%2Fqmp%2Flocal' \
|
||||
-F 'kafkaVersion=0.9.0.1' \
|
||||
-F 'jmxEnabled=true' \
|
||||
-F 'jmxUser=' \
|
||||
-F 'jmxPass=' \
|
||||
-F 'pollConsumers=true' \
|
||||
-F 'tuning.brokerViewUpdatePeriodSeconds=30' \
|
||||
-F 'tuning.clusterManagerThreadPoolSize=2' \
|
||||
-F 'tuning.clusterManagerThreadPoolQueueSize=100' \
|
||||
-F 'tuning.kafkaCommandThreadPoolSize=2' \
|
||||
-F 'tuning.kafkaCommandThreadPoolQueueSize=100' \
|
||||
-F 'tuning.logkafkaCommandThreadPoolSize=2' \
|
||||
-F 'tuning.logkafkaCommandThreadPoolQueueSize=100' \
|
||||
-F 'tuning.logkafkaUpdatePeriodSeconds=30' \
|
||||
-F 'tuning.partitionOffsetCacheTimeoutSecs=5' \
|
||||
-F 'tuning.brokerViewThreadPoolSize=2' \
|
||||
-F 'tuning.brokerViewThreadPoolQueueSize=1000' \
|
||||
-F 'tuning.offsetCacheThreadPoolSize=2' \
|
||||
-F 'tuning.offsetCacheThreadPoolQueueSize=1000' \
|
||||
-F 'tuning.kafkaAdminClientThreadPoolSize=2' \
|
||||
-F 'tuning.kafkaAdminClientThreadPoolQueueSize=1000' \
|
||||
-F 'securityProtocol=PLAINTEXT' \
|
||||
http://localhost:9000/clusters)"
|
||||
echo $res
|
||||
}
|
||||
|
||||
srcdir="$(pwd)"
|
||||
cd "$(dirname ${BASH_SOURCE[0]})"
|
||||
|
||||
echo "Would you like to set environment variables, start docker, stop docker, or initialize the cluster?"
|
||||
select opt in "start" "stop" "env"; do
|
||||
case $opt,$REPLY in
|
||||
*start*,*|*,*start*) start ; break;;
|
||||
*stop*,*|*,*stop*) stop ; break;;
|
||||
*env*,*|*,*env*) variables ; break;;
|
||||
*) break;;
|
||||
esac
|
||||
done
|
||||
|
||||
cd "$srcdir"
|
||||
Reference in New Issue
Block a user