diff --git a/.gitignore b/.gitignore index d87910ff..b98901e9 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,6 @@ img_data **/python_notify/ **/confluent-hub-components/ +**/ccloud_library.sh *.tar.gz log.* diff --git a/Docker-connect/README.md b/Docker-connect/README.md new file mode 100644 index 00000000..024be7b7 --- /dev/null +++ b/Docker-connect/README.md @@ -0,0 +1,5 @@ +![image](images/confluent-logo-300-2.png) + +# Documentation + +You can find the documentation and instructions for this repo at [https://docs.confluent.io/platform/current/tutorials/build-your-own-demos.html](https://docs.confluent.io/platform/current/tutorials/build-your-own-demos.html?utm_source=github&utm_medium=demo&utm_campaign=ch.examples_type.community_content.cp-all-in-one) diff --git a/Docker/connect.overrides.yml b/Docker-connect/connect.overrides.yml similarity index 100% rename from Docker/connect.overrides.yml rename to Docker-connect/connect.overrides.yml diff --git a/Docker/Dockerfile b/Docker-connect/distributed/Dockerfile similarity index 94% rename from Docker/Dockerfile rename to Docker-connect/distributed/Dockerfile index fa11a9e0..bea3eb3c 100644 --- a/Docker/Dockerfile +++ b/Docker-connect/distributed/Dockerfile @@ -18,4 +18,5 @@ FROM confluentinc/cp-server-connect-base:6.1.0 ENV CONNECT_PLUGIN_PATH="/usr/share/java,/usr/share/confluent-hub-components" ARG CONNECTOR_NAME -RUN confluent-hub install --no-prompt confluentinc/${CONNECTOR_NAME}:6.0.0 +ARG CONNECTOR_VERSION +RUN confluent-hub install --no-prompt confluentinc/${CONNECTOR_NAME}:${CONNECTOR_VERSION} diff --git a/Docker-connect/standalone/Dockerfile b/Docker-connect/standalone/Dockerfile new file mode 100644 index 00000000..d0191731 --- /dev/null +++ b/Docker-connect/standalone/Dockerfile @@ -0,0 +1,28 @@ +# +# Copyright 2020 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM confluentinc/cp-server-connect-base:6.1.0 + +ENV CONNECT_PLUGIN_PATH="/usr/share/java,/usr/share/confluent-hub-components" + +USER root +COPY --chown=appuser:appuser include/etc/confluent/docker /etc/confluent/docker +USER appuser + +ARG CONNECTOR_NAME +ARG CONNECTOR_VERSION +RUN confluent-hub install --no-prompt confluentinc/${CONNECTOR_NAME}:${CONNECTOR_VERSION} + +CMD ["/etc/confluent/docker/run"] diff --git a/Docker-connect/standalone/include/etc/confluent/docker/configure b/Docker-connect/standalone/include/etc/confluent/docker/configure new file mode 100755 index 00000000..75641061 --- /dev/null +++ b/Docker-connect/standalone/include/etc/confluent/docker/configure @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# +# Copyright 2016 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +. /etc/confluent/docker/bash-config + +# ensure STANDALONE (WORKER) properties +dub ensure STANDALONE_BOOTSTRAP_SERVERS +dub ensure STANDALONE_KEY_CONVERTER +dub ensure STANDALONE_VALUE_CONVERTER +# ensure STANDALONE (CONNECTOR) properties +dub ensure CONNECT_CONNECTOR_CLASS +dub ensure CONNECT_NAME + +# Default to 8083, which matches the mesos-overrides. This is here in case we extend the containers to remove the mesos overrides. +# if [ -z "$CONNECT_REST_PORT" ]; then +# export CONNECT_REST_PORT=8083 +# fi + +# Fix for https://issues.apache.org/jira/browse/KAFKA-3988 +# if [[ ${CONNECT_INTERNAL_KEY_CONVERTER-} == "org.apache.kafka.connect.json.JsonConverter" ]] || [[ ${CONNECT_INTERNAL_VALUE_CONVERTER-} == "org.apache.kafka.connect.json.JsonConverter" ]] +# then +# export CONNECT_INTERNAL_KEY_CONVERTER_SCHEMAS_ENABLE=false +# export CONNECT_INTERNAL_VALUE_CONVERTER_SCHEMAS_ENABLE=false +# fi + +# if [[ $CONNECT_KEY_CONVERTER == "io.confluent.connect.avro.AvroConverter" ]] +# then +# dub ensure CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL +# fi + +# if [[ $CONNECT_VALUE_CONVERTER == "io.confluent.connect.avro.AvroConverter" ]] +# then +# dub ensure CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL +# fi + +if [[ $STANDALONE_KEY_CONVERTER == "io.confluent.connect.avro.AvroConverter" ]] +then + dub ensure STANDALONE_KEY_CONVERTER_SCHEMA_REGISTRY_URL +fi + +if [[ $STANDALONE_VALUE_CONVERTER == "io.confluent.connect.avro.AvroConverter" ]] +then + dub ensure STANDALONE_VALUE_CONVERTER_SCHEMA_REGISTRY_URL +fi + +dub path /etc/"${COMPONENT}"/ writable + +dub template "/etc/confluent/docker/${COMPONENT}.properties.template" "/etc/${COMPONENT}/${COMPONENT}.properties" +dub template "/etc/confluent/docker/${COMPONENT}-standalone.properties.template" "/etc/${COMPONENT}/${COMPONENT}-standalone.properties" + +# The connect-distributed script expects the log4j config at /etc/kafka/connect-log4j.properties. +dub template "/etc/confluent/docker/log4j.properties.template" "/etc/kafka/connect-log4j.properties" diff --git a/Docker-connect/standalone/include/etc/confluent/docker/kafka-connect-standalone.properties.template b/Docker-connect/standalone/include/etc/confluent/docker/kafka-connect-standalone.properties.template new file mode 100644 index 00000000..a2780e06 --- /dev/null +++ b/Docker-connect/standalone/include/etc/confluent/docker/kafka-connect-standalone.properties.template @@ -0,0 +1,4 @@ +{% set connect_props = env_to_props('STANDALONE_', '') -%} +{% for name, value in connect_props.items() -%} +{{name}}={{value}} +{% endfor -%} \ No newline at end of file diff --git a/Docker-connect/standalone/include/etc/confluent/docker/launch b/Docker-connect/standalone/include/etc/confluent/docker/launch new file mode 100755 index 00000000..f25ad56b --- /dev/null +++ b/Docker-connect/standalone/include/etc/confluent/docker/launch @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# +# Copyright 2016 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Override this section from the script to include the com.sun.management.jmxremote.rmi.port property. +if [ -z "$KAFKA_JMX_OPTS" ]; then + export KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false " +fi + +# The JMX client needs to be able to connect to java.rmi.server.hostname. +# The default for bridged n/w is the bridged IP so you will only be able to connect from another docker container. +# For host n/w, this is the IP that the hostname on the host resolves to. + +# If you have more that one n/w configured, hostname -i gives you all the IPs, +# the default is to pick the first IP (or network). +export KAFKA_JMX_HOSTNAME=${KAFKA_JMX_HOSTNAME:-$(hostname -i | cut -d" " -f1)} + +if [ "$KAFKA_JMX_PORT" ]; then + # This ensures that the "if" section for JMX_PORT in kafka launch script does not trigger. + export JMX_PORT=$KAFKA_JMX_PORT + export KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=$KAFKA_JMX_HOSTNAME -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT -Dcom.sun.management.jmxremote.port=$JMX_PORT" +fi + +echo "===> Launching ${COMPONENT} ... " +# Add external jars to the classpath +# And this also makes sure that the CLASSPATH does not start with ":/etc/..." +# because this causes the plugin scanner to scan the entire disk. +if [ -z "$CLASSPATH" ]; then + export CLASSPATH="/etc/kafka-connect/jars/*" +fi +exec connect-standalone /etc/"${COMPONENT}"/"${COMPONENT}"-standalone.properties /etc/"${COMPONENT}"/"${COMPONENT}".properties diff --git a/Docker-connect/standalone/include/etc/confluent/docker/run b/Docker-connect/standalone/include/etc/confluent/docker/run new file mode 100755 index 00000000..6d35fb07 --- /dev/null +++ b/Docker-connect/standalone/include/etc/confluent/docker/run @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +. /etc/confluent/docker/bash-config + +echo "===> User" +id + +echo "===> Configuring ..." +/etc/confluent/docker/configure + +echo "===> Launching ... " +exec /etc/confluent/docker/launch \ No newline at end of file diff --git a/cp-all-in-one-cloud/docker-compose.connect.local.yml b/cp-all-in-one-cloud/docker-compose.connect.distributed.yml similarity index 96% rename from cp-all-in-one-cloud/docker-compose.connect.local.yml rename to cp-all-in-one-cloud/docker-compose.connect.distributed.yml index fc2a5394..0fb86d99 100644 --- a/cp-all-in-one-cloud/docker-compose.connect.local.yml +++ b/cp-all-in-one-cloud/docker-compose.connect.distributed.yml @@ -3,7 +3,7 @@ version: '2' services: connect: - image: localbuild/connect_custom_example:latest + image: localbuild/connect_distributed_with_${CONNECTOR_NAME}:${CONNECTOR_VERSION} hostname: connect container_name: connect ports: diff --git a/cp-all-in-one-cloud/docker-compose.connect.standalone.yml b/cp-all-in-one-cloud/docker-compose.connect.standalone.yml new file mode 100644 index 00000000..419c1860 --- /dev/null +++ b/cp-all-in-one-cloud/docker-compose.connect.standalone.yml @@ -0,0 +1,42 @@ +--- +version: '2' +services: + + connect: + image: localbuild/connect_standalone_with_${CONNECTOR_NAME}:${CONNECTOR_VERSION} + hostname: connect + container_name: connect + environment: + # STANDALONE_ properties are for the Connect worker + STANDALONE_BOOTSTRAP_SERVERS: $BOOTSTRAP_SERVERS + STANDALONE_OFFSET_STORAGE_FILE_FILENAME: /tmp/connect.offsets + STANDALONE_OFFSET_FLUSH_INTERVAL_MS: 10000 + STANDALONE_SECURITY_PROTOCOL: SASL_SSL + STANDALONE_SASL_JAAS_CONFIG: $SASL_JAAS_CONFIG + STANDALONE_SASL_MECHANISM: PLAIN + STANDALONE_PRODUCER_SECURITY_PROTOCOL: SASL_SSL + STANDALONE_PRODUCER_SASL_JAAS_CONFIG: $SASL_JAAS_CONFIG + STANDALONE_PRODUCER_SASL_MECHANISM: PLAIN + STANDALONE_CONSUMER_SECURITY_PROTOCOL: SASL_SSL + STANDALONE_CONSUMER_SASL_JAAS_CONFIG: $SASL_JAAS_CONFIG + STANDALONE_CONSUMER_SASL_MECHANISM: PLAIN + STANDALONE_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter + STANDALONE_KEY_CONVERTER_SCHEMA_REGISTRY_URL: $SCHEMA_REGISTRY_URL + STANDALONE_KEY_CONVERTER_BASIC_AUTH_CREDENTIALS_SOURCE: USER_INFO + STANDALONE_KEY_CONVERTER_SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO: $SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO + STANDALONE_KEY_CONVERTER_SCHEMAS_ENABLE: "true" + STANDALONE_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter + STANDALONE_VALUE_CONVERTER_SCHEMAS_ENABLE: "true" + STANDALONE_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: $SCHEMA_REGISTRY_URL + STANDALONE_VALUE_CONVERTER_BASIC_AUTH_CREDENTIALS_SOURCE: USER_INFO + STANDALONE_VALUE_CONVERTER_SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO: $SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO + STANDALONE_PLUGIN_PATH: /usr/share/confluent-hub-components/confluentinc-kafka-connect-elasticsearch/ + + # CONNECT_ properties are for the connector itself, example shown below + CONNECT_NAME: elasticsearch-sink-connector + CONNECT_CONNECTOR_CLASS: io.confluent.connect.elasticsearch.ElasticsearchSinkConnector + CONNECT_TYPE_NAME: _doc + CONNECT_TOPICS: test01 + CONNECT_KEY_IGNORE: "true" + CONNECT_SCHEMA_IGNORE: "false" + CONNECT_CONNECTION_URL: http://elasticsearch:9200