DEVX-2537: Standalone connect example into cp-all-in-one for Confluen…

…t Cloud (confluentinc#65)

.gitignore

@@ -5,5 +5,6 @@
 img_data
 **/python_notify/
 **/confluent-hub-components/
+**/ccloud_library.sh
 *.tar.gz
 log.*

Docker-connect/README.md

@@ -0,0 +1,5 @@
+![image](images/confluent-logo-300-2.png)
+
+# Documentation
+
+You can find the documentation and instructions for this repo at [https://docs.confluent.io/platform/current/tutorials/build-your-own-demos.html](https://docs.confluent.io/platform/current/tutorials/build-your-own-demos.html?utm_source=github&utm_medium=demo&utm_campaign=ch.examples_type.community_content.cp-all-in-one)

Docker/Dockerfile → Docker-connect/distributed/Dockerfile

@@ -18,4 +18,5 @@ FROM confluentinc/cp-server-connect-base:6.1.0
 ENV CONNECT_PLUGIN_PATH="/usr/share/java,/usr/share/confluent-hub-components"
 
 ARG CONNECTOR_NAME
-RUN confluent-hub install --no-prompt confluentinc/${CONNECTOR_NAME}:6.0.0
+ARG CONNECTOR_VERSION
+RUN confluent-hub install --no-prompt confluentinc/${CONNECTOR_NAME}:${CONNECTOR_VERSION}

Docker-connect/standalone/Dockerfile

@@ -0,0 +1,28 @@
+#
+# Copyright 2020 Confluent Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM confluentinc/cp-server-connect-base:6.1.0
+
+ENV CONNECT_PLUGIN_PATH="/usr/share/java,/usr/share/confluent-hub-components"
+
+USER root
+COPY --chown=appuser:appuser include/etc/confluent/docker /etc/confluent/docker
+USER appuser
+
+ARG CONNECTOR_NAME
+ARG CONNECTOR_VERSION
+RUN confluent-hub install --no-prompt confluentinc/${CONNECTOR_NAME}:${CONNECTOR_VERSION}
+
+CMD ["/etc/confluent/docker/run"]

Docker-connect/standalone/include/etc/confluent/docker/configure

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+#
+# Copyright 2016 Confluent Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+. /etc/confluent/docker/bash-config
+
+# ensure STANDALONE (WORKER) properties
+dub ensure STANDALONE_BOOTSTRAP_SERVERS
+dub ensure STANDALONE_KEY_CONVERTER
+dub ensure STANDALONE_VALUE_CONVERTER
+# ensure STANDALONE (CONNECTOR) properties
+dub ensure CONNECT_CONNECTOR_CLASS
+dub ensure CONNECT_NAME
+
+# Default to 8083, which matches the mesos-overrides. This is here in case we extend the containers to remove the mesos overrides.
+# if [ -z "$CONNECT_REST_PORT" ]; then
+#   export CONNECT_REST_PORT=8083
+# fi
+
+# Fix for https://issues.apache.org/jira/browse/KAFKA-3988
+# if [[ ${CONNECT_INTERNAL_KEY_CONVERTER-} == "org.apache.kafka.connect.json.JsonConverter" ]] || [[ ${CONNECT_INTERNAL_VALUE_CONVERTER-} == "org.apache.kafka.connect.json.JsonConverter" ]]
+# then
+#   export CONNECT_INTERNAL_KEY_CONVERTER_SCHEMAS_ENABLE=false
+#   export CONNECT_INTERNAL_VALUE_CONVERTER_SCHEMAS_ENABLE=false
+# fi
+
+# if [[ $CONNECT_KEY_CONVERTER == "io.confluent.connect.avro.AvroConverter" ]]
+# then
+#   dub ensure CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL
+# fi
+
+# if [[ $CONNECT_VALUE_CONVERTER == "io.confluent.connect.avro.AvroConverter" ]]
+# then
+#   dub ensure CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL
+# fi
+
+if [[ $STANDALONE_KEY_CONVERTER == "io.confluent.connect.avro.AvroConverter" ]]
+then
+  dub ensure STANDALONE_KEY_CONVERTER_SCHEMA_REGISTRY_URL
+fi
+
+if [[ $STANDALONE_VALUE_CONVERTER == "io.confluent.connect.avro.AvroConverter" ]]
+then
+  dub ensure STANDALONE_VALUE_CONVERTER_SCHEMA_REGISTRY_URL
+fi
+
+dub path /etc/"${COMPONENT}"/ writable
+
+dub template "/etc/confluent/docker/${COMPONENT}.properties.template" "/etc/${COMPONENT}/${COMPONENT}.properties"
+dub template "/etc/confluent/docker/${COMPONENT}-standalone.properties.template" "/etc/${COMPONENT}/${COMPONENT}-standalone.properties"
+
+# The connect-distributed script expects the log4j config at /etc/kafka/connect-log4j.properties.
+dub template "/etc/confluent/docker/log4j.properties.template" "/etc/kafka/connect-log4j.properties"

Docker-connect/standalone/include/etc/confluent/docker/kafka-connect-standalone.properties.template

@@ -0,0 +1,4 @@
+{% set connect_props = env_to_props('STANDALONE_', '') -%}
+{% for name, value in connect_props.items() -%}
+{{name}}={{value}}
+{% endfor -%}

Docker-connect/standalone/include/etc/confluent/docker/launch

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+#
+# Copyright 2016 Confluent Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Override this section from the script to include the com.sun.management.jmxremote.rmi.port property.
+if [ -z "$KAFKA_JMX_OPTS" ]; then
+  export KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false  -Dcom.sun.management.jmxremote.ssl=false "
+fi
+
+# The JMX client needs to be able to connect to java.rmi.server.hostname.
+# The default for bridged n/w is the bridged IP so you will only be able to connect from another docker container.
+# For host n/w, this is the IP that the hostname on the host resolves to.
+
+# If you have more that one n/w configured, hostname -i gives you all the IPs,
+# the default is to pick the first IP (or network).
+export KAFKA_JMX_HOSTNAME=${KAFKA_JMX_HOSTNAME:-$(hostname -i | cut -d" " -f1)}
+
+if [ "$KAFKA_JMX_PORT" ]; then
+  # This ensures that the "if" section for JMX_PORT in kafka launch script does not trigger.
+  export JMX_PORT=$KAFKA_JMX_PORT
+  export KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=$KAFKA_JMX_HOSTNAME -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT -Dcom.sun.management.jmxremote.port=$JMX_PORT"
+fi
+
+echo "===> Launching ${COMPONENT} ... "
+# Add external jars to the classpath
+# And this also makes sure that the CLASSPATH does not start with ":/etc/..."
+# because this causes the plugin scanner to scan the entire disk.
+if [ -z "$CLASSPATH" ]; then
+    export CLASSPATH="/etc/kafka-connect/jars/*"
+fi
+exec connect-standalone /etc/"${COMPONENT}"/"${COMPONENT}"-standalone.properties /etc/"${COMPONENT}"/"${COMPONENT}".properties 

Docker-connect/standalone/include/etc/confluent/docker/run

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+. /etc/confluent/docker/bash-config
+
+echo "===> User"
+id
+
+echo "===> Configuring ..."
+/etc/confluent/docker/configure
+
+echo "===> Launching ... "
+exec /etc/confluent/docker/launch

cp-all-in-one-cloud/docker-compose.connect.local.yml → cp-all-in-one-cloud/docker-compose.connect.distributed.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
 
   connect:
-    image: localbuild/connect_custom_example:latest
+    image: localbuild/connect_distributed_with_${CONNECTOR_NAME}:${CONNECTOR_VERSION}
     hostname: connect
     container_name: connect
     ports:

cp-all-in-one-cloud/docker-compose.connect.standalone.yml

@@ -0,0 +1,42 @@
+---
+version: '2'
+services:
+
+  connect:
+    image: localbuild/connect_standalone_with_${CONNECTOR_NAME}:${CONNECTOR_VERSION}
+    hostname: connect
+    container_name: connect
+    environment:
+      # STANDALONE_ properties are for the Connect worker
+      STANDALONE_BOOTSTRAP_SERVERS: $BOOTSTRAP_SERVERS
+      STANDALONE_OFFSET_STORAGE_FILE_FILENAME: /tmp/connect.offsets
+      STANDALONE_OFFSET_FLUSH_INTERVAL_MS: 10000
+      STANDALONE_SECURITY_PROTOCOL: SASL_SSL
+      STANDALONE_SASL_JAAS_CONFIG: $SASL_JAAS_CONFIG
+      STANDALONE_SASL_MECHANISM: PLAIN
+      STANDALONE_PRODUCER_SECURITY_PROTOCOL: SASL_SSL
+      STANDALONE_PRODUCER_SASL_JAAS_CONFIG: $SASL_JAAS_CONFIG
+      STANDALONE_PRODUCER_SASL_MECHANISM: PLAIN
+      STANDALONE_CONSUMER_SECURITY_PROTOCOL: SASL_SSL
+      STANDALONE_CONSUMER_SASL_JAAS_CONFIG: $SASL_JAAS_CONFIG
+      STANDALONE_CONSUMER_SASL_MECHANISM: PLAIN
+      STANDALONE_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
+      STANDALONE_KEY_CONVERTER_SCHEMA_REGISTRY_URL: $SCHEMA_REGISTRY_URL
+      STANDALONE_KEY_CONVERTER_BASIC_AUTH_CREDENTIALS_SOURCE: USER_INFO
+      STANDALONE_KEY_CONVERTER_SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO: $SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO
+      STANDALONE_KEY_CONVERTER_SCHEMAS_ENABLE: "true"
+      STANDALONE_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
+      STANDALONE_VALUE_CONVERTER_SCHEMAS_ENABLE: "true"
+      STANDALONE_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: $SCHEMA_REGISTRY_URL
+      STANDALONE_VALUE_CONVERTER_BASIC_AUTH_CREDENTIALS_SOURCE: USER_INFO
+      STANDALONE_VALUE_CONVERTER_SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO: $SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO
+      STANDALONE_PLUGIN_PATH: /usr/share/confluent-hub-components/confluentinc-kafka-connect-elasticsearch/
+
+      # CONNECT_ properties are for the connector itself, example shown below
+      CONNECT_NAME: elasticsearch-sink-connector
+      CONNECT_CONNECTOR_CLASS: io.confluent.connect.elasticsearch.ElasticsearchSinkConnector
+      CONNECT_TYPE_NAME: _doc
+      CONNECT_TOPICS: test01
+      CONNECT_KEY_IGNORE: "true"
+      CONNECT_SCHEMA_IGNORE: "false"
+      CONNECT_CONNECTION_URL: http://elasticsearch:9200