The springboot service of the windows system cannot connect to the kafka started with docker-compose in the cloud server - spring-boot

Question

Please! Help!
I built a stand-alone kafka service through docker-compose on the cloud server (ip: 49.234.12.199).
docker-compose.yml
version: '3'
services:
zookeeper:
image: wurstmeister/zookeeper
ports:
- "2181:2181"
kafka:
image: wurstmeister/kafka
depends_on: [ zookeeper ]
ports:
- "9092:9092"
environment:
KAFKA_ADVERTISED_HOST_NAME: 49.234.12.199
KAFKA_CREATE_TOPICS: "test:1:1"
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://49.234.12.199:9092
KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
volumes:
- /var/run/docker.sock:/var/run/docker.sock
I can normally call port 9092 of 49.234.12.199 through the telnet command, and I can also create topics normally through commands inside the container.
But on my local windows system computer, the service started by springboot can't call the kafka service in docker normally, which makes me crazy.
The relevant configuration of springboot is as follows：
maven
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.1.6.RELEASE</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<artifactId>boot-kafka</artifactId>
<name>boot-kafka</name>
<description>Kafka demo project for Spring Boot</description>
<properties>
<java.version>1.8</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>
application.yml
server:
port: 9090
spring:
kafka:
bootstrap-servers: 49.234.12.199:9092
consumer:
auto-offset-reset: earliest
producer:
value-serializer: org.springframework.kafka.support.serializer.JsonSerializer
retries: 3
kafka:
topic:
my-topic: my-topic
my-topic2: my-topic2
KafkaConfig.class
package org.liu.demo.config;
import org.apache.kafka.clients.admin.NewTopic;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.support.converter.RecordMessageConverter;
import org.springframework.kafka.support.converter.StringJsonMessageConverter;
#Configuration
public class KafkaConfig {
#Value("${kafka.topic.my-topic}")
private String myTopic;
#Value("${kafka.topic.my-topic2}")
private String myTopic2;
#Bean
public RecordMessageConverter jsonConverter() {
return new StringJsonMessageConverter();
}
#Bean
public NewTopic myTopic() {
return new NewTopic(myTopic, 2, (short) 1);
}
#Bean
public NewTopic myTopic2() {
return new NewTopic(myTopic2, 1, (short) 1);
}
}
Finally start the error message
E:\Java\jdk1.8.0_152\bin\java.exe -agentlib:jdwp=transport=dt_socket,address=127.0.0.1:52858,suspend=y,server=n -XX:TieredStopAtLevel=1 -noverify -Dspring.output.ansi.enabled=always -Dcom.sun.management.jmxremote -Dspring.jmx.enabled=true -Dspring.liveBeansView.mbeanDomain -Dspring.application.admin.enabled=true -javaagent:C:\Users\yue.liu2\AppData\Local\JetBrains\IntelliJIdea2020.2\captureAgent\debugger-agent.jar -Dfile.encoding=UTF-8 -classpath "E:\Java\jdk1.8.0_152\jre\lib\charsets.jar;E:\Java\jdk1.8.0_152\jre\lib\deploy.jar;E:\Java\jdk1.8.0_152\jre\lib\ext\access-bridge-64.jar;E:\Java\jdk1.8.0_152\jre\lib\ext\cldrdata.jar;E:\Java\jdk1.8.0_152\jre\lib\ext\dnsns.jar;E:\Java\jdk1.8.0_152\jre\lib\ext\jaccess.jar;E:\Java\jdk1.8.0_152\jre\lib\ext\jfxrt.jar;E:\Java\jdk1.8.0_152\jre\lib\ext\localedata.jar;E:\Java\jdk1.8.0_152\jre\lib\ext\nashorn.jar;E:\Java\jdk1.8.0_152\jre\lib\ext\sunec.jar;E:\Java\jdk1.8.0_152\jre\lib\ext\sunjce_provider.jar;E:\Java\jdk1.8.0_152\jre\lib\ext\sunmscapi.jar;E:\Java\jdk1.8.0_152\jre\lib\ext\sunpkcs11.jar;E:\Java\jdk1.8.0_152\jre\lib\ext\zipfs.jar;E:\Java\jdk1.8.0_152\jre\lib\javaws.jar;E:\Java\jdk1.8.0_152\jre\lib\jce.jar;E:\Java\jdk1.8.0_152\jre\lib\jfr.jar;E:\Java\jdk1.8.0_152\jre\lib\jfxswt.jar;E:\Java\jdk1.8.0_152\jre\lib\jsse.jar;E:\Java\jdk1.8.0_152\jre\lib\management-agent.jar;E:\Java\jdk1.8.0_152\jre\lib\plugin.jar;E:\Java\jdk1.8.0_152\jre\lib\resources.jar;E:\Java\jdk1.8.0_152\jre\lib\rt.jar;E:\个人项目\boot-related-frame-study\boot-kafka\target\classes;E:\Maven\Repository\org\springframework\boot\spring-boot-starter-web\2.1.6.RELEASE\spring-boot-starter-web-2.1.6.RELEASE.jar;E:\Maven\Repository\org\springframework\boot\spring-boot-starter\2.1.6.RELEASE\spring-boot-starter-2.1.6.RELEASE.jar;E:\Maven\Repository\org\springframework\boot\spring-boot\2.1.6.RELEASE\spring-boot-2.1.6.RELEASE.jar;E:\Maven\Repository\org\springframework\boot\spring-boot-autoconfigure\2.1.6.RELEASE\spring-boot-autoconfigure-2.1.6.RELEASE.jar;E:\Maven\Repository\org\springframework\boot\spring-boot-starter-logging\2.1.6.RELEASE\spring-boot-starter-logging-2.1.6.RELEASE.jar;E:\Maven\Repository\ch\qos\logback\logback-classic\1.2.3\logback-classic-1.2.3.jar;E:\Maven\Repository\ch\qos\logback\logback-core\1.2.3\logback-core-1.2.3.jar;E:\Maven\Repository\org\apache\logging\log4j\log4j-to-slf4j\2.11.2\log4j-to-slf4j-2.11.2.jar;E:\Maven\Repository\org\apache\logging\log4j\log4j-api\2.11.2\log4j-api-2.11.2.jar;E:\Maven\Repository\org\slf4j\jul-to-slf4j\1.7.26\jul-to-slf4j-1.7.26.jar;E:\Maven\Repository\javax\annotation\javax.annotation-api\1.3.2\javax.annotation-api-1.3.2.jar;E:\Maven\Repository\org\yaml\snakeyaml\1.23\snakeyaml-1.23.jar;E:\Maven\Repository\org\springframework\boot\spring-boot-starter-json\2.1.6.RELEASE\spring-boot-starter-json-2.1.6.RELEASE.jar;E:\Maven\Repository\com\fasterxml\jackson\core\jackson-databind\2.9.9\jackson-databind-2.9.9.jar;E:\Maven\Repository\com\fasterxml\jackson\core\jackson-annotations\2.9.0\jackson-annotations-2.9.0.jar;E:\Maven\Repository\com\fasterxml\jackson\core\jackson-core\2.9.9\jackson-core-2.9.9.jar;E:\Maven\Repository\com\fasterxml\jackson\datatype\jackson-datatype-jdk8\2.9.9\jackson-datatype-jdk8-2.9.9.jar;E:\Maven\Repository\com\fasterxml\jackson\datatype\jackson-datatype-jsr310\2.9.9\jackson-datatype-jsr310-2.9.9.jar;E:\Maven\Repository\com\fasterxml\jackson\module\jackson-module-parameter-names\2.9.9\jackson-module-parameter-names-2.9.9.jar;E:\Maven\Repository\org\springframework\boot\spring-boot-starter-tomcat\2.1.6.RELEASE\spring-boot-starter-tomcat-2.1.6.RELEASE.jar;E:\Maven\Repository\org\apache\tomcat\embed\tomcat-embed-core\9.0.21\tomcat-embed-core-9.0.21.jar;E:\Maven\Repository\org\apache\tomcat\embed\tomcat-embed-el\9.0.21\tomcat-embed-el-9.0.21.jar;E:\Maven\Repository\org\apache\tomcat\embed\tomcat-embed-websocket\9.0.21\tomcat-embed-websocket-9.0.21.jar;E:\Maven\Repository\org\hibernate\validator\hibernate-validator\6.0.17.Final\hibernate-validator-6.0.17.Final.jar;E:\Maven\Repository\javax\validation\validation-api\2.0.1.Final\validation-api-2.0.1.Final.jar;E:\Maven\Repository\org\jboss\logging\jboss-logging\3.3.2.Final\jboss-logging-3.3.2.Final.jar;E:\Maven\Repository\com\fasterxml\classmate\1.4.0\classmate-1.4.0.jar;E:\Maven\Repository\org\springframework\spring-web\5.1.8.RELEASE\spring-web-5.1.8.RELEASE.jar;E:\Maven\Repository\org\springframework\spring-beans\5.1.8.RELEASE\spring-beans-5.1.8.RELEASE.jar;E:\Maven\Repository\org\springframework\spring-webmvc\5.1.8.RELEASE\spring-webmvc-5.1.8.RELEASE.jar;E:\Maven\Repository\org\springframework\spring-aop\5.1.8.RELEASE\spring-aop-5.1.8.RELEASE.jar;E:\Maven\Repository\org\springframework\spring-expression\5.1.8.RELEASE\spring-expression-5.1.8.RELEASE.jar;E:\Maven\Repository\org\springframework\kafka\spring-kafka\2.2.7.RELEASE\spring-kafka-2.2.7.RELEASE.jar;E:\Maven\Repository\org\springframework\spring-context\5.1.8.RELEASE\spring-context-5.1.8.RELEASE.jar;E:\Maven\Repository\org\springframework\spring-messaging\5.1.8.RELEASE\spring-messaging-5.1.8.RELEASE.jar;E:\Maven\Repository\org\springframework\spring-tx\5.1.8.RELEASE\spring-tx-5.1.8.RELEASE.jar;E:\Maven\Repository\org\springframework\retry\spring-retry\1.2.4.RELEASE\spring-retry-1.2.4.RELEASE.jar;E:\Maven\Repository\org\apache\kafka\kafka-clients\2.0.1\kafka-clients-2.0.1.jar;E:\Maven\Repository\org\lz4\lz4-java\1.4.1\lz4-java-1.4.1.jar;E:\Maven\Repository\org\xerial\snappy\snappy-java\1.1.7.1\snappy-java-1.1.7.1.jar;E:\Maven\Repository\org\slf4j\slf4j-api\1.7.26\slf4j-api-1.7.26.jar;E:\Maven\Repository\org\springframework\spring-core\5.1.8.RELEASE\spring-core-5.1.8.RELEASE.jar;E:\Maven\Repository\org\springframework\spring-jcl\5.1.8.RELEASE\spring-jcl-5.1.8.RELEASE.jar;D:\IntelliJ IDEA 2020.2\lib\idea_rt.jar" org.liu.demo.BootKafkaApplication
Connected to the target VM, address: '127.0.0.1:52858', transport: 'socket'
. ____ _ __ _ _
/\\ / ___'_ __ _ _(_)_ __ __ _ \ \ \ \
( ( )\___ | '_ | '_| | '_ \/ _` | \ \ \ \
\\/ ___)| |_)| | | | | || (_| | ) ) ) )
' |____| .__|_| |_|_| |_\__, | / / / /
=========|_|==============|___/=/_/_/_/
:: Spring Boot :: (v2.1.6.RELEASE)
2020-09-30 17:45:09.656 INFO 3268 --- [ main] org.liu.demo.BootKafkaApplication : Starting BootKafkaApplication on SH-CODE-PC0638 with PID 3268 (started by yue.liu2 in E:\个人项目\boot-related-frame-study)
2020-09-30 17:45:09.658 INFO 3268 --- [ main] org.liu.demo.BootKafkaApplication : No active profile set, falling back to default profiles: default
2020-09-30 17:45:10.179 INFO 3268 --- [ main] trationDelegate$BeanPostProcessorChecker : Bean 'org.springframework.kafka.annotation.KafkaBootstrapConfiguration' of type [org.springframework.kafka.annotation.KafkaBootstrapConfiguration$$EnhancerBySpringCGLIB$$880d02f5] is not eligible for getting processed by all BeanPostProcessors (for example: not eligible for auto-proxying)
2020-09-30 17:45:10.339 INFO 3268 --- [ main] o.s.b.w.embedded.tomcat.TomcatWebServer : Tomcat initialized with port(s): 9090 (http)
2020-09-30 17:45:10.354 INFO 3268 --- [ main] o.apache.catalina.core.StandardService : Starting service [Tomcat]
2020-09-30 17:45:10.354 INFO 3268 --- [ main] org.apache.catalina.core.StandardEngine : Starting Servlet engine: [Apache Tomcat/9.0.21]
2020-09-30 17:45:10.359 INFO 3268 --- [ main] o.a.catalina.core.AprLifecycleListener : Loaded APR based Apache Tomcat Native library [1.2.24] using APR version [1.7.0].
2020-09-30 17:45:10.359 INFO 3268 --- [ main] o.a.catalina.core.AprLifecycleListener : APR capabilities: IPv6 [true], sendfile [true], accept filters [false], random [true].
2020-09-30 17:45:10.359 INFO 3268 --- [ main] o.a.catalina.core.AprLifecycleListener : APR/OpenSSL configuration: useAprConnector [false], useOpenSSL [true]
2020-09-30 17:45:10.361 INFO 3268 --- [ main] o.a.catalina.core.AprLifecycleListener : OpenSSL successfully initialized [OpenSSL 1.1.1g 21 Apr 2020]
2020-09-30 17:45:10.419 INFO 3268 --- [ main] o.a.c.c.C.[Tomcat].[localhost].[/] : Initializing Spring embedded WebApplicationContext
2020-09-30 17:45:10.419 INFO 3268 --- [ main] o.s.web.context.ContextLoader : Root WebApplicationContext: initialization completed in 732 ms
2020-09-30 17:45:10.576 INFO 3268 --- [ main] o.s.s.concurrent.ThreadPoolTaskExecutor : Initializing ExecutorService 'applicationTaskExecutor'
2020-09-30 17:45:10.749 INFO 3268 --- [ main] o.a.k.clients.admin.AdminClientConfig : AdminClientConfig values:
bootstrap.servers = [49.234.12.199:9092]
client.id =
connections.max.idle.ms = 300000
metadata.max.age.ms = 300000
metric.reporters = []
metrics.num.samples = 2
metrics.recording.level = INFO
metrics.sample.window.ms = 30000
receive.buffer.bytes = 65536
reconnect.backoff.max.ms = 1000
reconnect.backoff.ms = 50
request.timeout.ms = 120000
retries = 5
retry.backoff.ms = 100
sasl.client.callback.handler.class = null
sasl.jaas.config = null
sasl.kerberos.kinit.cmd = /usr/bin/kinit
sasl.kerberos.min.time.before.relogin = 60000
sasl.kerberos.service.name = null
sasl.kerberos.ticket.renew.jitter = 0.05
sasl.kerberos.ticket.renew.window.factor = 0.8
sasl.login.callback.handler.class = null
sasl.login.class = null
sasl.login.refresh.buffer.seconds = 300
sasl.login.refresh.min.period.seconds = 60
sasl.login.refresh.window.factor = 0.8
sasl.login.refresh.window.jitter = 0.05
sasl.mechanism = GSSAPI
security.protocol = PLAINTEXT
send.buffer.bytes = 131072
ssl.cipher.suites = null
ssl.enabled.protocols = [TLSv1.2, TLSv1.1, TLSv1]
ssl.endpoint.identification.algorithm = https
ssl.key.password = null
ssl.keymanager.algorithm = SunX509
ssl.keystore.location = null
ssl.keystore.password = null
ssl.keystore.type = JKS
ssl.protocol = TLS
ssl.provider = null
ssl.secure.random.implementation = null
ssl.trustmanager.algorithm = PKIX
ssl.truststore.location = null
ssl.truststore.password = null
ssl.truststore.type = JKS
2020-09-30 17:45:10.780 INFO 3268 --- [ main] o.a.kafka.common.utils.AppInfoParser : Kafka version : 2.0.1
2020-09-30 17:45:10.780 INFO 3268 --- [ main] o.a.kafka.common.utils.AppInfoParser : Kafka commitId : fa14705e51bd2ce5
2020-09-30 17:45:49.400 ERROR 3268 --- [ main] o.springframework.kafka.core.KafkaAdmin : Could not configure topics
org.springframework.kafka.KafkaException: Timed out waiting to get existing topics; nested exception is java.util.concurrent.TimeoutException
at org.springframework.kafka.core.KafkaAdmin.lambda$checkPartitions$2(KafkaAdmin.java:238) [spring-kafka-2.2.7.RELEASE.jar:2.2.7.RELEASE]
at java.util.HashMap.forEach(HashMap.java:1289) ~[na:1.8.0_152]
at org.springframework.kafka.core.KafkaAdmin.checkPartitions(KafkaAdmin.java:213) [spring-kafka-2.2.7.RELEASE.jar:2.2.7.RELEASE]
at org.springframework.kafka.core.KafkaAdmin.addTopicsIfNeeded(KafkaAdmin.java:199) [spring-kafka-2.2.7.RELEASE.jar:2.2.7.RELEASE]
at org.springframework.kafka.core.KafkaAdmin.initialize(KafkaAdmin.java:169) [spring-kafka-2.2.7.RELEASE.jar:2.2.7.RELEASE]
at org.springframework.kafka.core.KafkaAdmin.afterSingletonsInstantiated(KafkaAdmin.java:139) [spring-kafka-2.2.7.RELEASE.jar:2.2.7.RELEASE]
at org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:862) [spring-beans-5.1.8.RELEASE.jar:5.1.8.RELEASE]
at org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:877) [spring-context-5.1.8.RELEASE.jar:5.1.8.RELEASE]
at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:549) [spring-context-5.1.8.RELEASE.jar:5.1.8.RELEASE]
at org.springframework.boot.web.servlet.context.ServletWebServerApplicationContext.refresh(ServletWebServerApplicationContext.java:140) [spring-boot-2.1.6.RELEASE.jar:2.1.6.RELEASE]
at org.springframework.boot.SpringApplication.refresh(SpringApplication.java:742) [spring-boot-2.1.6.RELEASE.jar:2.1.6.RELEASE]
at org.springframework.boot.SpringApplication.refreshContext(SpringApplication.java:389) [spring-boot-2.1.6.RELEASE.jar:2.1.6.RELEASE]
at org.springframework.boot.SpringApplication.run(SpringApplication.java:311) [spring-boot-2.1.6.RELEASE.jar:2.1.6.RELEASE]
at org.springframework.boot.SpringApplication.run(SpringApplication.java:1213) [spring-boot-2.1.6.RELEASE.jar:2.1.6.RELEASE]
at org.springframework.boot.SpringApplication.run(SpringApplication.java:1202) [spring-boot-2.1.6.RELEASE.jar:2.1.6.RELEASE]
at org.liu.demo.BootKafkaApplication.main(BootKafkaApplication.java:10) [classes/:na]
Caused by: java.util.concurrent.TimeoutException: null
at org.apache.kafka.common.internals.KafkaFutureImpl$SingleWaiter.await(KafkaFutureImpl.java:108) ~[kafka-clients-2.0.1.jar:na]
at org.apache.kafka.common.internals.KafkaFutureImpl.get(KafkaFutureImpl.java:274) ~[kafka-clients-2.0.1.jar:na]
at org.springframework.kafka.core.KafkaAdmin.lambda$checkPartitions$2(KafkaAdmin.java:216) [spring-kafka-2.2.7.RELEASE.jar:2.2.7.RELEASE]
... 15 common frames omitted
2020-09-30 17:45:59.401 INFO 3268 --- [| adminclient-1] o.a.k.clients.admin.KafkaAdminClient : [AdminClient clientId=adminclient-1] Forcing a hard I/O thread shutdown. Requests in progress will be aborted.
2020-09-30 17:45:59.421 INFO 3268 --- [ main] o.s.b.w.embedded.tomcat.TomcatWebServer : Tomcat started on port(s): 9090 (http) with context path ''
2020-09-30 17:45:59.423 INFO 3268 --- [ main] org.liu.demo.BootKafkaApplication : Started BootKafkaApplication in 50.077 seconds (JVM running for 50.591)

Answer 1

There seems to be an issue with the KAFKA_ADVERTISED_LISTENERS config. I am guessing you would need the public host name for the Docker cloud server.
The following article might help with configuring this correctly:
https://rmoff.net/2018/08/02/kafka-listeners-explained/