Related
I'm using eureka-client, eureka-server, spring-cloud-starter-gateway and kafka to build my api. Using microservices, it works like this: the command sends a request to kafka for it to run, the Kafka that is installed on my machine and is not in a container. Command example:
#Autowired
private KafkaTemplate<String, ContactAdmSaveDto> kafkaTemplate;
#Override
public String create(ContactAdmSaveDto data) {
kafkaTemplate.send("contact-adm-insert", data);
return "Cadastrado com sucesso!";
}
application.properties command producer:
spring.kafka.producer.bootstrap-servers=springboot:9092
spring.kafka.producer.key-serializer=org.apache.kafka.common.serialization.StringSerializer
spring.kafka.producer.value-serializer=org.springframework.kafka.support.serializer.JsonSerializer
server.port = 30006
spring.application.name = contact-adm-command
eureka.client.serviceUrl.defaultZone = http://springboot:30002/eureka
eureka.instance.hostname=springboot
eureka.instance.prefer-ip-address=true
Example from Kafka:
#KafkaListener(topics = {"contact-adm-insert"}, groupId = "contact-adm")
public void consume(String record){
try {
ObjectMapper mapper = new ObjectMapper();
ContactAdm data = mapper.readValue(record, ContactAdm.class);
ContactAdm cat = new ContactAdm();
cat.setCell_phone(data.getCell_phone());
cat.setEmail(data.getEmail());
cat.setTelephone(data.getTelephone());
ContactAdm c = contactAdmRepository.save(cat);
ContactAdmMongo catm = new ContactAdmMongo();
catm.setCell_phone(data.getCell_phone());
catm.setEmail(data.getEmail());
catm.setTelephone(data.getTelephone());
catm.setContact_id(c.getContact_id());
contactAdmRepositoryMongo.save(catm);
} catch (Exception e) {
logger.info(e.toString());
}
}
application.properties kafka consumer:
server.port = 30005
spring.kafka.consumer.bootstrap-servers=springboot:9092
spring.kafka.consumer.auto-offset-reset=earliest
spring.kafka.consumer.key-deserializer=org.apache.kafka.common.serialization.StringDeserializer
spring.kafka.consumer.value-deserializer=org.apache.kafka.common.serialization.StringDeserializer
spring.kafka.consumer.group-id=contact-adm
springboot is a host name for my machine's ip
follow my gateway. Remembering that all kafka services are not registered in the gateway they are only to run when they are called by the command:
server.port=30000
spring.application.name=routing
eureka.client.serviceUrl.defaultZone=http://springboot:30002/eureka/
eureka.instance.hostname=springboot
eureka.instance.prefer-ip-address=true
#spring.cloud.gateway.discovery.locator.enabled=true
#spring.main.web-application-type=reactive
spring.cloud.gateway.enabled=true
spring.cloud.gateway.routes[0].id=user
spring.cloud.gateway.routes[0].uri=lb://USER
spring.cloud.gateway.routes[0].predicates=Path=/user/**
spring.cloud.gateway.routes[1].id=testes
spring.cloud.gateway.routes[1].uri=lb://TESTES
spring.cloud.gateway.routes[1].predicates=Path=/testes/**
spring.cloud.gateway.routes[2].id=user-command
spring.cloud.gateway.routes[2].uri=lb://USER-COMMAND
spring.cloud.gateway.routes[2].predicates=Path=/user-command/**
spring.cloud.gateway.routes[3].id=category-product-command
spring.cloud.gateway.routes[3].uri=lb://CATEGORY-PRODUCT-COMMAND
spring.cloud.gateway.routes[3].predicates=Path=/category-product-command/**
spring.cloud.gateway.routes[4].id=category-product-query
spring.cloud.gateway.routes[4].uri=lb://CATEGORY-PRODUCT-QUERY
spring.cloud.gateway.routes[4].predicates=Path=/category-product-query/**
spring.cloud.gateway.routes[5].id=cart-purchase-command
spring.cloud.gateway.routes[5].uri=lb://CART-PURCHASE-COMMAND
spring.cloud.gateway.routes[5].predicates=Path=/cart-purchase-command/**
spring.cloud.gateway.routes[6].id=cart-purchase-query
spring.cloud.gateway.routes[6].uri=lb://CART-PURCHASE-QUERY
spring.cloud.gateway.routes[6].predicates=Path=/cart-purchase-query/**
spring.cloud.gateway.routes[7].id=contact-adm-command
spring.cloud.gateway.routes[7].uri=lb://CONTACT-ADM-COMMAND
spring.cloud.gateway.routes[7].predicates=Path=/contact-adm-command/**
spring.cloud.gateway.routes[8].id=contact-adm-query
spring.cloud.gateway.routes[8].uri=lb://CONTACT-ADM-QUERY
spring.cloud.gateway.routes[8].predicates=Path=/contact-adm-query/**
This all works fine but I want to put it on kubernetes so I created the services images with the following command: mvn spring-boot:build-image and with the DockerFile:
FROM openjdk:17-alpine
EXPOSE 30000
ADD src/main/resources/routing/public.pem src/main/resources/routing/public.pem
ADD /target/routes-0.0.1-SNAPSHOT.jar routes-0.0.1-SNAPSHOT.jar
ENTRYPOINT ["java","-jar","routes-0.0.1-SNAPSHOT.jar"]
Generating all services images and placing them in the docker hub to be pulled by docker kubernetes with the following deployment:
apiVersion: apps/v1
kind: Deployment
metadata:
name: cart-purchase-kafka
labels:
app: cart-purchase-kafka
spec:
replicas: 1
# strategy:
# rollingUpdate:
# maxUnavailable: 0
# maxSurge: 1
selector:
matchLabels:
run: cart-purchase-kafka
template:
metadata:
labels:
run: cart-purchase-kafka
spec:
containers:
- name: cart-purchase-kafka
image: rafaelribeirosouza86/shopping:cart-purchase-kafka
imagePullPolicy: Always
ports:
- containerPort: 30011
protocol: TCP
imagePullSecrets:
- name: regcred
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: cart-purchase-command
labels:
app: cart-purchase-command
spec:
replicas: 1
# strategy:
# rollingUpdate:
# maxUnavailable: 0
# maxSurge: 1
selector:
matchLabels:
run: cart-purchase-command
template:
metadata:
labels:
run: cart-purchase-command
spec:
containers:
- name: cart-purchase-command
image: rafaelribeirosouza86/shopping:cart-purchase-command
imagePullPolicy: Always
ports:
- containerPort: 30012
protocol: TCP
imagePullSecrets:
- name: regcred
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: cart-purchase-query
labels:
app: cart-purchase-query
spec:
replicas: 1
# strategy:
# rollingUpdate:
# maxUnavailable: 0
# maxSurge: 1
selector:
matchLabels:
run: cart-purchase-query
template:
metadata:
labels:
run: cart-purchase-query
spec:
containers:
- name: cart-purchase-query
image: rafaelribeirosouza86/shopping:cart-purchase-query
imagePullPolicy: Always
ports:
- containerPort: 30010
protocol: TCP
imagePullSecrets:
- name: regcred
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: user
labels:
app: user
spec:
replicas: 1
# strategy:
# rollingUpdate:
# maxUnavailable: 0
# maxSurge: 1
selector:
matchLabels:
run: user
template:
metadata:
labels:
run: user
spec:
containers:
- name: user
image: rafaelribeirosouza86/shopping:user
imagePullPolicy: Always
ports:
- containerPort: 30015
protocol: TCP
imagePullSecrets:
- name: regcred
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: user-command
labels:
app: user-command
spec:
replicas: 1
# strategy:
# rollingUpdate:
# maxUnavailable: 0
# maxSurge: 1
selector:
matchLabels:
run: user-command
template:
metadata:
labels:
run: user-command
spec:
containers:
- name: user-command
image: rafaelribeirosouza86/shopping:user-command
imagePullPolicy: Always
ports:
- containerPort: 30004
protocol: TCP
imagePullSecrets:
- name: regcred
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: user-command-insert
labels:
app: user-command-insert
spec:
replicas: 1
# strategy:
# rollingUpdate:
# maxUnavailable: 0
# maxSurge: 1
selector:
matchLabels:
run: user-command-insert
template:
metadata:
labels:
run: user-command-insert
spec:
containers:
- name: user-command-insert
image: rafaelribeirosouza86/shopping:user-command-insert
imagePullPolicy: Always
ports:
- containerPort: 30003
protocol: TCP
imagePullSecrets:
- name: regcred
The big problem so far is that when I run it without Kubernetes it works fine but when I generate the pods it gives errors like:
NAME READY STATUS RESTARTS AGE
category-product-command-565f758d5d-4wwnf 0/1 Evicted 0 10m
category-product-command-565f758d5d-54pd5 0/1 Error 0 29m
category-product-command-565f758d5d-hmb8k 0/1 Pending 0 2m47s
category-product-command-565f758d5d-k6gmf 0/1 Evicted 0 10m
category-product-command-565f758d5d-lkd25 0/1 Error 0 41m
category-product-command-565f758d5d-ltbnl 0/1 Evicted 0 10m
category-product-command-565f758d5d-m7wwx 0/1 ContainerStatusUnknown 1 35m
category-product-command-565f758d5d-p42td 0/1 Error 0 54m
category-product-command-565f758d5d-pmfmh 0/1 Error 0 10m
category-product-command-565f758d5d-qbthd 0/1 Evicted 0 10m
category-product-command-565f758d5d-qf969 0/1 Evicted 0 10m
category-product-command-565f758d5d-twjvq 0/1 Evicted 0 10m
category-product-command-565f758d5d-vfrwq 0/1 ContainerStatusUnknown 1 22m
category-product-command-565f758d5d-xftpq 0/1 Error 0 47m
category-product-command-565f758d5d-xsg47 0/1 Evicted 0 10m
category-product-kafka-67d4fdbf76-262n8 0/1 Evicted 0 35m
category-product-kafka-67d4fdbf76-2klh8 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-2mgp8 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-2rlmm 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-2z57p 0/1 Evicted 0 35m
category-product-kafka-67d4fdbf76-424pj 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-4cnp2 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-4v586 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-5d7sg 0/1 Evicted 0 35m
category-product-kafka-67d4fdbf76-5mndm 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-5rcgg 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-5rlz7 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-69w7h 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-6czbj 0/1 Evicted 0 36m
category-product-kafka-67d4fdbf76-6rtvb 0/1 Evicted 0 35m
category-product-kafka-67d4fdbf76-6t4km 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-7pkd7 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-99z2b 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-9lfqq 0/1 Error 1 (42m ago) 53m
category-product-kafka-67d4fdbf76-9nrm4 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-bzx52 0/1 Evicted 0 35m
category-product-kafka-67d4fdbf76-d62b5 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-dbhp4 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-dscdk 0/1 Evicted 0 35m
category-product-kafka-67d4fdbf76-fnjdd 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-gnbnp 0/1 Evicted 0 35m
category-product-kafka-67d4fdbf76-gsrs8 0/1 Evicted 0 35m
category-product-kafka-67d4fdbf76-h69px 0/1 Evicted 0 35m
category-product-kafka-67d4fdbf76-hcljj 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-hmxmk 0/1 Evicted 0 35m
category-product-kafka-67d4fdbf76-hqngl 0/1 Evicted 0 35m
category-product-kafka-67d4fdbf76-j2bx2 0/1 Evicted 0 35m
category-product-kafka-67d4fdbf76-jjpkl 0/1 ContainerStatusUnknown 1 35m
category-product-kafka-67d4fdbf76-jqzlr 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-kbc25 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-khljn 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-kqht4 0/1 Error 0 54m
category-product-kafka-67d4fdbf76-kqxf5 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-l52p9 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-l8x4p 0/1 Evicted 0 35m
category-product-kafka-67d4fdbf76-ljhrm 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-m6l8c 0/1 Evicted 0 35m
category-product-kafka-67d4fdbf76-n49br 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-q4z79 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-qgqch 0/1 ContainerStatusUnknown 1 15m
category-product-kafka-67d4fdbf76-qjrf8 0/1 Evicted 0 35m
category-product-kafka-67d4fdbf76-qntzw 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-qv7s9 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-rkhq6 0/1 Evicted 0 35m
category-product-kafka-67d4fdbf76-rl2g6 0/1 Evicted 0 35m
category-product-kafka-67d4fdbf76-rl7dl 0/1 Evicted 0 35m
category-product-kafka-67d4fdbf76-sbpw6 0/1 Evicted 0 35m
category-product-kafka-67d4fdbf76-slww4 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-ssm24 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-txtjw 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-v9976 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-vl9gp 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-vns2z 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-vqcz9 0/1 Evicted 0 35m
category-product-kafka-67d4fdbf76-vst56 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-w5hpg 1/1 Running 0 8m53s
category-product-kafka-67d4fdbf76-w8tbb 0/1 Evicted 0 35m
category-product-kafka-67d4fdbf76-wpkwb 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-wvmtt 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-xp5t6 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-xtqwp 0/1 Evicted 0 23m
category-product-kafka-67d4fdbf76-z56s4 0/1 Error 0 23m
category-product-query-58897978b9-7csd7 1/1 Running 0 54m
contact-adm-command-56bb8f75db-9pvvz 1/1 Running 0 54m
contact-adm-kafka-858d968996-tgqkn 1/1 Running 0 54m
contact-adm-query-6b6b7487bb-2mqp6 1/1 Running 0 54m
gateway-7cbcb7bc4c-48b42 0/1 Pending 0 3m35s
gateway-7cbcb7bc4c-672mb 0/1 Evicted 0 42m
gateway-7cbcb7bc4c-d9hxn 0/1 ContainerStatusUnknown 1 42m
gateway-7cbcb7bc4c-g97cs 0/1 Error 0 16m
gateway-7cbcb7bc4c-hpntm 0/1 Evicted 0 42m
gateway-7cbcb7bc4c-js7nc 0/1 Evicted 0 42m
gateway-7cbcb7bc4c-lctsk 0/1 Error 0 30m
gateway-7cbcb7bc4c-stwbk 0/1 Evicted 0 42m
gateway-7cbcb7bc4c-zl4rb 0/1 Error 0 54m
routes-cb9ffbb47-tmmw9 1/1 Running 0
an error in the container:
Caused by: org.apache.hc.core5.http.NoHttpResponseException: springboot:30002 failed to respond
Does anyone have any idea what the problem might be?
[SOLVED]
I made it work with StatefulSet see how my deployments look:
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: gateway
spec:
serviceName: "gateway"
podManagementPolicy: "Parallel"
replicas: 1
selector:
matchLabels:
app: gateway
template:
metadata:
labels:
app: gateway
spec:
containers:
- name: gateway
#command: ["java", "-Djava.security.egd=file:/dev/./urandom", "-jar","gateway-0.0.1-SNAPSHOT.jar"]
image: rafaelribeirosouza86/shopping:gateway
imagePullPolicy: Always
ports:
- containerPort: 30002
imagePullSecrets:
- name: regcred
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: routes
spec:
serviceName: "routes"
podManagementPolicy: "Parallel"
replicas: 1
selector:
matchLabels:
app: routes
template:
metadata:
labels:
app: routes
spec:
containers:
- name: routes
#command: ["java", "-Djava.security.egd=file:/dev/./urandom", "-jar","routes-0.0.1-SNAPSHOT.jar"]
image: rafaelribeirosouza86/shopping:routes
imagePullPolicy: Always
ports:
- containerPort: 30000
imagePullSecrets:
- name: regcred
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: user
spec:
serviceName: "user"
podManagementPolicy: "Parallel"
replicas: 1
selector:
matchLabels:
app: user
template:
metadata:
labels:
app: user
spec:
containers:
- name: user
#command: ["java", "-Djava.security.egd=file:/dev/./urandom", "-jar","user-0.0.1-SNAPSHOT.jar"]
image: rafaelribeirosouza86/shopping:user
imagePullPolicy: Always
ports:
- containerPort: 30015
imagePullSecrets:
- name: regcred
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: user-command
spec:
serviceName: "user-command"
podManagementPolicy: "Parallel"
replicas: 1
selector:
matchLabels:
app: user-command
template:
metadata:
labels:
app: user-command
spec:
containers:
- name: user-command
#command: ["java", "-Djava.security.egd=file:/dev/./urandom", "-jar","user-command-0.0.1-SNAPSHOT.jar"]
image: rafaelribeirosouza86/shopping:user-command
imagePullPolicy: Always
ports:
- containerPort: 30004
imagePullSecrets:
- name: regcred
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: user-command-insert
spec:
serviceName: "user-command-insert"
podManagementPolicy: "Parallel"
replicas: 1
selector:
matchLabels:
app: user-command-insert
template:
metadata:
labels:
app: user-command-insert
spec:
containers:
- name: user-command-insert
#command: ["java", "-Djava.security.egd=file:/dev/./urandom", "-jar","user-command-insert-0.0.1-SNAPSHOT.jar"]
image: rafaelribeirosouza86/shopping:user-command-insert
imagePullPolicy: Always
ports:
- containerPort: 30003
imagePullSecrets:
- name: regcred
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: category-product-kafka
spec:
serviceName: "category-product-kafka"
podManagementPolicy: "Parallel"
replicas: 1
selector:
matchLabels:
app: category-product-kafka
template:
metadata:
labels:
app: category-product-kafka
spec:
containers:
- name: category-product-kafka
#command: ["java", "-Djava.security.egd=file:/dev/./urandom", "-jar","category-product-kafka-0.0.1-SNAPSHOT.jar"]
image: rafaelribeirosouza86/shopping:category-product-kafka
imagePullPolicy: Always
ports:
- containerPort: 30008
imagePullSecrets:
- name: regcred
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: category-product-command
spec:
serviceName: "category-product-command"
podManagementPolicy: "Parallel"
replicas: 1
selector:
matchLabels:
app: category-product-command
template:
metadata:
labels:
app: category-product-command
spec:
containers:
- name: category-product-command
#command: ["java", "-Djava.security.egd=file:/dev/./urandom", "-jar","category-product-command-0.0.1-SNAPSHOT.jar"]
image: rafaelribeirosouza86/shopping:category-product-command
imagePullPolicy: Always
ports:
- containerPort: 31533
imagePullSecrets:
- name: regcred
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: category-product-query
spec:
serviceName: "category-product-query"
podManagementPolicy: "Parallel"
replicas: 1
selector:
matchLabels:
app: category-product-query
template:
metadata:
labels:
app: category-product-query
spec:
containers:
- name: category-product-query
#command: ["java", "-Djava.security.egd=file:/dev/./urandom", "-jar","category-product-query-0.0.1-SNAPSHOT.jar"]
image: rafaelribeirosouza86/shopping:category-product-query
imagePullPolicy: Always
ports:
- containerPort: 30007
imagePullSecrets:
- name: regcred
Services:
apiVersion: v1
kind: Service
metadata:
name: gateway-service
namespace: default
spec:
# clusterIP: 10.99.233.224
ports:
- port: 30002
protocol: TCP
targetPort: 30002
nodePort: 30002
# externalTrafficPolicy: Local
type: NodePort
selector:
app: gateway
# type: LoadBalancer
#status:
# loadBalancer: {}
---
apiVersion: v1
kind: Service
metadata:
name: routes-service
namespace: default
spec:
# clusterIP: 10.99.233.224
# protocol: ##The default is TCP
# port: ##Exposes the service within the cluster. Also, other Pods use this to access the Service
# targetPort: ##The service sends request while containers accept traffic on this port.
ports:
- port: 30000
protocol: TCP
targetPort: 30000
nodePort: 30000
# externalTrafficPolicy: Local
type: NodePort
selector:
app: routes
# type: LoadBalancer
#status:
# loadBalancer: {}
# type: ClusterIP
# type: LoadBalancer
# type: NodePort
---
apiVersion: v1
kind: Service
metadata:
name: contact-adm-query-service
namespace: default
spec:
# clusterIP: 10.99.233.224
# protocol: ##The default is TCP
# port: ##Exposes the service within the cluster. Also, other Pods use this to access the Service
# targetPort: ##The service sends request while containers accept traffic on this port.
ports:
- port: 30001
protocol: TCP
targetPort: 30001
nodePort: 30001
# externalTrafficPolicy: Local
type: NodePort
selector:
app: contact-adm-query
# type: LoadBalancer
#status:
# loadBalancer: {}
# type: ClusterIP
# type: LoadBalancer
# type: NodePort
---
apiVersion: v1
kind: Service
metadata:
name: contact-adm-kafka-service
namespace: default
spec:
# clusterIP: 10.99.233.224
# protocol: ##The default is TCP
# port: ##Exposes the service within the cluster. Also, other Pods use this to access the Service
# targetPort: ##The service sends request while containers accept traffic on this port.
ports:
- port: 30005
protocol: TCP
targetPort: 30005
nodePort: 30005
# externalTrafficPolicy: Local
type: NodePort
selector:
app: contact-adm-kafka
---
apiVersion: v1
kind: Service
metadata:
name: contact-adm-command-service
namespace: default
spec:
# clusterIP: 10.99.233.224
# protocol: ##The default is TCP
# port: ##Exposes the service within the cluster. Also, other Pods use this to access the Service
# targetPort: ##The service sends request while containers accept traffic on this port.
ports:
- port: 30006
protocol: TCP
targetPort: 30006
nodePort: 30006
# externalTrafficPolicy: Local
type: NodePort
selector:
app: contact-adm-command
---
apiVersion: v1
kind: Service
metadata:
name: category-product-kafka-service
namespace: default
spec:
# clusterIP: 10.99.233.224
# protocol: ##The default is TCP
# port: ##Exposes the service within the cluster. Also, other Pods use this to access the Service
# targetPort: ##The service sends request while containers accept traffic on this port.
ports:
- port: 30008
protocol: TCP
targetPort: 30008
nodePort: 30008
# externalTrafficPolicy: Local
type: NodePort
selector:
app: category-product-kafka
---
apiVersion: v1
kind: Service
metadata:
name: category-product-command-service
namespace: default
spec:
# clusterIP: 10.99.233.224
# protocol: ##The default is TCP
# port: ##Exposes the service within the cluster. Also, other Pods use this to access the Service
# targetPort: ##The service sends request while containers accept traffic on this port.
ports:
- port: 31533
protocol: TCP
targetPort: 31533
nodePort: 31533
# externalTrafficPolicy: Local
type: NodePort
selector:
app: category-product-command
---
apiVersion: v1
kind: Service
metadata:
name: category-product-query-service
namespace: default
spec:
# clusterIP: 10.99.233.224
# protocol: ##The default is TCP
# port: ##Exposes the service within the cluster. Also, other Pods use this to access the Service
# targetPort: ##The service sends request while containers accept traffic on this port.
ports:
- port: 30007
protocol: TCP
targetPort: 30007
nodePort: 30007
# externalTrafficPolicy: Local
type: NodePort
selector:
app: category-product-query
---
apiVersion: v1
kind: Service
metadata:
name: cart-purchase-kafka-service
namespace: default
spec:
# clusterIP: 10.99.233.224
# protocol: ##The default is TCP
# port: ##Exposes the service within the cluster. Also, other Pods use this to access the Service
# targetPort: ##The service sends request while containers accept traffic on this port.
ports:
- port: 30011
protocol: TCP
targetPort: 30011
nodePort: 30011
# externalTrafficPolicy: Local
type: NodePort
selector:
app: cart-purchase-kafka
---
apiVersion: v1
kind: Service
metadata:
name: cart-purchase-command-service
namespace: default
spec:
# clusterIP: 10.99.233.224
# protocol: ##The default is TCP
# port: ##Exposes the service within the cluster. Also, other Pods use this to access the Service
# targetPort: ##The service sends request while containers accept traffic on this port.
ports:
- port: 30012
protocol: TCP
targetPort: 30012
nodePort: 30012
# externalTrafficPolicy: Local
type: NodePort
selector:
app: cart-purchase-command
---
apiVersion: v1
kind: Service
metadata:
name: cart-purchase-query-service
namespace: default
spec:
# clusterIP: 10.99.233.224
# protocol: ##The default is TCP
# port: ##Exposes the service within the cluster. Also, other Pods use this to access the Service
# targetPort: ##The service sends request while containers accept traffic on this port.
ports:
- port: 30010
protocol: TCP
targetPort: 30010
nodePort: 30010
# externalTrafficPolicy: Local
type: NodePort
selector:
app: cart-purchase-query
---
apiVersion: v1
kind: Service
metadata:
name: user-command-insert-service
namespace: default
spec:
# clusterIP: 10.99.233.224
# protocol: ##The default is TCP
# port: ##Exposes the service within the cluster. Also, other Pods use this to access the Service
# targetPort: ##The service sends request while containers accept traffic on this port.
ports:
- port: 30003
protocol: TCP
targetPort: 30003
nodePort: 30003
# externalTrafficPolicy: Local
type: NodePort
selector:
app: user-command-insert
---
apiVersion: v1
kind: Service
metadata:
name: user-command-service
namespace: default
spec:
# clusterIP: 10.99.233.224
# protocol: ##The default is TCP
# port: ##Exposes the service within the cluster. Also, other Pods use this to access the Service
# targetPort: ##The service sends request while containers accept traffic on this port.
ports:
- port: 30004
protocol: TCP
targetPort: 30004
nodePort: 30004
# externalTrafficPolicy: Local
type: NodePort
selector:
app: user-command
---
apiVersion: v1
kind: Service
metadata:
name: user-service
namespace: default
spec:
# clusterIP: 10.99.233.224
# protocol: ##The default is TCP
# port: ##Exposes the service within the cluster. Also, other Pods use this to access the Service
# targetPort: ##The service sends request while containers accept traffic on this port.
ports:
- port: 30015
protocol: TCP
targetPort: 30015
nodePort: 30015
# externalTrafficPolicy: Local
type: NodePort
selector:
app: user
I was testing on a kubernetes setup with AWS EKS on Fargate, and encountered an issue on the container startup.
It is a java application making use of hibernate. It seems it failed to connect to the MySQL server on startup, giving a "Communications link failure" error. The database server is running properly on AWS RDS, and the docker image can be run as expected in local.
I wonder if this is caused by the MySQL port 3306 not being configured properly on the container/node/service. Would like to see if you can spot out what the issue is and please don't hesitate to point out any mis-configuration, thank you very much.
Pod startup log
/\\ / ___'_ __ _ _(_)_ __ __ _ \ \ \ \
( ( )\___ | '_ | '_| | '_ \/ _` | \ \ \ \
\\/ ___)| |_)| | | | | || (_| | ) ) ) )
' |____| .__|_| |_|_| |_\__, | / / / /
=========|_|==============|___/=/_/_/_/
:: Spring Boot :: (v2.3.1.RELEASE)
2020-08-13 11:39:39.930 INFO 1 --- [ main] com.example.demo.DemoApplication : The following profiles are active: prod
2020-08-13 11:39:58.536 INFO 1 --- [ main] .s.d.r.c.RepositoryConfigurationDelegate : Bootstrapping Spring Data JPA repositories in DEFERRED mode.
...
......
2020-08-13 11:41:27.606 ERROR 1 --- [ task-1] com.zaxxer.hikari.pool.HikariPool : HikariPool-1 - Exception during pool initialization.
com.mysql.cj.jdbc.exceptions.CommunicationsException: Communications link failure
The last packet sent successfully to the server was 0 milliseconds ago. The driver has not received any packets from the server.
at com.mysql.cj.jdbc.exceptions.SQLError.createCommunicationsException(SQLError.java:174) ~[mysql-connector-java-8.0.20.jar!/:8.0.20]
at com.mysql.cj.jdbc.exceptions.SQLExceptionsMapping.translateException(SQLExceptionsMapping.java:64) ~[mysql-connector-java-8.0.20.jar!/:8.0.20]
at com.mysql.cj.jdbc.ConnectionImpl.createNewIO(ConnectionImpl.java:836) ~[mysql-connector-java-8.0.20.jar!/:8.0.20]
at com.mysql.cj.jdbc.ConnectionImpl.<init>(ConnectionImpl.java:456) ~[mysql-connector-java-8.0.20.jar!/:8.0.20]
at com.mysql.cj.jdbc.ConnectionImpl.getInstance(ConnectionImpl.java:246) ~[mysql-connector-java-8.0.20.jar!/:8.0.20]
at com.mysql.cj.jdbc.NonRegisteringDriver.connect(NonRegisteringDriver.java:197) ~[mysql-connector-java-8.0.20.jar!/:8.0.20]
at com.zaxxer.hikari.util.DriverDataSource.getConnection(DriverDataSource.java:138) ~[HikariCP-3.4.5.jar!/:na]
at com.zaxxer.hikari.pool.PoolBase.newConnection(PoolBase.java:358) ~[HikariCP-3.4.5.jar!/:na]
at com.zaxxer.hikari.pool.PoolBase.newPoolEntry(PoolBase.java:206) ~[HikariCP-3.4.5.jar!/:na]
at com.zaxxer.hikari.pool.HikariPool.createPoolEntry(HikariPool.java:477) [HikariCP-3.4.5.jar!/:na]
at com.zaxxer.hikari.pool.HikariPool.checkFailFast(HikariPool.java:560) [HikariCP-3.4.5.jar!/:na]
at com.zaxxer.hikari.pool.HikariPool.<init>(HikariPool.java:115) [HikariCP-3.4.5.jar!/:na]
at com.zaxxer.hikari.HikariDataSource.getConnection(HikariDataSource.java:112) [HikariCP-3.4.5.jar!/:na]
at org.hibernate.engine.jdbc.connections.internal.DatasourceConnectionProviderImpl.getConnection(DatasourceConnectionProviderImpl.java:122) [hibernate-core-5.4.17.Final.jar!/:5.4.17.Final]
at org.hibernate.engine.jdbc.env.internal.JdbcEnvironmentInitiator$ConnectionProviderJdbcConnectionAccess.obtainConnection(JdbcEnvironmentInitiator.java:180) [hibernate-core-5.4.17.Final.jar!/:5.4.17.Final]
at org.hibernate.engine.jdbc.env.internal.JdbcEnvironmentInitiator.initiateService(JdbcEnvironmentInitiator.java:68) [hibernate-core-5.4.17.Final.jar!/:5.4.17.Final]
at org.hibernate.engine.jdbc.env.internal.JdbcEnvironmentInitiator.initiateService(JdbcEnvironmentInitiator.java:35) [hibernate-core-5.4.17.Final.jar!/:5.4.17.Final]
at org.hibernate.boot.registry.internal.StandardServiceRegistryImpl.initiateService(StandardServiceRegistryImpl.java:101) [hibernate-core-5.4.17.Final.jar!/:5.4.17.Final]
at org.hibernate.service.internal.AbstractServiceRegistryImpl.createService(AbstractServiceRegistryImpl.java:263) [hibernate-core-5.4.17.Final.jar!/:5.4.17.Final]
at org.hibernate.service.internal.AbstractServiceRegistryImpl.initializeService(AbstractServiceRegistryImpl.java:237) [hibernate-core-5.4.17.Final.jar!/:5.4.17.Final]
at org.hibernate.service.internal.AbstractServiceRegistryImpl.getService(AbstractServiceRegistryImpl.java:214) [hibernate-core-5.4.17.Final.jar!/:5.4.17.Final]
at org.hibernate.id.factory.internal.DefaultIdentifierGeneratorFactory.injectServices(DefaultIdentifierGeneratorFactory.java:152) [hibernate-core-5.4.17.Final.jar!/:5.4.17.Final]
at org.hibernate.service.internal.AbstractServiceRegistryImpl.injectDependencies(AbstractServiceRegistryImpl.java:286) [hibernate-core-5.4.17.Final.jar!/:5.4.17.Final]
at org.hibernate.service.internal.AbstractServiceRegistryImpl.initializeService(AbstractServiceRegistryImpl.java:243) [hibernate-core-5.4.17.Final.jar!/:5.4.17.Final]
at org.hibernate.service.internal.AbstractServiceRegistryImpl.getService(AbstractServiceRegistryImpl.java:214) [hibernate-core-5.4.17.Final.jar!/:5.4.17.Final]
at org.hibernate.boot.internal.InFlightMetadataCollectorImpl.<init>(InFlightMetadataCollectorImpl.java:176) [hibernate-core-5.4.17.Final.jar!/:5.4.17.Final]
at org.hibernate.boot.model.process.spi.MetadataBuildingProcess.complete(MetadataBuildingProcess.java:118) [hibernate-core-5.4.17.Final.jar!/:5.4.17.Final]
at org.hibernate.jpa.boot.internal.EntityManagerFactoryBuilderImpl.metadata(EntityManagerFactoryBuilderImpl.java:1224) [hibernate-core-5.4.17.Final.jar!/:5.4.17.Final]
at org.hibernate.jpa.boot.internal.EntityManagerFactoryBuilderImpl.build(EntityManagerFactoryBuilderImpl.java:1255) [hibernate-core-5.4.17.Final.jar!/:5.4.17.Final]
at org.springframework.orm.jpa.vendor.SpringHibernateJpaPersistenceProvider.createContainerEntityManagerFactory(SpringHibernateJpaPersistenceProvider.java:58) [spring-orm-5.2.7.RELEASE.jar!/:5.2.7.RELEASE]
at org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean.createNativeEntityManagerFactory(LocalContainerEntityManagerFactoryBean.java:365) [spring-orm-5.2.7.RELEASE.jar!/:5.2.7.RELEASE]
at org.springframework.orm.jpa.AbstractEntityManagerFactoryBean.buildNativeEntityManagerFactory(AbstractEntityManagerFactoryBean.java:391) [spring-orm-5.2.7.RELEASE.jar!/:5.2.7.RELEASE]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_212]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_212]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_212]
at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_212]
...
......
Service
patricks-mbp:test patrick$ kubectl get services -n test
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
test NodePort 10.100.160.22 <none> 80:31176/TCP 4h57m
service.yaml
kind: Service
apiVersion: v1
metadata:
name: test
namespace: test
spec:
selector:
app: test
type: NodePort
ports:
- protocol: TCP
port: 80
targetPort: 8080
Deployment
patricks-mbp:test patrick$ kubectl get deployments -n test
NAME READY UP-TO-DATE AVAILABLE AGE
test 0/1 1 0 4h42m
deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: test
namespace: test
labels:
app: test
spec:
replicas: 1
selector:
matchLabels:
app: test
strategy: {}
template:
metadata:
labels:
app: test
spec:
containers:
- name: test
image: <image location>
ports:
- containerPort: 8080
resources: {}
Pods
patricks-mbp:test patrick$ kubectl get pods -n test
NAME READY STATUS RESTARTS AGE
test-8648f7959-4gdvm 1/1 Running 6 21m
patricks-mbp:test patrick$ kubectl describe pod test-8648f7959-4gdvm -n test
Name: test-8648f7959-4gdvm
Namespace: test
Priority: 2000001000
Priority Class Name: system-node-critical
Node: fargate-ip-192-168-123-170.ec2.internal/192.168.123.170
Start Time: Thu, 13 Aug 2020 21:29:07 +1000
Labels: app=test
eks.amazonaws.com/fargate-profile=fp-1a0330f1
pod-template-hash=8648f7959
Annotations: kubernetes.io/psp: eks.privileged
Status: Running
IP: 192.168.123.170
IPs:
IP: 192.168.123.170
Controlled By: ReplicaSet/test-8648f7959
Containers:
test:
Container ID: containerd://a1517a13d66274e1d7f8efcea950d0fe3d944d1f7208d057494e208223a895a7
Image: <image location>
Image ID: <image ID>
Port: 8080/TCP
Host Port: 0/TCP
State: Terminated
Reason: Error
Exit Code: 1
Started: Thu, 13 Aug 2020 21:48:07 +1000
Finished: Thu, 13 Aug 2020 21:50:28 +1000
Last State: Terminated
Reason: Error
Exit Code: 1
Started: Thu, 13 Aug 2020 21:43:04 +1000
Finished: Thu, 13 Aug 2020 21:45:22 +1000
Ready: False
Restart Count: 6
Environment: <none>
Mounts:
/var/run/secrets/kubernetes.io/serviceaccount from default-token-5hdzd (ro)
Conditions:
Type Status
Initialized True
Ready False
ContainersReady False
PodScheduled True
Volumes:
default-token-5hdzd:
Type: Secret (a volume populated by a Secret)
SecretName: default-token-5hdzd
Optional: false
QoS Class: BestEffort
Node-Selectors: <none>
Tolerations: node.kubernetes.io/not-ready:NoExecute for 300s
node.kubernetes.io/unreachable:NoExecute for 300s
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Scheduled <unknown> fargate-scheduler Successfully assigned test/test-8648f7959-4gdvm to fargate-ip-192-168-123-170.ec2.internal
Normal Pulling 21m kubelet, fargate-ip-192-168-123-170.ec2.internal Pulling image "174304792831.dkr.ecr.us-east-1.amazonaws.com/test:v2"
Normal Pulled 21m kubelet, fargate-ip-192-168-123-170.ec2.internal Successfully pulled image "174304792831.dkr.ecr.us-east-1.amazonaws.com/test:v2"
Normal Created 11m (x5 over 21m) kubelet, fargate-ip-192-168-123-170.ec2.internal Created container test
Normal Started 11m (x5 over 21m) kubelet, fargate-ip-192-168-123-170.ec2.internal Started container test
Normal Pulled 11m (x4 over 19m) kubelet, fargate-ip-192-168-123-170.ec2.internal Container image "174304792831.dkr.ecr.us-east-1.amazonaws.com/test:v2" already present on machine
Warning BackOff 11s (x27 over 17m) kubelet, fargate-ip-192-168-123-170.ec2.internal Back-off restarting failed container
Ingress
patricks-mbp:~ patrick$ kubectl describe ing -n test test
Name: test
Namespace: test
Address: <ALB public address>
Default backend: default-http-backend:80 (<none>)
Rules:
Host Path Backends
---- ---- --------
*
/ test:80 (192.168.72.15:8080)
Annotations:
alb.ingress.kubernetes.io/scheme: internet-facing
alb.ingress.kubernetes.io/target-type: ip
kubectl.kubernetes.io/last-applied-configuration: {"apiVersion":"extensions/v1beta1","kind":"Ingress","metadata":{"annotations":{"alb.ingress.kubernetes.io/scheme":"internet-facing","alb.ingress.kubernetes.io/target-type":"ip","kubernetes.io/ingress.class":"alb"},"name":"test","namespace":"test"},"spec":{"rules":[{"http":{"paths":[{"backend":{"serviceName":"test","servicePort":80},"path":"/"}]}}]}}
kubernetes.io/ingress.class: alb
Events: <none>
ingress.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: test
namespace: test
annotations:
kubernetes.io/ingress.class: alb
alb.ingress.kubernetes.io/target-type: ip
alb.ingress.kubernetes.io/scheme: internet-facing
spec:
rules:
- http:
paths:
- path: /
backend:
serviceName: test
servicePort: 80
AWS ALB ingress controller
Permission for ALB ingress controller to communicate with cluster
-> similar to https://raw.githubusercontent.com/kubernetes-sigs/aws-alb-ingress-controller/v1.1.8/docs/examples/rbac-role.yaml
Creation of Ingress Controller which uses ALB
-> similar to https://raw.githubusercontent.com/kubernetes-sigs/aws-alb-ingress-controller/v1.1.8/docs/examples/alb-ingress-controller.yaml
To allow pod from Fargate to connect to RDS you need to open security group.
Find the security group ID of your Fargate service
In your RDS security group rules, instead of putting a CIDR in your source field, put the Fargate service security group ID. Port 3306
I recently asked this question on how to upgrade Istio 1.1.11 from using http1.1 to http2.
I followed the advice and my resultant services YAML looks like this.
##################################################################################################
# Details service
##################################################################################################
apiVersion: v1
kind: Service
metadata:
name: details
labels:
app: details
service: details
spec:
ports:
- port: 9080
name: http2
selector:
app: details
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: details-v1
labels:
app: details
version: v1
spec:
replicas: 1
template:
metadata:
labels:
app: details
version: v1
spec:
containers:
- name: details
image: istio/examples-bookinfo-details-v1:1.13.0
imagePullPolicy: IfNotPresent
ports:
- containerPort: 9080
---
##################################################################################################
# Ratings service
##################################################################################################
apiVersion: v1
kind: Service
metadata:
name: ratings
labels:
app: ratings
service: ratings
spec:
ports:
- port: 9080
name: http2
selector:
app: ratings
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: ratings-v1
labels:
app: ratings
version: v1
spec:
replicas: 1
template:
metadata:
labels:
app: ratings
version: v1
spec:
containers:
- name: ratings
image: istio/examples-bookinfo-ratings-v1:1.13.0
imagePullPolicy: IfNotPresent
ports:
- containerPort: 9080
---
##################################################################################################
# Reviews service
##################################################################################################
apiVersion: v1
kind: Service
metadata:
name: reviews
labels:
app: reviews
service: reviews
spec:
ports:
- port: 9080
name: http2
selector:
app: reviews
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: reviews-v1
labels:
app: reviews
version: v1
spec:
replicas: 1
template:
metadata:
labels:
app: reviews
version: v1
spec:
containers:
- name: reviews
image: istio/examples-bookinfo-reviews-v1:1.13.0
imagePullPolicy: IfNotPresent
ports:
- containerPort: 9080
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: reviews-v2
labels:
app: reviews
version: v2
spec:
replicas: 1
template:
metadata:
labels:
app: reviews
version: v2
spec:
containers:
- name: reviews
image: istio/examples-bookinfo-reviews-v2:1.13.0
imagePullPolicy: IfNotPresent
ports:
- containerPort: 9080
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: reviews-v3
labels:
app: reviews
version: v3
spec:
replicas: 1
template:
metadata:
labels:
app: reviews
version: v3
spec:
containers:
- name: reviews
image: istio/examples-bookinfo-reviews-v3:1.13.0
imagePullPolicy: IfNotPresent
ports:
- containerPort: 9080
---
##################################################################################################
# Productpage services
##################################################################################################
apiVersion: v1
kind: Service
metadata:
name: productpage
labels:
app: productpage
service: productpage
spec:
ports:
- port: 9080
name: http2
selector:
app: productpage
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: productpage-v1
labels:
app: productpage
version: v1
spec:
replicas: 1
template:
metadata:
labels:
app: productpage
version: v1
spec:
containers:
- name: productpage
image: istio/examples-bookinfo-productpage-v1:1.13.0
imagePullPolicy: IfNotPresent
ports:
- containerPort: 9080
---
I successfully followed this tutorial to curl the service using HTTPS.
curl before:
curl -o /dev/null -s -v -w "%{http_code}\n" -HHost:localhost --resolve
localhost:$SECURE_INGRESS_PORT:$INGRESS_HOST --cacert example.com.crt -HHost:localhost https://localhost:443/productpage
* Address in 'localhost:443:localhost' found illegal!
* Trying ::1...
* TCP_NODELAY set
* Connected to localhost (::1) port 443 (#0)
* ALPN, offering h2
* ALPN, offering http/1.1
* Cipher selection: ALL:!EXPORT:!EXPORT40:!EXPORT56:!aNULL:!LOW:!RC4:#STRENGTH
* successfully set certificate verify locations:
* CAfile: example.com.crt
CApath: none
* TLSv1.2 (OUT), TLS handshake, Client hello (1):
} [215 bytes data]
* TLSv1.2 (IN), TLS handshake, Server hello (2):
{ [96 bytes data]
* TLSv1.2 (IN), TLS handshake, Certificate (11):
{ [740 bytes data]
* TLSv1.2 (IN), TLS handshake, Server key exchange (12):
{ [300 bytes data]
* TLSv1.2 (IN), TLS handshake, Server finished (14):
{ [4 bytes data]
* TLSv1.2 (OUT), TLS handshake, Client key exchange (16):
} [37 bytes data]
* TLSv1.2 (OUT), TLS change cipher, Client hello (1):
} [1 bytes data]
* TLSv1.2 (OUT), TLS handshake, Finished (20):
} [16 bytes data]
* TLSv1.2 (IN), TLS change cipher, Client hello (1):
{ [1 bytes data]
* TLSv1.2 (IN), TLS handshake, Finished (20):
{ [16 bytes data]
* SSL connection using TLSv1.2 / ECDHE-RSA-CHACHA20-POLY1305
* ALPN, server accepted to use h2
* Server certificate:
* subject: CN=localhost; O=Localhost organization
* start date: Jan 13 05:22:09 2020 GMT
* expire date: Jan 12 05:22:09 2021 GMT
* common name: localhost (matched)
* issuer: O=example Inc.; CN=example.com
* SSL certificate verify ok.
* Using HTTP2, server supports multi-use
* Connection state changed (HTTP/2 confirmed)
* Copying HTTP/2 data in stream buffer to connection buffer after upgrade: len=0
* Using Stream ID: 1 (easy handle 0x7fe244006400)
> GET /productpage HTTP/2
> Host:localhost
> User-Agent: curl/7.54.0
> Accept: */*
>
* Connection state changed (MAX_CONCURRENT_STREAMS updated)!
< HTTP/2 200
< content-type: text/html; charset=utf-8
< content-length: 4415
< server: istio-envoy
< date: Tue, 14 Jan 2020 03:22:30 GMT
< x-envoy-upstream-service-time: 1294
<
{ [4415 bytes data]
* Connection #0 to host localhost left intact
200
If I hit the service from a browser it works perfectly fine using url https://localhost/productpage
But, it stops working after I apply the above YAML. The browser just says
"upstream connect error or disconnect/reset before headers. reset reason: connection termination"
curl after:
curl -o /dev/null -s -v -w "%{http_code}\n" -HHost:localhost --resolve localhost:$SECURE_INGRESS_PORT:$INGRESS_HOST --cacert example.com.crt -HHost:localhost https://localhost:443/productpage
* Address in 'localhost:443:localhost' found illegal!
* Trying ::1...
* TCP_NODELAY set
* Connected to localhost (::1) port 443 (#0)
* ALPN, offering h2
* ALPN, offering http/1.1
* Cipher selection: ALL:!EXPORT:!EXPORT40:!EXPORT56:!aNULL:!LOW:!RC4:#STRENGTH
* successfully set certificate verify locations:
* CAfile: example.com.crt
CApath: none
* TLSv1.2 (OUT), TLS handshake, Client hello (1):
} [215 bytes data]
* TLSv1.2 (IN), TLS handshake, Server hello (2):
{ [96 bytes data]
* TLSv1.2 (IN), TLS handshake, Certificate (11):
{ [740 bytes data]
* TLSv1.2 (IN), TLS handshake, Server key exchange (12):
{ [300 bytes data]
* TLSv1.2 (IN), TLS handshake, Server finished (14):
{ [4 bytes data]
* TLSv1.2 (OUT), TLS handshake, Client key exchange (16):
} [37 bytes data]
* TLSv1.2 (OUT), TLS change cipher, Client hello (1):
} [1 bytes data]
* TLSv1.2 (OUT), TLS handshake, Finished (20):
} [16 bytes data]
* TLSv1.2 (IN), TLS change cipher, Client hello (1):
{ [1 bytes data]
* TLSv1.2 (IN), TLS handshake, Finished (20):
{ [16 bytes data]
* SSL connection using TLSv1.2 / ECDHE-RSA-CHACHA20-POLY1305
* ALPN, server accepted to use h2
* Server certificate:
* subject: CN=localhost; O=Localhost organization
* start date: Jan 13 05:22:09 2020 GMT
* expire date: Jan 12 05:22:09 2021 GMT
* common name: localhost (matched)
* issuer: O=example Inc.; CN=example.com
* SSL certificate verify ok.
* Using HTTP2, server supports multi-use
* Connection state changed (HTTP/2 confirmed)
* Copying HTTP/2 data in stream buffer to connection buffer after upgrade: len=0
* Using Stream ID: 1 (easy handle 0x7fe13a005200)
> GET /productpage HTTP/2
> Host:localhost
> User-Agent: curl/7.54.0
> Accept: */*
>
* Connection state changed (MAX_CONCURRENT_STREAMS updated)!
< HTTP/2 503
< content-length: 95
< content-type: text/plain
< date: Tue, 14 Jan 2020 03:16:49 GMT
< server: istio-envoy
< x-envoy-upstream-service-time: 57
<
{ [95 bytes data]
* Connection #0 to host localhost left intact
503
My destination rules look like this
(Note: It fails only if I change the above YAML, designation rules seem to be working just fine):
apiVersion: networking.istio.io/v1alpha3
kind: DestinationRule
metadata:
name: productpage
spec:
host: productpage
trafficPolicy:
connectionPool:
http:
h2UpgradePolicy: UPGRADE
tls:
mode: ISTIO_MUTUAL
subsets:
- name: v1
labels:
version: v1
---
apiVersion: networking.istio.io/v1alpha3
kind: DestinationRule
metadata:
name: reviews
spec:
host: reviews
trafficPolicy:
connectionPool:
http:
h2UpgradePolicy: UPGRADE
tls:
mode: ISTIO_MUTUAL
subsets:
- name: v1
labels:
version: v1
- name: v2
labels:
version: v2
- name: v3
labels:
version: v3
---
apiVersion: networking.istio.io/v1alpha3
kind: DestinationRule
metadata:
name: ratings
spec:
host: ratings
trafficPolicy:
connectionPool:
http:
h2UpgradePolicy: UPGRADE
tls:
mode: ISTIO_MUTUAL
subsets:
- name: v1
labels:
version: v1
- name: v2
labels:
version: v2
- name: v2-mysql
labels:
version: v2-mysql
- name: v2-mysql-vm
labels:
version: v2-mysql-vm
---
apiVersion: networking.istio.io/v1alpha3
kind: DestinationRule
metadata:
name: details
spec:
host: details
trafficPolicy:
connectionPool:
http:
h2UpgradePolicy: UPGRADE
tls:
mode: ISTIO_MUTUAL
subsets:
- name: v1
labels:
version: v1
- name: v2
labels:
version: v2
---
Few questions:
1) What could be the cause? How can I fix this? Is this a bug in Istio?
2) I'm able to hit the service from the browser before making the changes and I've read here that modern browsers only support HTTP2. Does that mean I'm automatically compliant to HTTP2? How to verify this?
3) How to gather the relevant logs to track what protocol is being used and for inter-pod communication?
The issue here is that You are most likely trying to serve HTTP (bookinfo app) content via HTTP2 protocol deployment/cluster configuration.
The bookinfo sample application from istio documentation does not support HTTP2 in its base configuration.
You can verify if You web-server supports HTTP2 protocol with this web tool: http2-test
From the other case You linked it appears You are looking into switching internal cluster communication from HTTP to HTTP2.
If You chose to continue going this path I suggest deploying service like nginx with with HTTP2 configuration similar to this found in nginx documentation for debugging purposes.
This can have alternative approach as described in google cloud documentation. In this case You can use HTTP as internal protocol in Your cluster configuration and web-server and then translate the traffic to HTTP2 on istio gateway/external loadbalancer.
We have a 6 node kafka cluster and one node 4 we are facing issue with consumer offset topic ex:
[root#kafka012-broker-shared-server2 kafka_2.11-1.1.1]# bin/kafka-topics.sh --describe --zookeeper localhost:2181 --topic __consumer_offsets | grep -w "Leader: 4"
Topic: __consumer_offsets Partition: 0 Leader: 4 Replicas: 1,3,5 Isr: 4
Topic: __consumer_offsets Partition: 4 Leader: 4 Replicas: 5,1,3 Isr: 4
Topic: __consumer_offsets Partition: 10 Leader: 4 Replicas: 5,1,6,2,3 Isr: 4
Topic: __consumer_offsets Partition: 12 Leader: 4 Replicas: 1,3,5,2 Isr: 4
Topic: __consumer_offsets Partition: 16 Leader: 4 Replicas: 5,1,3,6 Isr: 4
even these partitions folder are unavailable on /var/broker/data path. Kafka cluster is created on aws and uses EBS volumes. We have tried reassigning partitions to other nodes in cluster however it says assignment already done.
We are noticing following error :
Group coordinator kafka012-broker-shared-server2:9094 (id: 2147483643 rack: null) is unavailable or invalid, will attempt rediscovery – vamsi krishna 2 hours ago Delete
I have a SpringBoot project with graceful shutdown configured. Deployed on k8s 1.12.7 Here are the logs,
2019-07-20 10:23:16.180 INFO [service,,,] 1 --- [ Thread-7] com.jay.util.GracefulShutdown : Received shutdown event
2019-07-20 10:23:16.180 INFO [service,,,] 1 --- [ Thread-7] com.jay.util.GracefulShutdown : Waiting for 30s to finish
2019-07-20 10:23:16.273 INFO [service,fd964ebaa631a860,75a07c123397e4ff,false] 1 --- [io-8080-exec-10] com.jay.resource.ProductResource : GET /products?id=59
2019-07-20 10:23:16.374 INFO [service,9a569ecd8c448e98,00bc11ef2776d7fb,false] 1 --- [nio-8080-exec-1] com.jay.resource.ProductResource : GET /products?id=68
...
2019-07-20 10:23:33.711 INFO [service,1532d6298acce718,08cfb8085553b02e,false] 1 --- [nio-8080-exec-9] com.jay.resource.ProductResource : GET /products?id=209
2019-07-20 10:23:46.181 INFO [service,,,] 1 --- [ Thread-7] com.jay.util.GracefulShutdown : Resumed after hibernation
2019-07-20 10:23:46.216 INFO [service,,,] 1 --- [ Thread-7] o.s.s.concurrent.ThreadPoolTaskExecutor : Shutting down ExecutorService 'applicationTaskExecutor'
Application has received the SIGTERM at 10:23:16.180 from Kubernetes. As per Termination of Pods point#5 says that the terminating pod is removed from the endpoints list of service, but it is contradicting that it forwarded the requests for 17 seconds (until 10:23:33.711) after sending SIGTERM signal. Is there any configuration missing?
Dockerfile
FROM openjdk:8-jre-slim
MAINTAINER Jay
RUN apt update && apt install -y curl libtcnative-1 gcc && apt clean
ADD build/libs/sample-service.jar /
CMD ["java", "-jar" , "sample-service.jar"]
GracefulShutdown
// https://github.com/spring-projects/spring-boot/issues/4657
class GracefulShutdown(val waitTime: Long, val timeout: Long) : TomcatConnectorCustomizer, ApplicationListener<ContextClosedEvent> {
#Volatile
private var connector: Connector? = null
override fun customize(connector: Connector) {
this.connector = connector
}
override fun onApplicationEvent(event: ContextClosedEvent) {
log.info("Received shutdown event")
val executor = this.connector?.protocolHandler?.executor
if (executor is ThreadPoolExecutor) {
try {
val threadPoolExecutor: ThreadPoolExecutor = executor
log.info("Waiting for ${waitTime}s to finish")
hibernate(waitTime * 1000)
log.info("Resumed after hibernation")
this.connector?.pause()
threadPoolExecutor.shutdown()
if (!threadPoolExecutor.awaitTermination(timeout, TimeUnit.SECONDS)) {
log.warn("Tomcat thread pool did not shut down gracefully within $timeout seconds. Proceeding with forceful shutdown")
threadPoolExecutor.shutdownNow()
if (!threadPoolExecutor.awaitTermination(timeout, TimeUnit.SECONDS)) {
log.error("Tomcat thread pool did not terminate")
}
}
} catch (ex: InterruptedException) {
log.info("Interrupted")
Thread.currentThread().interrupt()
}
}else
this.connector?.pause()
}
private fun hibernate(time: Long){
try {
Thread.sleep(time)
}catch (ex: Exception){}
}
companion object {
private val log = LoggerFactory.getLogger(GracefulShutdown::class.java)
}
}
#Configuration
class GracefulShutdownConfig(#Value("\${app.shutdown.graceful.wait-time:30}") val waitTime: Long,
#Value("\${app.shutdown.graceful.timeout:30}") val timeout: Long) {
companion object {
private val log = LoggerFactory.getLogger(GracefulShutdownConfig::class.java)
}
#Bean
fun gracefulShutdown(): GracefulShutdown {
return GracefulShutdown(waitTime, timeout)
}
#Bean
fun webServerFactory(gracefulShutdown: GracefulShutdown): ConfigurableServletWebServerFactory {
log.info("GracefulShutdown configured with wait: ${waitTime}s and timeout: ${timeout}s")
val factory = TomcatServletWebServerFactory()
factory.addConnectorCustomizers(gracefulShutdown)
return factory
}
}
deployment file
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
labels:
k8s-app: service
name: service
spec:
progressDeadlineSeconds: 420
replicas: 1
revisionHistoryLimit: 1
selector:
matchLabels:
k8s-app: service
strategy:
rollingUpdate:
maxSurge: 2
maxUnavailable: 0
type: RollingUpdate
template:
metadata:
labels:
k8s-app: service
spec:
terminationGracePeriodSeconds: 60
containers:
- env:
- name: SPRING_PROFILES_ACTIVE
value: dev
image: service:2
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 20
httpGet:
path: /actuator/health
port: 8080
initialDelaySeconds: 60
periodSeconds: 30
timeoutSeconds: 5
name: service
ports:
- containerPort: 8080
protocol: TCP
readinessProbe:
failureThreshold: 60
httpGet:
path: /actuator/health
port: 8080
initialDelaySeconds: 100
periodSeconds: 10
timeoutSeconds: 5
UPDATE:
Added custom health check endpoint
#RestControllerEndpoint(id = "live")
#Component
class LiveEndpoint {
companion object {
private val log = LoggerFactory.getLogger(LiveEndpoint::class.java)
}
#Autowired
private lateinit var gracefulShutdownStatus: GracefulShutdownStatus
#GetMapping
fun live(): ResponseEntity<Any> {
val status = if(gracefulShutdownStatus.isTerminating())
HttpStatus.INTERNAL_SERVER_ERROR.value()
else
HttpStatus.OK.value()
log.info("Status: $status")
return ResponseEntity.status(status).build()
}
}
Changed the livenessProbe,
livenessProbe:
httpGet:
path: /actuator/live
port: 8080
initialDelaySeconds: 100
periodSeconds: 5
timeoutSeconds: 5
failureThreshold: 3
Here are the logs after the change,
2019-07-21 14:13:01.431 INFO [service,9b65b26907f2cf8f,9b65b26907f2cf8f,false] 1 --- [nio-8080-exec-2] com.jay.util.LiveEndpoint : Status: 200
2019-07-21 14:13:01.444 INFO [service,3da259976f9c286c,64b0d5973fddd577,false] 1 --- [nio-8080-exec-3] com.jay.resource.ProductResource : GET /products?id=52
2019-07-21 14:13:01.609 INFO [service,,,] 1 --- [ Thread-7] com.jay.util.GracefulShutdown : Received shutdown event
2019-07-21 14:13:01.610 INFO [service,,,] 1 --- [ Thread-7] com.jay.util.GracefulShutdown : Waiting for 30s to finish
...
2019-07-21 14:13:06.431 INFO [service,002c0da2133cf3b0,002c0da2133cf3b0,false] 1 --- [nio-8080-exec-3] com.jay.util.LiveEndpoint : Status: 500
2019-07-21 14:13:06.433 INFO [service,072abbd7275103ce,d1ead06b4abf2a34,false] 1 --- [nio-8080-exec-4] com.jay.resource.ProductResource : GET /products?id=96
...
2019-07-21 14:13:11.431 INFO [service,35aa09a8aea64ae6,35aa09a8aea64ae6,false] 1 --- [io-8080-exec-10] com.jay.util.LiveEndpoint : Status: 500
2019-07-21 14:13:11.508 INFO [service,a78c924f75538a50,0314f77f21076313,false] 1 --- [nio-8080-exec-2] com.jay.resource.ProductResource : GET /products?id=110
...
2019-07-21 14:13:16.431 INFO [service,38a940dfda03956b,38a940dfda03956b,false] 1 --- [nio-8080-exec-9] com.jay.util.LiveEndpoint : Status: 500
2019-07-21 14:13:16.593 INFO [service,d76e81012934805f,b61cb062154bb7f0,false] 1 --- [io-8080-exec-10] com.jay.resource.ProductResource : GET /products?id=152
...
2019-07-21 14:13:29.634 INFO [service,38a32a20358a7cc4,2029de1ed90e9539,false] 1 --- [nio-8080-exec-6] com.jay.resource.ProductResource : GET /products?id=191
2019-07-21 14:13:31.610 INFO [service,,,] 1 --- [ Thread-7] com.jay.util.GracefulShutdown : Resumed after hibernation
2019-07-21 14:13:31.692 INFO [service,,,] 1 --- [ Thread-7] o.s.s.concurrent.ThreadPoolTaskExecutor : Shutting down ExecutorService 'applicationTaskExecutor'
With the livenessProbe of 3 failures, kubernetes served the traffic for 13 seconds after liveness failures i.e., from 14:13:16.431 to 14:13:29.634.
UPDATE 2:
The sequence of events (thanks to Eamonn McEvoy)
seconds | healthy | events
0 | ✔ | * liveness probe healthy
1 | ✔ | - SIGTERM
2 | ✔ |
3 | ✔ |
4 | ✔ |
5 | ✔ | * liveness probe unhealthy (1/3)
6 | ✔ |
7 | ✔ |
8 | ✔ |
9 | ✔ |
10 | ✔ | * liveness probe unhealthy (2/3)
11 | ✔ |
12 | ✔ |
13 | ✔ |
14 | ✔ |
15 | ✘ | * liveness probe unhealthy (3/3)
.. | ✔ | * traffic is served
28 | ✔ | * traffic is served
29 | ✘ | * pod restarts
SIGTERM isn't putting the pod into a terminating state immediately. You can see in the logs your application begins graceful shutdown at 10:23:16.180 and takes >20 seconds to complete. At this point, the container stops and pod can enter the terminating state.
As far as kubernetes is concerned the pod looks ok during the graceful shutdown period. You need to add a liveness probe to your deployment; when it becomes unhealthy the traffic will stop.
livenessProbe:
httpGet:
path: /actuator/health
port: 8080
initialDelaySeconds: 100
periodSeconds: 10
timeoutSeconds: 5
Update:
This is because you have a failure threshold of 3, so you are allowing traffic for up to 15 seconds after the sigterm;
e.g.
seconds | healthy | events
0 | ✔ | * liveness probe healthy
1 | ✔ | - SIGTERM
2 | ✔ |
3 | ✔ |
4 | ✔ |
5 | ✔ | * liveness probe issued
6 | ✔ | .
7 | ✔ | .
8 | ✔ | .
9 | ✔ | .
10 | ✔ | * liveness probe timeout - unhealthy (1/3)
11 | ✔ |
12 | ✔ |
13 | ✔ |
14 | ✔ |
15 | ✔ | * liveness probe issued
16 | ✔ | .
17 | ✔ | .
18 | ✔ | .
19 | ✔ | .
20 | ✔ | * liveness probe timeout - unhealthy (2/3)
21 | ✔ |
22 | ✔ |
23 | ✔ |
24 | ✔ |
25 | ✔ | * liveness probe issued
26 | ✔ | .
27 | ✔ | .
28 | ✔ | .
29 | ✔ | .
30 | ✘ | * liveness probe timeout - unhealthy (3/3)
| | * pod restarts
This is assuming that the endpoint returns an unhealthy response during the graceful shutdown. Since you have timeoutSeconds: 5, if the probe simply times out this will take much longer, with a 5 second delay between issuing a liveness probe request and receiving its response. It could be the case that the container actually dies before the liveness threshold is hit and you are still seeing the original behaviour