i am working on my bachelor degree project and i have to use the Ballerina integrator middle-ware.
Now i need to make a service which will be able to index data coming to ballerina into Elasticsearch.Is there a way to communicate with Elasticsearch using only ballerina (without using Log stash or File beat..) just like we were communicating to a SQL database?
If there is someone looking for the same thing, i just found a way to communicate with Elastic and it's working very well
---here is the code---
import ballerina/http;
import ballerina/log;
listener http:Listener httpListener = new(9090);
http:Client elasticDB = new("http://localhost:9200/");
#http:ServiceConfig{
basePath: "/elastic"
}
service GetElasticSource on httpListener{
#http:ResourceConfig {
methods: ["GET"],
path: "/{index}/_source/{id}"
}
resource function retrieveElasticIndexById(http:Caller httpCaller, http:Request httpRequest, string index, string id){
http:Response resp = new;
var data = elasticDB->get(<#untained> ("/"+index+"/_source/"+id));
if(data is http:Response){
resp = data;
}
var respRet = httpCaller->respond(resp);
if(respRet is error){
log:printError("error responding to the client", err = respRet);
}
}
}
Related
I am trying to implement peer discovery logic through DNS in go using SRV records in cluster. I have headless service and statefulset pods ready and I am able to list all SRV records by using
kubectl run -it srvlookup --image=tutum/dnsutils --rm --restart=Never -- dig SRV demoapp.default.svc.cluster.local
but the following code does not work in cluster:
func pingdns() (url string) {
log.Println("start ping demoapp.default.svc.cluster.local.")
_, addrs, err := net.LookupSRV("dns-tcp", "tcp", "demoapp.default.svc.cluster.local")
if err != nil {
log.Println(err.Error())
return "dns wrong"
}
fmt.Println(addrs)
return "dns done."
}
error output:
lookup _dns-tcp._tcp.demoapp.default.svc.cluster.local on 10.96.0.10:53: no such host
I found example in this k8s-in-action book but it is written in NodeJS. How to do it in Golang ?
const dns = require('dns');
const dataFile = "/var/data/kubia.txt";
const serviceName = "kubia.default.svc.cluster.local";
const port = 8080;
...
var handler = function(request, response) {
if (request.method == 'POST') {
...
} else {
response.writeHead(200);
if (request.url == '/data') {
var data = fileExists(dataFile)
? fs.readFileSync(dataFile, 'utf8')
: "No data posted yet";
response.end(data);
} else {
response.write("You've hit " + os.hostname() + "\n");
response.write("Data stored in the cluster:\n");
dns.resolveSrv(serviceName, function (err, addresses) {
The app performs a DNS lookup to obtain SRV records.
if (err) {
response.end("Could not look up DNS SRV records: " + err);
return;
}
var numResponses = 0;
if (addresses.length == 0) {
response.end("No peers discovered.");
} else {
addresses.forEach(function (item) { ...
Thanks Shubham ! I read the medium post and found using grpc to make connection to SRV should browse all IPs as a RR fashion. But I am still looking for get all IPs.
Medium article:
https://medium.com/google-cloud/loadbalancing-grpc-for-kubernetes-cluster-services-3ba9a8d8fc03
Gitrepo:
https://github.com/jtattermusch/grpc-loadbalancing-kubernetes-examples#example-1-round-robin-loadbalancing-with-grpcs-built-in-loadbalancing-policy
import (
"google.golang.org/grpc/balancer/roundrobin"
"google.golang.org/grpc/credentials"
)
conn, err := grpc.Dial("dns:///be-srv-lb.default.svc.cluster.local", grpc.WithTransportCredentials(ce), grpc.WithBalancerName(roundrobin.Name))
c := echo.NewEchoServerClient(conn)
It makes calls on a list of IPs one by one. RR
Creating channel with target greeter-server.default.svc.cluster.local:8000
Greeting: Hello you (Backend IP: 10.0.2.95)
Greeting: Hello you (Backend IP: 10.0.0.74)
Greeting: Hello you (Backend IP: 10.0.1.51)
I have found my main issue is related to this problem.
Why golang Lookup*** function can't provide a server parameter?
It looks like you are looking for headless service.
Look for an example implementation: https://medium.com/google-cloud/loadbalancing-grpc-for-kubernetes-cluster-services-3ba9a8d8fc03
Kubernetes documentation: https://kubernetes.io/docs/concepts/services-networking/service/#headless-services for your reference.
To explain this: Kubernetes has a few implementations for services basically Endpoint based service and Headless service. Endpoint based services could be either with or without selectors. LoadBalancers are generally provided by Cloud providers.
Headless service are designed for client-side load balancing implementation. It looks like you are trying to implement your own DNS driven client-side load balancer.
Folks, I'm new to all this data streaming process but I was able to build and submit a Flink job that will read some CSV data from Kafka and aggregate it then put it in Elasticsearch.
I was able to do the first two parts, and print out my aggregation to STDOUT. But when I added the code to put it to Elasticsearch, it seems nothing is happening there (no data being added). I looked at the Flink job manager log and it looks fine (no errors) and says:
2020-03-03 16:18:03,877 INFO
org.apache.flink.streaming.connectors.elasticsearch7.Elasticsearch7ApiCallBridge
- Created Elasticsearch RestHighLevelClient connected to [http://elasticsearch-elasticsearch-coordinating-only.default.svc.cluster.local:9200]
Here is my code at this point:
/*
* This Scala source file was generated by the Gradle 'init' task.
*/
package flinkNamePull
import java.time.LocalDateTime
import java.util.Properties
import org.apache.flink.api.common.serialization.SimpleStringSchema
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.connectors.kafka.{FlinkKafkaConsumer010, FlinkKafkaProducer010}
import org.apache.flink.api.common.functions.RichMapFunction
import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
import org.apache.flink.table.api.{DataTypes, Table}
import org.apache.flink.table.api.scala.StreamTableEnvironment
import org.apache.flink.table.descriptors.{Elasticsearch, Json, Schema}
object Demo {
/**
* MapFunction to generate Transfers POJOs from parsed CSV data.
*/
class TransfersMapper extends RichMapFunction[String, Transfers] {
private var formatter = null
#throws[Exception]
override def open(parameters: Configuration): Unit = {
super.open(parameters)
//formatter = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss")
}
#throws[Exception]
override def map(csvLine: String): Transfers = {
//var splitCsv = csvLine.stripLineEnd.split("\n")(1).split(",")
var splitCsv = csvLine.stripLineEnd.split(",")
val arrLength = splitCsv.length
val i = 0
if (arrLength != 13) {
for (i <- arrLength + 1 to 13) {
if (i == 13) {
splitCsv = splitCsv :+ "0.0"
} else {
splitCsv = splitCsv :+ ""
}
}
}
var trans = new Transfers()
trans.rowId = splitCsv(0)
trans.subjectId = splitCsv(1)
trans.hadmId = splitCsv(2)
trans.icuStayId = splitCsv(3)
trans.dbSource = splitCsv(4)
trans.eventType = splitCsv(5)
trans.prev_careUnit = splitCsv(6)
trans.curr_careUnit = splitCsv(7)
trans.prev_wardId = splitCsv(8)
trans.curr_wardId = splitCsv(9)
trans.inTime = splitCsv(10)
trans.outTime = splitCsv(11)
trans.los = splitCsv(12).toDouble
return trans
}
}
def main(args: Array[String]) {
// Create streaming execution environment
val env = StreamExecutionEnvironment.getExecutionEnvironment
env.setParallelism(1)
// Set properties per KafkaConsumer API
val properties = new Properties()
properties.setProperty("bootstrap.servers", "kafka.kafka:9092")
properties.setProperty("group.id", "test")
// Add Kafka source to environment
val myKConsumer = new FlinkKafkaConsumer010[String]("raw.data3", new SimpleStringSchema(), properties)
// Read from beginning of topic
myKConsumer.setStartFromEarliest()
val streamSource = env
.addSource(myKConsumer)
// Transform CSV (with a header row per Kafka event into a Transfers object
val streamTransfers = streamSource.map(new TransfersMapper())
// create a TableEnvironment
val tEnv = StreamTableEnvironment.create(env)
println("***** NEW EXECUTION STARTED AT " + LocalDateTime.now() + " *****")
// register a Table
val tblTransfers: Table = tEnv.fromDataStream(streamTransfers)
tEnv.createTemporaryView("transfers", tblTransfers)
tEnv.connect(
new Elasticsearch()
.version("7")
.host("elasticsearch-elasticsearch-coordinating-only.default.svc.cluster.local", 9200, "http") // required: one or more Elasticsearch hosts to connect to
.index("transfers-sum")
.documentType("_doc")
.keyNullLiteral("n/a")
)
.withFormat(new Json().jsonSchema("{type: 'object', properties: {curr_careUnit: {type: 'string'}, sum: {type: 'number'}}}"))
.withSchema(new Schema()
.field("curr_careUnit", DataTypes.STRING())
.field("sum", DataTypes.DOUBLE())
)
.inUpsertMode()
.createTemporaryTable("transfersSum")
val result = tEnv.sqlQuery(
"""
|SELECT curr_careUnit, sum(los)
|FROM transfers
|GROUP BY curr_careUnit
|""".stripMargin)
result.insertInto("transfersSum")
// Elasticsearch elasticsearch-elasticsearch-coordinating-only.default.svc.cluster.local:9200
env.execute("Flink Streaming Demo Dump to Elasticsearch")
}
}
I'm not sure how I can debug this beast... Wondering if somebody can help me figure out why the Flink job is not adding data to Elasticsearch :(
From my Flink cluster, I'm able to query Elasticsearch just fine (manually) and add records to my index:
curl -XPOST "http://elasticsearch-elasticsearch-coordinating-only.default.svc.cluster.local:9200/transfers-sum/_doc" -H 'Content-Type: application/json' -d'{"curr_careUnit":"TEST123","sum":"123"}'
A kind soul in the Flink mailist pointed out the fact that it could be Elasticsearch buffering my records... Well, it was. ;)
I have added the following options to the Elasticsearch connector:
.bulkFlushMaxActions(2)
.bulkFlushInterval(1000L)
Flink Elasticsearch Connector 7 using Scala
Please find a working and detailed answer which I have provided here.
I am having fun with using moleculer-runner instead of creating a ServiceBroker instance in a moleculer-web project I am working on. The Runner simplifies setting up services for moleculer-web, and all the services - including the api.service.js file - look and behave the same, using a module.exports = { blah } format.
I can cleanly define the REST endpoints in the api.service.js file, and create the connected functions in the appropriate service files. For example aliases: { 'GET sensors': 'sensors.list' } points to the list() action/function in sensors.service.js . It all works great using some dummy data in an array.
The next step is to get the service(s) to open up a socket and talk to a local program listening on an internal set address/port. The idea is to accept a REST call from the web, talk to a local program over a socket to get some data, then format and return the data back via REST to the client.
BUT When I want to use sockets with moleculer, I'm having trouble finding useful info and examples on integrating moleculer-io with a moleculer-runner-based setup. All the examples I find use the ServiceBroker model. I thought my Google-Fu was pretty good, but I'm at a loss as to where to look to next. Or, can i modify the ServiceBroker examples to work with moleculer-runner? Any insight or input is welcome.
If you want the following chain:
localhost:3000/sensor/list -> sensor.list() -> send message to local program:8071 -> get response -> send response as return message to the REST caller.
Then you need to add a socket io client to your sensor service (which has the list() action). Adding a client will allow it to communicate with "outside world" via sockets.
Check the image below. I think it has everything that you need.
As a skeleton I've used moleculer-demo project.
What I have:
API service api.service.js. That handles the HTTP requests and passes them to the sensor.service.js
The sensor.service.js will be responsible for communicating with remote socket.io server so it needs to have a socket.io client. Now, when the sensor.service.js service has started() I'm establishing a connection with a remote server located at port 8071. After this I can use this connection in my service actions to communicate with socket.io server. This is exactly what I'm doing in sensor.list action.
I've also created remote-server.service.js to mock your socket.io server. Despite being a moleculer service, the sensor.service.js communicates with it via socket.io protocol.
It doesn't matter if your services use (or not) socket.io. All the services are declared in the same way, i.e., module.exports = {}
Below is a working example with socket.io.
const { ServiceBroker } = require("moleculer");
const ApiGateway = require("moleculer-web");
const SocketIOService = require("moleculer-io");
const io = require("socket.io-client");
const IOService = {
name: "api",
// SocketIOService should be after moleculer-web
// Load the HTTP API Gateway to be able to reach "greeter" action via:
// http://localhost:3000/hello/greeter
mixins: [ApiGateway, SocketIOService]
};
const HelloService = {
name: "hello",
actions: {
greeter() {
return "Hello Via Socket";
}
}
};
const broker = new ServiceBroker();
broker.createService(IOService);
broker.createService(HelloService);
broker.start().then(async () => {
const socket = io("http://localhost:3000", {
reconnectionDelay: 300,
reconnectionDelayMax: 300
});
socket.on("connect", () => {
console.log("Connection with the Gateway established");
});
socket.emit("call", "hello.greeter", (error, res) => {
console.log(res);
});
});
To make it work with moleculer-runner just copy the service declarations into my-service.service.js. So for example, your api.service.js could look like:
// api.service.js
module.exports = {
name: "api",
// SocketIOService should be after moleculer-web
// Load the HTTP API Gateway to be able to reach "greeter" action via:
// http://localhost:3000/hello/greeter
mixins: [ApiGateway, SocketIOService]
}
and your greeter service:
// greeter.service.js
module.exports = {
name: "hello",
actions: {
greeter() {
return "Hello Via Socket";
}
}
}
And run npm run dev or moleculer-runner --repl --hot services
I'm running my services in ballerina composer, then I'm using Windows cmd to invoke the service using the curl but cmd is complaining that curl is not a recognized command.
How could I do it in cmd?
Please help.
Create your service file, here it is hello_service.bal.
import ballerina/http;
endpoint http:Listener listener {
port:9090
};
service<http:Service> hello bind listener {
sayHello (endpoint caller, http:Request request) {
http:Response response = new;
response.setTextPayload("Hello World!\n");
_ = caller -> respond(response);
}
}
2. Run it on cmd. ballerina run hello_service.bal
3. Make a new main.bal file.
import ballerina/http;
import ballerina/log;
import ballerina/io;
endpoint http:Client clientEndpoint {
url: "http://localhost:9090"
};
function main(string... args) {
// Send a GET request to the Hello World service endpoint.
var response = clientEndpoint->get("/hello/sayHello");
match response {
http:Response resp => {
io:println(resp.getTextPayload());
}
error err => {
log:printError(err.message, err = err);
}
}
}
4. ballerina run main.bal.
5. You can see the result as Hello World! on cmd now.
You don't need to use cURL to invoke Ballerina HTTP services. You can use the HTTP client you normally use (e.g., Postman). If you really want to, you can install cURL in Windows as well: https://curl.haxx.se/download.html.
I need to write a tftp client implementation to send a file from a windows phone 8.1 to a piece of hardware.
Because I need to be able to support windows 8.1 I need to use the Windows.Networking.Sockets classes.
I'm able to send my Write request package but I am having troubles to receive the ack package (wireshark). This ack package is sent to an "ephemeral port" according to the TFTP specification but the port is blocked according to wireshark.
I know how to use sockets on a specific port but I don't know how to be able to receive ack packages send to different (ephemeral) ports. I need to use the port used for that ack package to continue the TFTP communication.
How would I be able to receive the ACK packages and continue to work on a different port? Do I need to bind the socket to multiple ports? I've been trying to find answers on the microsoft docs and google but other implementations gave me no luck so far.
As reference my current implementation:
try {
hostName = new Windows.Networking.HostName(currentIP);
} catch (error) {
WinJS.log && WinJS.log("Error: Invalid host name.", "sample", "error");
return;
}
socketsSample.clientSocket = new Windows.Networking.Sockets.DatagramSocket();
socketsSample.clientSocket.addEventListener("messagereceived", onMessageReceived);
socketsSample.clientSocket.bindEndpointAsync(new Windows.Networking.HostName(hostName), currentPort);
WinJS.log && WinJS.log("Client: connection started.", "sample", "status");
socketsSample.clientSocket.connectAsync(hostName, serviceName).done(function () {
WinJS.log && WinJS.log("Client: connection completed.", "sample", "status");
socketsSample.connected = true;
var remoteFile = "test.txt";
var tftpMode = Modes.Octet;
var sndBuffer = createRequestPacket(Opcodes.Write, remoteFile, tftpMode);
if (!socketsSample.clientDataWriter) {
socketsSample.clientDataWriter =
new Windows.Storage.Streams.DataWriter(socketsSample.clientSocket.outputStream);
}
var writer = socketsSample.clientDataWriter;
var reader;
var stream;
writer.writeBytes(sndBuffer);
// The call to store async sends the actual contents of the writer
// to the backing stream.
writer.storeAsync().then(function () {
// For the in-memory stream implementation we are using, the flushAsync call
// is superfluous, but other types of streams may require it.
return writer.flushAsync();
});
}, onError);
Finally found the issue.
Instead of connectAsynch I used getOutputStreamAsynch and now it receives messages on the client socket:
Some code:
tftpSocket.clientSocket.getOutputStreamAsync(new Windows.Networking.HostName(self.hostName), tftpSocket.serviceNameConnect).then(function (stream) {
console.log("Client: connection completed.", "sample", "status");
var writer = new Windows.Storage.Streams.DataWriter(stream); //use the stream that was created when calling getOutputStreamAsync
tftpSocket.clientDataWriter = writer; //keep the writer in case we need to close sockets we also close the writer
writer.writeBytes(sndBytes);
// The call to store async sends the actual contents of the writer
// to the backing stream.
writer.storeAsync().then(function () {
// For the in-memory stream implementation we are using, the flushAsync call
// is superfluous, but other types of streams may require it.
return writer.flushAsync();
});
}, self.onError);