This is on client side
socket.on('connect', () => {
console.log('client connect', socket.id);
const token = getToken();
socket.emit('token', token);
});
socket.on('message', data => {
....
//handle message
});
This is on server side
io.on('connection', (client) => {
client.on('token', token => {
verifyToken(token)
.then(({ _id: clientId }) => {
if (!clientId) return;
if (!connections[clientId]) {
connections[clientId] = new Map();
}
connections[clientId].set(client, 1);
client.on('disconnect', () => {
connections[clientId].delete(client);
});
});
});
});
}
async sendMessageToClients (workspaceId, message) {
const workspace = await getWorkspaceQuery(workspaceId);
if (!workspace) return;
const workspaceMembers = workspace.members.map(({ user }) => user);
for (const memberId of workspaceMembers) {
if (connections[memberId]) {
for (const clientConnection of connections[memberId].keys()) {
console.log('send to client', memberId, message.content, clientConnection.connected, clientConnection.id);
clientConnection.emit('message', message);
}
}
}
};
}
I purposely make a client offline by disconnect the wifi connection (make it in offline mode), what happen is that
a. if the disconnection is short, socket.id stay the same and I can get the buffered message send by other client when comes online;
b. but if I the disconnection is longer, the socket.id will change, and I can't get the buffered message send by other client when comes online.
How should I address that?
Since according to here the messages should be ideally buffered after reconnection.
Related
My ampq system seems loosing messages, so I'd like a way to see if messages are effectively queued before being consumed.
I have several MicroServices communicating by amqp messages on NodeJs, using CloudAmqp. One of this microservice MS[B] generates .pdf, the process it's pretty heavy and requires about a minute for each request. So I send the .pdf asyncronously, triggering a webhook once finished, and generate once per time using a PreFetch = 1
So one MS[A] collects all the requests from the user, answers back to them saying "ok, request received, listen on the webhook" and in parallel it asks to the MS[B] to generate pdfs. MS[B] has prefetch=1, so consumes just one request per time. Once finished, sends the response to the callback queue of MS[A], which triggers the user webhook saying "the pdf, it's ready".
The problem is that MS[B] misses all the messages while busy:
it consumes one request from MS[A]
starts generating the .pdf
while generating, it discards all the other messages that MS[A] sends, as if there would be not any queue
it finishes the .pdf, sending ACK to MS[A]
then it starts again accepting messages, taking the last one received after being idle, losing all the previous ones.
Why? How can I find the problem, what could I monitor?
Communications between other MSs works well, with messages correctly ordered in queues. Just this one, with prefetch=1, loses messages.
I am NOT using the NO-ACK rule. I don't know what try, what test and what monitor to find the problem.
How can I see (if) messages are correctly queued before being consumed, ora just lost?
Below, the implementation of the messaging system
Channel Creation
/*
Starting Point of a connection to the CLOUDAMQP_URL server, then exec the callback
*/
start(callback) {
var self = this;
// if the connection is closed or fails to be established at all, we will reconnect
amqp.connect(process.env.CLOUDAMQP_URL + "?heartbeat=60")
.then(
function (conn) {
// create queues and consume mechanism
self.amqpConn = conn;
setTimeout(() => {
startPublisher();
}, 200);
setTimeout(() => {
createCallbackQueue();
}, 1000);
setTimeout(() => {
callback();
}, 2000);
});
// create publisher channels
function startPublisher() {
self.amqpConn.createConfirmChannel()
.then(function (ch) {
self.pubChannel = ch;
logger.debug("🗣️ pubChannel ready");
while (true) {
var m = self.offlinePubQueue.shift();
if (!m) break;
self.publish(m[0], // exchange
m[1], // routingKey
m[2], // content,
undefined // correlationId
);
}
});
}
// create callback channel
function createCallbackQueue() {
self.amqpConn.createChannel()
.then(function (channel) {
channel.assertQueue(self.CALLBACK_QUEUE_NAME, {
durable: true,
exclusive: true, // callback are exclusive
})
.then(function (q) {
logger.debug(" 👂 Waiting for RPC RESP in " + self.CALLBACK_QUEUE_NAME);
channel.consume(q.queue,
processCallback, {
noAck: false
}
);
});
// process messages of the callback
function processCallback(msg) {
var correlationId = msg.properties.correlationId;
}
//callback received
if (self.correlationIds_map[correlationId]) {
delete self.correlationIds_map[correlationId];
var content = JSON.parse(msg.content.toString());
self.eventEmitter.emit(correlationId, content);
}
}
});
}
return deferred.promise;
}
Consuming Messages
/*
#worker_queue - the name of the queue
*/
// Consume message from 'worker_queue', A worker that acks messages only if processed succesfully
startWorker(worker_queue, routes) {
var self = this;
logger.debug("startWorker " + self.CALLBACK_QUEUE_NAME);
var channel;
worker_queue = self.MICROSERVICE_NAME + worker_queue;
self.amqpConn.createChannel()
.then(
function (ch) {
channel = ch;
ch.prefetch(self.opt.prefetch); // = 1 for MS[B] generating pdf
channel.assertQueue(worker_queue, {
durable: true,
exclusive: true
})
.then(function (q) {
channel.consume(worker_queue, processMsg, {
noAck: false
});
});
});
// call the 'function from interface' passing params, and send the ACK
function processMsg(msg) {
work(msg)
.then(function (data) {
channel.ack(msg, false); // allUpTo = false
})
.catch(function (err) {
channel.ack(msg, false);
// channel.reject(msg, false); // requeue = false
// this.closeOnErr(e);
});
}
// execute the command, and queue back a response, checking if it's an error or not
function work(msg) {
var deferred = Q.defer();
var correlationId;
try {
correlationId = msg.properties.correlationId;
} catch (err) {}
work_function(msg.content, correlationId)
.then(function (resp) {
var content = {
data: resp
};
content = Buffer.from(JSON.stringify(content));
channel.sendToQueue(msg.properties.replyTo,
content, {
correlationId: correlationId,
content_type: 'application/json'
}
);
deferred.resolve(resp);
});
return deferred.promise;
}
}
Publish Messages
publish(exchange, routingKey, content, correlationId) {
var self = this;
var deferred = Q.defer();
self.correlationIds_map[correlationId] = true;
self.pubChannel.publish(exchange, routingKey, content,
{
replyTo: self.CALLBACK_QUEUE_NAME,
content_type : 'application/json',
correlationId: correlationId,
persistent : true
},
function(err, ok) {
if (err)
{
self.offlinePubQueue.push([exchange, routingKey, content]); // try again
self.pubChannel.connection.close();
deferred.resolve('requeued');
}
else
{
deferred.resolve(ok);
}
});
return deferred.promise;
}
I have a websocket in api gateway connected to a lambda that looks like this:
const AWS = require('aws-sdk');
const amqp = require('amqplib');
const api = new AWS.ApiGatewayManagementApi({
endpoint: 'MY_ENDPOINT',
});
async function sendMsgToApp(response, connectionId) {
console.log('=========== posting reply');
const params = {
ConnectionId: connectionId,
Data: Buffer.from(response),
};
return api.postToConnection(params).promise();
}
let rmqServerUrl =
'MY_RMQ_SERVER_URL';
let rmqServerConn = null;
exports.handler = async event => {
console.log('websocket event:', event);
const { routeKey: route, connectionId } = event.requestContext;
switch (route) {
case '$connect':
console.log('user connected');
const creds = event.queryStringParameters.x;
console.log('============ x.length:', creds.length);
const decodedCreds = Buffer.from(creds, 'base64').toString('utf-8');
try {
const conn = await amqp.connect(
`amqps://${decodedCreds}#${rmqServerUrl}`
);
const channel = await conn.createChannel();
console.log('============ created channel successfully:');
rmqServerConn = conn;
const [userId] = decodedCreds.split(':');
const { queue } = await channel.assertQueue(userId, {
durable: true,
autoDelete: false,
});
console.log('============ userId:', userId, 'queue:', queue);
channel.consume(queue, msg => {
console.log('========== msg:', msg);
const { content } = msg;
const msgString = content.toString('utf-8');
console.log('========== msgString:', msgString);
sendMsgToApp(msgString, connectionId)
.then(res => {
console.log(
'================= sent queued message to the app, will ack, outcome:',
res
);
try {
channel.ack(msg);
} catch (e) {
console.log(
'================= error acking message:',
e
);
}
})
.catch(e => {
console.log(
'================= error sending queued message to the app, will not ack, error:',
e
);
});
});
} catch (e) {
console.log(
'=========== error initializing amqp connection',
e
);
if (rmqServerConn) {
await rmqServerConn.close();
}
const response = {
statusCode: 401,
body: JSON.stringify('failed auth!'),
};
return response;
}
break;
case '$disconnect':
console.log('user disconnected');
if (rmqServerConn) {
await rmqServerConn.close();
}
break;
case 'message':
console.log('message route');
await sendMsgToApp('test', connectionId);
break;
default:
console.log('unknown route', route);
break;
}
const response = {
statusCode: 200,
body: JSON.stringify('Hello from websocket Lambda!'),
};
return response;
};
The amqp connection is for a rabbitmq server that's provisioned by amazonmq. The problem I have is that messages published to the queue either do not show up at all in the .consume callback, or they only show up after the websocket is disconnected and reconnected. Essentially they're missing until a point much later after which they show up unexpectedly. That's within the websocket. Even when they do show up, they don't get sent to the client (app in this case) that's connected to the websocket. What could be the problem here?
The problem here is that I had the wrong idea about how API Gateway's websockets work. API gateway maintains the websocket connection but not the lambda itself. I put my .consume subscription logic inside the lambda, which doesn't work because the lambda runs and terminates instead of being kept alive. A better method would be to make the queue an event source for the lambda. However this also didn't work for me because it requires you to know your queues when setting up the lambda. My queues are dynamically created so that violated the requirement. I ended up standing up a rmq server on a vps.
I am creating an Electron app with Socket.io. When the user's computer goes into sleep mode the server disconnects from the client throwing an error "transport close". When the user tries to reconnect I check if the tokens are still valid, if they are not, I refresh them and try to send them to the socketIo server.
The problem I have is that on "reconnect_attempt" socket.io doesn't wait until I refresh the tokens to try reconnecting, it tries reconnecting right away with the old tokens, which get rejected by the server, which also seems to terminate the connection with the user impeding future reconnect attempts.
This is part of my code to connect to the server
module.exports.connect = async (JWT) => {
return new Promise( async resolve => {
console.log("connecting to the server")
const connectionOptions = {
secure: true,
query: {token: JWT},
reconnectionDelay: 4000
}
let socket = await socketIo.connect(`${process.env.SERVER_URL}:${process.env.SERVER_PORT}`, connectionOptions);
resolve(socket)
})
}
This is my code for reconnect_attempt
socket.on('reconnect_attempt', async () => {
const getCurrentJWT = require("../../main").getCurrentJWT;
let JWT = await getCurrentJWT(); //By the time this line returns, socket.io has already tried to reconnect
if(JWT.success) { //if refreshed successfully
console.log("Trying to submit new token......", JWT);
socket.query.token = JWT.JWT;
} else {
console.log("Token not refreshed.")
}
});
And this is part of what I have on the server
io.use(async (socket, next) => {
let token = socket.handshake.query.token;
//and the instruction from here https://docs.aws.amazon.com/cognito/latest/developerguide/amazon-cognito-user-pools-using-tokens-verifying-a-jwt.html
let tokenIsValid = await checkTokenValidity(token);
if( tokenIsValid ) {
next();
} else {
next(new Error('invalidToken'));
console.log("Not valid token")
}
})
In short, you can use auth for this.
While connecting
auth: {
token: token
}
In the time of reconnection
socket.auth.token = "NEW_TOKEN";
socket.connect();
I can share socket io implementation for this and you can modify it as your need.
For the client-side,
let unauthorized = false;
let socket = io.connect('ws://localhost:8080', {
transports: ["websocket"],
auth: {
token: GET_YOUR_TOKEN()
}
});
socket.on("connect", () => {
unauthorized = false;
});
socket.on('UNAUTHORIZED', () => {
unauthorized = true;
});
socket.on("disconnect", (reason) => {
if (reason === "io server disconnect") {
if(unauthorized) {
socket.auth.token = token;
}
socket.connect();
}
});
socket.on('PING', ()=>{
socket.emit('PONG', token);
});
For the server-side
io.on("connection", (socket) => {
socket.on('PONG', function (token) {
if (isValidToken(token) == false) {
socket.emit("UNAUTHORIZED");
socket.disconnect();
}
});
setInterval(() => {
socket.emit('PING');
}, <YOUR-TIME>);
});
Having the following in your server.
io.use( async function(socket, next) {
let address = socket.handshake.address;
run++; // 0 -> 1
// Validate Token
const token = socket.handshake.auth.token;
if(token !== undefined){
try{
await tokenVerify(token).then((payload) => {
const serverTimestamp = Math.floor(Date.now() / 1000);
const clientTimestamp = payload.exp;
if(clientTimestamp > serverTimestamp){
console.log("Connection from: " + address + " was accepted");
console.log("Token [" + token + "] from: " + address + " was accepted");
next();
}else{
console.log("Connection from: " + address + " was rejected");
console.log("Token [" + token + "] from: " + address + " was rejected");
next(new Error("unauthorized"));
}
});
}catch (e) {
console.log(e);
}
}
})
With the code above, the server will respond "unauthorized" if the token isn't valid.
So, on the client-side, we can catch that message as shown below.
socket_io.on("connect_error", (err) => {
if(err?.message === 'unauthorized'){
var timeout = (socket_reconnection_attempts === 0 ? 5000 : 60000)
console.log("Trying to reconnect in the next " + (timeout / 1000) + ' seconds')
setTimeout(function (){
console.log('Trying to reconnect manually')
socket_reconnection_attempts++;
loadAuthToken().then(function (token) {
socket_io.auth.token = token;
socket_io.connect();
})
}, timeout)
}
});
With the code above, the client-side will try to reconnect and refresh the token only if the error message is "unauthorized."
The variable "socket_reconnection_attempts" is to avoid sending a massive number of reconnection attempts in a short period of time.
Socket client is getting disconnected either due to transport close or pingtimeout error. And it happens randomly. Sometime the socket client is stable for couple of hours and after that is start disconnecting randomly.Can anyone help me finding the issue.
Socket-Client version : 2.1.0
Socket Server version : 2.1.0,
Client Code
const socket = require('socket.io-client')
let url = 'http://localhost:5050'
let clientSocket = socket.connect(url, {
reconnection: true,
forceNew: true,
secure: true
})
clientSocket.on("connect", function (data) {
// console.log(clientSocket)
console.log("connection established");
});
clientSocket.on("event", function(data) {
console.log(data)
})
Server Code
const socketio = require('socket.io');
this.io = socketio.listen(this.server,
{
'pingInterval': PING_INTERVAL,
'pingTimeout': PING_TIMEOUT
});
this.io.on('connection', function (socket) {
// const consumer = new ConsumerGroup(options, topic);
// reading data from add event and sending back the same data
console.log('Connected', socket.id);
const token = socket.handshake.query.token;
socket.on('disconnect', function () {
console.log(socket.id + ' -> Disconnected');
});
consumer.on('ready', function (message) {
console.log('Ready');
});
consumer.on('message', function (message) {
// sending message on socket when we recieve the message from kafka\
socket.emit('alarm', message);
});
consumer.on('error', function (err) {
console.log('error', err);
});
});
In the socket.io acknowledgement example we see a client's send/emit being called back with the server's response. Is the same functionality available in the reverse direction - i.e. how does the server confirm client reception for a send/emit from the server? It would be nice to have a send/emit callback even just to indicate reception success. Didn't see this functionality documented anywhere...
Thanks!
Looking in the socket.io source I found that indeed ACKs are supported for server-sent messages (but not in broadcasts!) (lines 115-123 of socket.io/lib/socket.js):
if ('function' == typeof args[args.length - 1]) {
if (this._rooms || (this.flags && this.flags.broadcast)) {
throw new Error('Callbacks are not supported when broadcasting');
}
debug('emitting packet with ack id %d', this.nsp.ids);
this.acks[this.nsp.ids] = args.pop();
packet.id = this.nsp.ids++;
}
An example of how the ack should work (not tested):
// server-side:
io.on('msg', (data, ackCallback) => {
console.log('data from client', data);
ackCallback('roger roger');
});
// client-side:
socket.emit('msg', someData, (answer) => {
console.log('server\'s acknowledgement:', answer);
});
If we want to be 100% sure the reception success, just add ack call is not enough because we also need to know whether the ack call is run or not.
The socket.io 3.0 document add this timeout example to show how to do that. But the timeout value is the tricky one.
const withTimeout = (onSuccess, onTimeout, timeout) => {
let called = false;
const timer = setTimeout(() => {
if (called) return;
called = true;
onTimeout();
}, timeout);
return (...args) => {
if (called) return;
called = true;
clearTimeout(timer);
onSuccess.apply(this, args);
}
}
socket.emit("hello", 1, 2, withTimeout(() => {
console.log("success!");
}, () => {
console.log("timeout!");
}, 1000));
Yes we can send a response back to the server from the Client(as acknowledgment )
According to new documentation of socket(4.x)
Server-side
let dataToSend={
test:"fromServer"
}
socket.timeout(5000).emit("my-event",dataToSend, (err, response) => {
if (err) {
// the other side did not acknowledge the event in the given delay
} else {
console.log(response);
}
});
Cleint- side
socket.on("my-event", (data, callback) => {
// any logic for data(that data come from server i.e { test:"fromServer" }
callback({ test: "test" });
});