I'm utilizing websockets for passing json messages but I don't want multiple ws connections if multiple tabs are open.
To reduce the connections I want to implement a single ws connection object that can send/receive messages to and from all tabs to my website. The object should forward the json to all tabs and each tab will process the message.
I've been looking at web/shared/service workers and I'm not sure the 2018 path to solve the issue and browser support seems to be a concern as well.
Looks like shared workers are not supported in Safari in support of service workers. Chrome/ff/opera seem to support shared workers.
In short it's a little confusing, bit of a mess and I want to know the best path forward with the best support.
If you know of a good resource, example code to implement ws with your suggested method please provide it as well.
After further research I've decided to implement web workers.
At this point I'm having success and I wanted to add an important piece that I got stuck on for future readers.
In my worker.js file I put this at the top to kick things off. The importScripts function threw an error if I didn't do it otherwise.
Also for the sake of helping, this is my skeleton code in my worker.js file that works. Message processing from the html pages are separated from the ws messages received from the server. You can start, stop the worker from the html page.
All tabs will get the messages from the worker, each page needs to process the messages as needed.
I'm also using robust-websockets so it auto reconnects from this github as this code works with web workers and is maintained. There is another project by the same name that isn't as updated by the time of this post. The reconnecting-websockets does not support web workers and you will get an error. - https://github.com/nathanboktae/robust-websocket
html
<script>
document.addEventListener('DOMContentLoaded', init);
function init(){
worker = new Worker('js/ws_worker.js');
worker.addEventListener('message', workerMessaged);
worker.postMessage({ "args": '<username_admin>', "cmd": 'start', "url": '127.0.0.1:8000' });
worker.postMessage({ "message": 'Initialize new worker'});
console.log('Message posted to worker, start');
}
// Received a json message from the Worker, process it.
function workerMessaged(ev){
console.log('Message received from worker');
console.log(ev.data);
worker.postMessage({ "cmd": 'message', "message": 'Sending reply over ws'});
}
worker.js
// proper initialization
if( 'function' === typeof importScripts) {
importScripts('robust-websocket.js');
var WebSocket;
self.addEventListener("message", function(e) {
var args = e.data.args;
var cmd = e.data.cmd;
var roomName = e.data.args;
var url = e.data.url;
var message = e.data;
// Code to process ws messages from the server
WebSocket.onmessage = function(event) {
console.log(" WebSocket message received: " + event.data, event);
self.postMessage(event.data);
};
WebSocket.onerror = function(event) {
console.log(" WebSocket message received: " + event.data, event);
self.postMessage(event.data);
};
if (cmd === 'start') {
WebSocket = new RobustWebSocket(
'ws://' + url +
'/ws/' + roomName + '/');
console.log('Connected via websockets');
/* Send initial message to open the connection and finalize the ws object*/
WebSocket.onopen = function() {
var obj = { "message": "hello" };
var json = JSON.stringify(obj);
WebSocket.send(json);
};
} else if (cmd === 'stop') {
WebSocket.onclose = function() {
console.log('Closing WebSocket');
WebSocket.close();
console.log('Closed WebSocket');
console.log('Terminating Worker');
self.close(); // Terminates the worker.
};
} else if (cmd === 'message') {
WebSocket.onopen = function() {
var json = JSON.stringify(message);
WebSocket.send(json);
};
console.log('message sent over websocket');
console.log('message');
} else {
console.log('logging error ' + e.data);
console.log(e.data);
self.postMessage('Unknown command: ');
}
}, false);
};
Related
My ampq system seems loosing messages, so I'd like a way to see if messages are effectively queued before being consumed.
I have several MicroServices communicating by amqp messages on NodeJs, using CloudAmqp. One of this microservice MS[B] generates .pdf, the process it's pretty heavy and requires about a minute for each request. So I send the .pdf asyncronously, triggering a webhook once finished, and generate once per time using a PreFetch = 1
So one MS[A] collects all the requests from the user, answers back to them saying "ok, request received, listen on the webhook" and in parallel it asks to the MS[B] to generate pdfs. MS[B] has prefetch=1, so consumes just one request per time. Once finished, sends the response to the callback queue of MS[A], which triggers the user webhook saying "the pdf, it's ready".
The problem is that MS[B] misses all the messages while busy:
it consumes one request from MS[A]
starts generating the .pdf
while generating, it discards all the other messages that MS[A] sends, as if there would be not any queue
it finishes the .pdf, sending ACK to MS[A]
then it starts again accepting messages, taking the last one received after being idle, losing all the previous ones.
Why? How can I find the problem, what could I monitor?
Communications between other MSs works well, with messages correctly ordered in queues. Just this one, with prefetch=1, loses messages.
I am NOT using the NO-ACK rule. I don't know what try, what test and what monitor to find the problem.
How can I see (if) messages are correctly queued before being consumed, ora just lost?
Below, the implementation of the messaging system
Channel Creation
/*
Starting Point of a connection to the CLOUDAMQP_URL server, then exec the callback
*/
start(callback) {
var self = this;
// if the connection is closed or fails to be established at all, we will reconnect
amqp.connect(process.env.CLOUDAMQP_URL + "?heartbeat=60")
.then(
function (conn) {
// create queues and consume mechanism
self.amqpConn = conn;
setTimeout(() => {
startPublisher();
}, 200);
setTimeout(() => {
createCallbackQueue();
}, 1000);
setTimeout(() => {
callback();
}, 2000);
});
// create publisher channels
function startPublisher() {
self.amqpConn.createConfirmChannel()
.then(function (ch) {
self.pubChannel = ch;
logger.debug("🗣️ pubChannel ready");
while (true) {
var m = self.offlinePubQueue.shift();
if (!m) break;
self.publish(m[0], // exchange
m[1], // routingKey
m[2], // content,
undefined // correlationId
);
}
});
}
// create callback channel
function createCallbackQueue() {
self.amqpConn.createChannel()
.then(function (channel) {
channel.assertQueue(self.CALLBACK_QUEUE_NAME, {
durable: true,
exclusive: true, // callback are exclusive
})
.then(function (q) {
logger.debug(" 👂 Waiting for RPC RESP in " + self.CALLBACK_QUEUE_NAME);
channel.consume(q.queue,
processCallback, {
noAck: false
}
);
});
// process messages of the callback
function processCallback(msg) {
var correlationId = msg.properties.correlationId;
}
//callback received
if (self.correlationIds_map[correlationId]) {
delete self.correlationIds_map[correlationId];
var content = JSON.parse(msg.content.toString());
self.eventEmitter.emit(correlationId, content);
}
}
});
}
return deferred.promise;
}
Consuming Messages
/*
#worker_queue - the name of the queue
*/
// Consume message from 'worker_queue', A worker that acks messages only if processed succesfully
startWorker(worker_queue, routes) {
var self = this;
logger.debug("startWorker " + self.CALLBACK_QUEUE_NAME);
var channel;
worker_queue = self.MICROSERVICE_NAME + worker_queue;
self.amqpConn.createChannel()
.then(
function (ch) {
channel = ch;
ch.prefetch(self.opt.prefetch); // = 1 for MS[B] generating pdf
channel.assertQueue(worker_queue, {
durable: true,
exclusive: true
})
.then(function (q) {
channel.consume(worker_queue, processMsg, {
noAck: false
});
});
});
// call the 'function from interface' passing params, and send the ACK
function processMsg(msg) {
work(msg)
.then(function (data) {
channel.ack(msg, false); // allUpTo = false
})
.catch(function (err) {
channel.ack(msg, false);
// channel.reject(msg, false); // requeue = false
// this.closeOnErr(e);
});
}
// execute the command, and queue back a response, checking if it's an error or not
function work(msg) {
var deferred = Q.defer();
var correlationId;
try {
correlationId = msg.properties.correlationId;
} catch (err) {}
work_function(msg.content, correlationId)
.then(function (resp) {
var content = {
data: resp
};
content = Buffer.from(JSON.stringify(content));
channel.sendToQueue(msg.properties.replyTo,
content, {
correlationId: correlationId,
content_type: 'application/json'
}
);
deferred.resolve(resp);
});
return deferred.promise;
}
}
Publish Messages
publish(exchange, routingKey, content, correlationId) {
var self = this;
var deferred = Q.defer();
self.correlationIds_map[correlationId] = true;
self.pubChannel.publish(exchange, routingKey, content,
{
replyTo: self.CALLBACK_QUEUE_NAME,
content_type : 'application/json',
correlationId: correlationId,
persistent : true
},
function(err, ok) {
if (err)
{
self.offlinePubQueue.push([exchange, routingKey, content]); // try again
self.pubChannel.connection.close();
deferred.resolve('requeued');
}
else
{
deferred.resolve(ok);
}
});
return deferred.promise;
}
I have succesfully implemented this mechanism in my application:
https://vividcode.io/Spring-5-WebFlux-with-Server-Sent-Events/
I can receive events with curl every second, as shown in the example.
My problem is: I cannot receive these events in Angular 5. I have tried many things. Currently my service code looks like this:
public getMigrationProgress(processName: string): Observable<any> {
let headers: HttpHeaders = new HttpHeaders();
headers = headers.append('X-Authorization', this._sessionService.getAuthToken());
headers = headers.append('accept', 'text/event-stream');
let url = config.restApi.url + this.getResource() + '/' + processName;
return Observable.create(observer => {
let eventSource = new EventSourcePolyfill(url, { headers: headers });
eventSource.onmessage = (event => {
observer.next(event);
this.zone.run(() => {
console.log('prpprpr');
});
});
eventSource.onopen = (event) => {
observer.next(event);
};
eventSource.onerror = (error) => {
if (eventSource.readyState === 0) {
console.log('The stream has been closed by the server.');
eventSource.close();
observer.complete();
} else {
observer.error('EventSource error: ' + error);
}
};
});
}
It only opens connection, does not receive events (Method onopen works once, onmessage - never). Server sends them though.
Any ideas how to fix this?
Turned out that if you set event name on server, you cannot receive it by onmessage method.
In the example the event name was set to "random". In order to receive it you have to do it like this:
eventSource.addEventListener('random', function (event) {
console.log(event);
});
I want to open a web-socket to a ColdFusion 2016 server, but I want to open it from HTML page (not cfm) so I don't have the option to use cfwebsocket tag.
what I want is a replacement for it..
I have tried the following code
var webSocket_IP = '192.168.1.223';
var chatSocket = new WebSocket("ws://"+webSocket_IP+":8579/cfusion/cfusion");
chatSocket.onopen = function () {
alert('OPEN');
};
chatSocket.onmessage = function () {
alert('a message was recieved');
};
chatSocket.onError = function () {
alert('Error');
};
the problem is that I cant open the connection and the onOpen method does not run
another problem is that when I want to subscribe to any channel
chatSocket.subscribeTo('chat');
I keep getting the following error
TypeError: chatSocket.subscribeTo is not a function
For clarification on Hamzeh's Answer.
How to establish a connection
var chatSocket = new WebSocket("ws://"+webSocket_IP+":8579/cfusion/cfusion");
How to subscribe to a channel
chatSocket.send(
JSON.stringify( {
appName: "customoptionexample1", //App Name
authKey: "739CAAF6CA8CA73DCCDB9305225F7D48",
ns: "coldfusion.websocket.channels",
subscribeTo: "bidchannel", //Channel subscribing to
type: "welcome"
} )
);
How to Send Data
chatSocket.send(
JSON.stringify( {
"ns": "coldfusion.websocket.channels",
"type": "publish",
"channel": "bidchannel", // Channel Name
"appName": "customoptionexample1", //App Name
"data": "Bid placed by adfadfadf Amount 66",
"customOptions": {
"value": "66"
}
} )
);
Setting up normal web socket callbacks
chatSocket.onopen = function() {
console.log( 'opened' );
};
chatSocket.onclose = function() {
console.log( 'onclose' );
};
chatSocket.onerror = function() {
console.log( 'onerror' );
};
chatSocket.onmessage = function( event ) {
//This parses the data and just prints the data and not the meta data.
console.log( 'onmessage', JSON.parse(event.data).data );
};
in case someone stumbled with the same issue , I have found the solution
first connect to coldfusion web socket path
var chatSocket = new WebSocket("ws://"+webSocket_IP+":8579/cfusion/cfusion");
then write the following command on the web socket object to subscribe to any channel
{"ns":"coldfusion.websocket.channels","type":"welcome","subscribeTo":"CHANNELNAME","appName":"APPNAME"}
and in case you want to write a message use the following:
{"ns":"coldfusion.websocket.channels","type":"publish","channel":"CHANNELNAME","data":"hi","appName":"APPNAME"}
I'm learning ZeroMQ and just went through the tutorial and a few examples. I'm using Node.js as my main environment ( with Python eventually used to replace my workers ).
Trying to sort out how I can create a fully asynchronous messaging system that will allow my API to push tasks ( via a REQ socket ) to a router, have a dealer pass the message to a worker, process the message and send its results back up to my client ( which is an Express route ).
I believe the pattern for this would work something like this ( haven't tested or properly implemented code yet, so please take it as a conceptual outline ):
router.js
const zmq = require('zmq');;
const frontend = zmq.socket('router');
const backend = zmq.socket('dealer');
frontend.on('message', function() {
var args = Array.apply(null, arguments);
backend.send(args);
});
backend.on('message', function() {
var args = Array.apply(null, arguments);
frontend.send(args);
});
frontend.bindSync('tcp://*:5559');
backend.bindSync('tcp://*:5560');
client.js
var zmq = require('zmq'),
var express = require('express');
var app = express();
app.post('send', function(req, res) {
var client = zmq.socket('req');
// listen for responses from the server
client.on('message', function(data) {
console.log(data);
client.close();
});
// connect to the server port
client.connect('tcp://0.0.0.0:5454');
client.send('Request from ' + process.id);
});
app.listen('80');
worker.js
var zmq = require('zmq');
var server = zmq.socket('rep');
server.on('message', function(d){
server.send('Response from ' + process.id);
});
// bind to port 5454
server.bind('tcp://0.0.0.0:5454', function(err){
if (err){
console.error("something bad happened");
console.error( err.msg );
console.error( err.stack );
process.exit(0);
}
});
What I'm not fully understanding is if the ROUTER/DEALER will handle sending the response worker to the correct client. Also in this case the Dealer handles the Fair Queueing as I want my work distributed amongst the workers evenly.
My client could be distributed amongst many different boxes ( load balancer API server ), my router will be on its own server and the workers would be distributed amongst multiple boxes as well.
Forget REQ/REP in any production-grade app, can fall in mutual deadlock
You might find this subject in many other posts on high-risk mutual FSM-FSM deadlocking in REQ/REP Formal Scalable Communication Pattern.
Be sure, XREQ/XREP == DEALER/ROUTER ( already since 2011 )
source code removes all hidden magics behind this, XREQ == DEALER and XREP == ROUTER
+++b/include/zmq.h
...
-#define ZMQ_XREQ 5
-#define ZMQ_XREP 6
+#define ZMQ_DEALER 5
+#define ZMQ_ROUTER 6
...
+#define ZMQ_XREQ ZMQ_DEALER /* Old alias, remove in 3.x */
+#define ZMQ_XREP ZMQ_ROUTER /* Old alias, remove in 3.x */
For anyone reading this in the future, in my further research I stumbled on the Majordomo Protocol/pattern. It's precisely what I'm trying to implement. Documentation on the implementation, benefits and disadvantages can be read here: https://rfc.zeromq.org/spec:18/MDP/. Here's the broker implementation: https://github.com/zeromq/majordomo
Seems like I was using DEALER/ROUTER when I should have been using XREQ and XREP.
broker.js
var zmq = require('zmq');
var frontPort = 'tcp://127.0.0.1:5559';
var backPort = 'tcp://127.0.0.1:5560';
var frontSocket = zmq.socket('xrep');
var backSocket = zmq.socket('xreq');
frontSocket.identity = 'xrep_' + process.pid;
backSocket.identity = 'xreq_' + process.pid;
frontSocket.bind(frontPort, function (err) {
console.log('bound', frontPort);
});
frontSocket.on('message', function() {
//pass to back
console.log('router: sending to server', arguments[0].toString(), arguments[2].toString());
backSocket.send(Array.prototype.slice.call(arguments));
});
backSocket.bind(backPort, function (err) {
console.log('bound', backPort);
});
backSocket.on('message', function() {
//pass to front
console.log('dealer: sending to client', arguments[0].toString(), arguments[2].toString());
frontSocket.send(Array.prototype.slice.call(arguments));
});
console.log('Broker started...');
worker.js
var zmq = require('zmq');
var socket = zmq.socket('rep');
socket.identity = 'worker_' + process.pid;
socket.on('message', function(data) {
console.log(socket.identity + ': received ' + data.toString());
socket.send(data * 2);
});
socket.connect('tcp://127.0.0.1:5560', function(err) {
if (err) throw err;
console.log('server connected!');
});
console.log('Worker started...');
client.js
var zmq = require('zmq');
var socket = zmq.socket('req');
socket.identity = 'client_' + process.pid;
socket.on('message', function(data) {
console.log(socket.identity + ': answer data ' + data);
});
socket.connect('tcp://127.0.0.1:5559');
setInterval(function() {
var value = Math.floor(Math.random()*100);
console.log(socket.identity + ': asking ' + value);
socket.send(value);
}, 100);
console.log('Client started...');
I'm still not sure if it's safe opening a connection on EVERY API inbound request.
I'm subscribing to 'queue:1.2.3'. How does the client know if they have successfully subscribed? Is there a mechanism for the server to respond to the client?
Below is a simple program that I'm using to test the subscriptions. I'm not sure what to show on the server side.
var sess;
var wsuri = 'ws://test.lan:8000';
window.onload = function() {
// connect to WAMP server
ab.connect(wsuri,
// WAMP session was established
function (session) {
sess = session;
console.log("Connected to " + wsuri);
},
// WAMP session is gone
function (code, reason) {
sess = null;
console.log("Connection lost (" + reason + ")");
}
);
};
function subscribe()
{
sess.subscribe("queue:1.2.3", topicReceived)
}
function topicReceived(topicUri, event)
{
console.log(topicUri + ' ' + event)
}
I feel like this post touches on my question: Calling dispatch in subscribe in Autobahn
Not in WAMPv1. A acknowledgement (with subsription ID) or a subscribe error (with error URI) will be part of WAMPv2 which is currently in the cooking.