i'm having an issue configuring the sidekiq server, the process seems to be running in the foreground as soon as i refresh my page. /consumers/fetch i need to put it in the background permanently.
consumers_controller.rb
require 'kafka'
class ConsumersController < ApplicationController
def fetch
#consumer = Kafka::Consumer.new( { :host => ENV["host"],
:port => ENV["port"],
:topic => ENV["topic"]})
#consumer.loop do |message|
logger.info "-------------#{message.inspect}--------------"
logger.info "-------------#{message.first.payload.inspect}--------------"
unless message.blank?
ConsumerWorker.perform_async(message.first.payload)
end
end
end
end
consumer_worker.rb
class ConsumerWorker
include Sidekiq::Worker
def perform(message)
payload = message.first["payload"]
hash = JSON.parse(payload)
return #message = Message.new(hash) if hash["concern"] == 'order_create' or hash["concern"] == 'first_payment'
end
end
message.rb
class Message
attr_reader :bundle_id, :order_id, :order_number, :event
def initialize(message)
#payload = message["payload"]
#bundle_id = #payload["bundle_id"]
#order_id = #payload["order_id"]
#order_number = #payload["order_number"]
#event = message["concern"]
end
end
I think you need to move this block
#consumer.loop do |message|
end
inside your worker somehow, as I think the consumption is done after block execution.
Related
Using the following:
Hanami cookbook websockets
IoT Saga - Part 3 - Websockets! Connecting LiteCable to Hanami
I've been able to add WebSockets to Hanami, however as this is for production code I want to add specs; but I can't find information on how to test WebSockets and Hanami using Rspec.
I've been able to find this for RoR but nothing non-Rails specific or Hanami Specific, I have asked on the Hanami Gitter but not gotten a response yet.
Is the TCR gem the only way? I would prefer something simpler but If I must how would I set it up for anycable-go via litecable.
How can I test WebSockets for Hanami using Rspec?
To get this working requires several moving parts, the first is the Socket simulator which simulates the receiving socket on the webserver:
Note: url_path should be customized to what works for your web socket specific endpoint
# frozen_string_literal: true
require 'puma'
require 'lite_cable/server'
require_relative 'sync_client'
class SocketSimulator
def initialize(x_site_id_header: nil)
#server_logs = []
#x_site_id_header = x_site_id_header
end
attr_accessor :server_logs
def client
return #client if #client
url_path = "/ws?connection_token=#{connection_token}"
#client = SyncClient.new("ws://127.0.0.1:3099#{url_path}", headers: headers, cookies: '')
end
def connection_token
#connection_token ||= SecureRandom.hex
end
def user
return #user if #user
email = "#{SecureRandom.hex}#mailinator.com"
password = SecureRandom.hex
#user = Fabricate.create :user, email: email, site_id: site_id, password: password
end
def start
#server = Puma::Server.new(
LiteCable::Server::Middleware.new(nil, connection_class: Api::Sockets::Connection),
Puma::Events.strings
).tap do |server|
server.add_tcp_listener '127.0.0.1', 3099
server.min_threads = 1
server.max_threads = 4
end
#server_thread = Thread.new { #server.run.join }
end
def teardown
#server&.stop(true)
#server_thread&.join
#server_logs.clear
end
def headers
{
'AUTHORIZATION' => "Bearer #{jwt}",
'X_HANAMI_DIRECT_BOOKINGS_SITE_ID' => #x_site_id_header || site_id
}
end
def site_id
#site_id ||= SecureRandom.hex
end
def jwt
#jwt ||= Interactors::Users::GenerateJwt.new(user, site_id).call.jwt
end
end
The next thing is the SyncClient which is a fake client you can use to actually connect to the simulated socket:
# frozen_string_literal: true
# Synchronous websocket client
# Copied and modified from https://github.com/palkan/litecable/blob/master/spec/support/sync_client.rb
class SyncClient
require 'websocket-client-simple'
require 'concurrent'
require 'socket'
WAIT_WHEN_EXPECTING_EVENT = 5
WAIT_WHEN_NOT_EXPECTING_EVENT = 0.5
attr_reader :pings
def initialize(url, headers: {}, cookies: '')
#messages = Queue.new
#closed = Concurrent::Event.new
#has_messages = Concurrent::Semaphore.new(0)
#pings = Concurrent::AtomicFixnum.new(0)
#open = Concurrent::Promise.new
#ws = set_up_web_socket(url, headers.merge('COOKIE' => cookies))
#open.wait!(WAIT_WHEN_EXPECTING_EVENT)
end
def ip
Socket.ip_address_list.detect(&:ipv4_private?).try(:ip_address)
end
def set_up_web_socket(url, headers)
WebSocket::Client::Simple.connect(
url,
headers: headers
) do |ws|
ws.on(:error, &method(:on_error))
ws.on(:open, &method(:on_open))
ws.on(:message, &method(:on_message))
ws.on(:close, &method(:on_close))
end
end
def on_error(event)
event = RuntimeError.new(event.message) unless event.is_a?(Exception)
if #open.pending?
#open.fail(event)
else
#messages << event
#has_messages.release
end
end
def on_open(_event = nil)
#open.set(true)
end
def on_message(event)
if event.type == :close
#closed.set
else
message = JSON.parse(event.data)
if message['type'] == 'ping'
#pings.increment
else
#messages << message
#has_messages.release
end
end
end
def on_close(_event = nil)
#closed.set
end
def read_message
#has_messages.try_acquire(1, WAIT_WHEN_EXPECTING_EVENT)
msg = #messages.pop(true)
raise msg if msg.is_a?(Exception)
msg
end
def read_messages(expected_size = 0)
list = []
loop do
list_is_smaller = list.size < expected_size ? WAIT_WHEN_EXPECTING_EVENT : WAIT_WHEN_NOT_EXPECTING_EVENT
break unless #has_messages.try_acquire(1, list_is_smaller)
msg = #messages.pop(true)
raise msg if msg.is_a?(Exception)
list << msg
end
list
end
def send_message(message)
#ws.send(JSON.generate(message))
end
def close
sleep WAIT_WHEN_NOT_EXPECTING_EVENT
raise "#{#messages.size} messages unprocessed" unless #messages.empty?
#ws.close
wait_for_close
end
def wait_for_close
#closed.wait(WAIT_WHEN_EXPECTING_EVENT)
end
def closed?
#closed.set?
end
end
The last part is a fake channel to test against:
# frozen_string_literal: true
class FakeChannel < Api::Sockets::ApplicationChannel
identifier :fake
def subscribed
logger.info "Can Reject? #{can_reject?}"
reject if can_reject?
logger.debug "Streaming from #{stream_location}"
stream_from stream_location
end
def unsubscribed
transmit message: 'Goodbye channel!'
end
def can_reject?
logger.info "PARAMS: #{params}"
params.fetch('value_to_check', 0) > 5
end
def foo
transmit('bar')
end
end
To use in specs:
# frozen_string_literal: true
require_relative '../../../websockets-test-utils/fake_channel'
require_relative '../../../websockets-test-utils/socket_simulator'
RSpec.describe Interactors::Channels::Broadcast, db_truncation: true do
subject(:interactor) { described_class.new(token: connection_token, loc: 'fake', message: message) }
let(:identifier) { { channel: 'fake' }.to_json }
let(:socket_simulator) { SocketSimulator.new }
let(:client) { socket_simulator.client }
let(:user) { socket_simulator.user }
let(:connection_token) { socket_simulator.connection_token }
let(:channel) { 'fake' }
let(:message) { 'woooooo' }
before do
socket_simulator.start
end
after do
socket_simulator.teardown
end
describe 'call' do
before do
client.send_message command: 'subscribe',
identifier: identifier
end
it 'broadcasts a message to the correct channel' do
expect(client.read_message).to eq('type' => 'welcome')
expect(client.read_message).to eq(
'identifier' => identifier,
'type' => 'confirm_subscription'
)
interactor.call
expect(client.read_message).to eq(
'identifier' => identifier,
'message' => message
)
end
context 'with other connection' do
let(:user2) { Fabricate.create :user }
let(:jwt) { Interactors::Users::GenerateJwt.new(user2, site_id).call.jwt }
let(:site_id) { socket_simulator.site_id }
let(:url_path) { "/ws?connection_token=#{SecureRandom.hex}" }
let(:client2) { SyncClient.new("ws://127.0.0.1:3099#{url_path}", headers: {}, cookies: '') }
before do
client2.send_message command: 'subscribe',
identifier: identifier
end
it "doesn't broadcast to connections that shouldn't get it" do
aggregate_failures 'broadcast!' do
expect(client2.read_message).to eq('type' => 'welcome')
expect(client2.read_message).to eq(
'identifier' => identifier,
'type' => 'confirm_subscription'
)
expect(client.read_message).to eq('type' => 'welcome')
expect(client.read_message).to eq(
'identifier' => identifier,
'type' => 'confirm_subscription'
)
interactor.call
sleep 1
expect(client.read_message).to eq(
'identifier' => identifier,
'message' => message
)
expect { client2.close }.not_to raise_exception
end
end
end
end
end
I'm learning how to work with HTTParty and API and I'm having an issue with my code.
Users/admin/.rbenv/versions/2.0.0-p481/lib/ruby/2.0.0/uri/generic.rb:214:in `initialize': the scheme http does not accept registry part: :80 (or bad hostname?)
I've tried using debug_output STDOUT both as an argument to my method and after including HTTParty to have a clue but with no success. Nothing gets displayed:
require 'httparty'
class LolObserver
include HTTParty
default_timeout(1) #timeout after 1 second
attr_reader :api_key, :playerid
attr_accessor :region
def initialize(region,playerid,apikey)
#region = region_server(region)
#playerid = playerid
#api_key = apikey
end
def region_server(region)
case region
when "euw"
self.class.base_uri "https://euw.api.pvp.net"
self.region = "EUW1"
when "na"
self.class.base_uri "https://na.api.pvp.net"
self.region = "NA1"
end
end
def handle_timeouts
begin
yield
#Timeout::Error, is raised if a chunk of the response cannot be read within the read_timeout.
#Timeout::Error, is raised if a connection cannot be created within the open_timeout.
rescue Net::OpenTimeout, Net::ReadTimeout
#todo
end
end
def base_path
"/observer-mode/rest/consumer/getSpectatorGameInfo"
end
def current_game_info
handle_timeouts do
url = "#{ base_path }/#{region}/#{playerid}?api_key=#{api_key}"
puts '------------------------------'
puts url
HTTParty.get(url,:debug_output => $stdout)
end
end
end
I verified my URL which is fine so I'm lost as to where the problem is coming from.
I tested with a static base_uri and it doesn't change anything.
The odd thing is when I do:
HTTParty.get("https://euw.api.pvp.net/observer-mode/rest/consumer/getSpectatorGameInfo/EUW1/randomid?api_key=myapikey")
Everything is working fine and I'm getting a response.
HTTParty doesn't seem to like the way you set your base_uri.
Unless you need it to be like that just add another attr_reader called domain and it will work.
require 'httparty'
class LolObserver
include HTTParty
default_timeout(1) #timeout after 1 second
attr_reader :api_key, :playerid, :domain
attr_accessor :region
def initialize(region,playerid,apikey)
#region = region_server(region)
#playerid = playerid
#api_key = apikey
end
def region_server(region)
case region
when "euw"
#domain = "https://euw.api.pvp.net"
self.region = "EUW1"
when "na"
#domain = "https://na.api.pvp.net"
self.region = "NA1"
end
end
def handle_timeouts
begin
yield
#Timeout::Error, is raised if a chunk of the response cannot be read within the read_timeout.
#Timeout::Error, is raised if a connection cannot be created within the open_timeout.
rescue Net::OpenTimeout, Net::ReadTimeout
#todo
end
end
def base_path
"/observer-mode/rest/consumer/getSpectatorGameInfo"
end
def current_game_info
handle_timeouts do
url = "#{domain}/#{ base_path }/#{region}/#{playerid}?api_key=#{api_key}"
puts '------------------------------'
puts url
HTTParty.get(url,:debug_output => $stdout)
end
end
end
I am working on wrapping the ruby-mqtt gem into a class which implements a subscribe and publish method. The subscribe method connects to the server and listens in a separate thread because this call is synchronous.
module PubSub
class MQTT
attr_accessor :host, :port, :username, :password
def initialize(params = {})
params.each do |attr, value|
self.public_send("#{attr}=", value)
end if params
super()
end
def connection_options
{
remote_host: self.host,
remote_port: self.port,
username: self.username,
password: self.password,
}
end
def subscribe(name, &block)
channel = name
connect_opts = connection_options
code_block = block
::Thread.new do
::MQTT::Client.connect(connect_opts) do |c|
c.get(channel) do |topic, message|
puts "channel: #{topic} data: #{message.inspect}"
code_block.call topic, message
end
end
end
end
def publish(channel = nil, data)
::MQTT::Client.connect(connection_options) do |c|
c.publish(channel, data)
end
end
end
end
I have a test that I have written using rspec to test the class but it does not pass.
mqtt = ::PubSub::MQTT.new({host: "localhost",port: 1883})
block = lambda { |channel, data| puts "channel: #{channel} data: #{data.inspect}"}
block.should_receive(:call).with("channel", {"some" => "data"})
thr = mqtt.subscribe("channel", &block)
mqtt.publish("channel", {"some" => "data"})
When I run the following ruby-mqtt-example I have now problems at all.
uri = URI.parse ENV['CLOUDMQTT_URL'] || 'mqtt://localhost:1883'
conn_opts = {
remote_host: uri.host,
remote_port: uri.port,
username: uri.user,
password: uri.password,
}
# Subscribe example
Thread.new do
puts conn_opts
MQTT::Client.connect(conn_opts) do |c|
# The block will be called when you messages arrive to the topic
c.get('test') do |topic, message|
puts "#{topic}: #{message}"
end
end
end
# Publish example
puts conn_opts
MQTT::Client.connect(conn_opts) do |c|
# publish a message to the topic 'test'
loop do
c.publish('test', 'Hello World')
sleep 1
end
end
So my question is, what am I doing wrong when I simply create a class and separate out the publish and subscribe logic? My guess is that it has something to do with Threading in the function call but I can't seem to figure it out. Any help is much appreciated.
UPDATE
I believe I know why the test is not passing and it is because when I pass a lambda in to subscribe expecting it to receive a call it actually will not receive the call when it exits the method or until publish is called. So I would like to rephrase the question to: How do I test that a block is called within a thread? If someone answers, "you don't", then the question is: How do you test that block is being called in an infinite loop like in the example of calling get within ruby-mqtt gem.
The RSpec expectations machinery will work fine with threads, as evidenced by the following example, which passes:
def foo(&block)
block.call(42)
end
describe "" do
it "" do
l = lambda {}
expect(l).to receive(:call).with(42)
Thread.new { foo(&l) }.join
end
end
The join waits for the thread(s) to finish before going further.
[2013-01-29 09:17:50] INFO WEBrick 1.3.1
[2013-01-29 09:17:50] INFO ruby 1.8.7 (2012-10-12) [i386-linux]
[2013-01-29 09:17:50] WARN TCPServer Error: Address already in use - bind(2)
[2013-01-29 09:17:50] INFO WEBrick::HTTPServer#start: pid=4107 port=8080
When I run the file attached below in linux I get the error described. I tried all possible command and strategies online to listen to processes (including rogue) and kill them. I did this in lots of ports. No luck.
As soon as I run the script in Mac OS and it works. Nevertheless I have to mount it on a server and clients have to communicate with it. It happens on every instance of amazon ec2 and on heroku. I have seen this error one too many times and spend many hours trying to fix it. I configured the security group of ec2 instances and still did not work. I am beyond desperate. At this point I have to think that the problem must be WEBrick itself or something in my code.
require 'webrick'
require 'uri'
require 'net/http'
$own_address = 8080
class AuctionInfo
# The representation is a hash mapping item names to [highest_bidder, highest_bid, end_time]
def initialize
#data = {}
end
def new_item(item, endTime)
#data[item] = ["UNKNOWN", 0, endTime]
end
def bid(item, bid, client)
if #data.has_key?(item)
endTime = #data[item][2]
if #data[item][1].to_i < bid.to_i and Time.new.to_i < endTime.to_i
#data[item] = [client, bid, endTime]
end
end
end
def get_status(item)
if #data.has_key?(item)
return #data[item][0]
end
end
def winner(item)
if #data.has_key?(item)
if #data[item][2].to_i + 1 <= Time.new.to_i
return #data[item][0]
else return "UNKNOWN"
end
end
end
def reset
#data = {}
end
def has_item(item)
return #data.has_key?(item)
end
def get_data
return {}.replace(#data)
end
end
class StartAuctionServlet < WEBrick::HTTPServlet::AbstractServlet
def initialize(server, data)
#data = data
end
def do_POST(request, response)
if request.query['name'] and request.query['end_time']
#data.new_item(request.query['name'], request.query['end_time'].to_i)
end
response.status = 200
end
alias_method :do_GET, :do_POST
end
class BidServlet < WEBrick::HTTPServlet::AbstractServlet
def initialize(server, data)
#data = data
end
def do_POST(request, response)
if request.query['name'] and request.query['client'] and request.query['bid']
#data.bid(request.query['name'], request.query['bid'].to_i, request.query['client'])
end
response.status = 200
end
alias_method :do_GET, :do_POST
end
class StatusServlet < WEBrick::HTTPServlet::AbstractServlet
def initialize(server, data)
#data = data
end
def do_GET(request, response)
if request.query['name']
response.body = #data.get_status(request.query['name'])
end
response.status = 200
end
alias_method :do_POST, :do_GET
end
class WinnerServlet < WEBrick::HTTPServlet::AbstractServlet
def initialize(server, data)
#data = data
end
def do_GET(request, response)
if request.query['name']
response.body = #data.winner(request.query['name'])
end
response.status = 200
end
alias_method :do_POST, :do_GET
end
class ResetServlet < WEBrick::HTTPServlet::AbstractServlet
def initialize(server, data)
#data = data
end
def do_POST(request, response)
#data.reset
response.status = 200
end
alias_method :do_GET, :do_POST
end
class RandomServlet < WEBrick::HTTPServlet::AbstractServlet
def initialize(server, data)
#data = data
end
def do_GET(request, response)
response.status = 200
response.body = #data.get_data.to_s
end
alias_method :do_POST, :do_GET
end
data = AuctionInfo.new
server = WEBrick::HTTPServer.new(:Port => $own_address)
server.mount '/start_auction', StartAuctionServlet, data
server.mount '/bid', BidServlet, data
server.mount '/status', StatusServlet, data
server.mount '/winner', WinnerServlet, data
server.mount '/rst', ResetServlet, data
server.mount '/', RandomServlet, data
trap("INT") { server.shutdown }
server.start
Have you checked whether the linux server is running apache, tomcat, trinidad or any other web server? Odds are one of them is already running on port 8080 on the server.
lsof is a useful command. Try lsof | grep 8080 and see whether anything shows up
Please consider this test:
def test_ok_on_second_request
bad_response = #request.get "/bad-response"
assert_equal 404, bad_response.status
good_response = #request.get "/test-title"
assert_equal 200, good_response.status
assert_equal "text/html", good_response.content_type
end
I have assured that /test-title is a valid path. The assertion that's supposed to return 200 is in fact returning 404. How is Rack behaving in order to return two different results for the same request?
This is the code for the Server class inside the project:
module Blogrite
class Server
attr_accessor :status, :mimetype, :body, :provider
def initialize *args, &block
#status, #mimetype = 200, "text/html"
provider = args[0][:with].nil? ? :filesystem : args[0][:with]
#provider = Blogrite.const_get(provider.capitalize).new
# p "Server is running with #{#provider.class}."
end
def call env
begin
article = go env['PATH_INFO'].delete("/")
rescue Blogrite::Article::NoBodyError
#status = 404
end
#status = 404 if !article
#status = 403 if env["REQUEST_METHOD"] == 'POST'
#mimetype = "text/css" if env["PATH_INFO"].include?("css")
#body = if article then article.render
elsif env.respond_to?(:to_yaml) then "<pre>#{env.to_yaml}</pre>"
else "oops"
end
[#status,{ "Content-Type" => #mimetype},[#body]]
end
def go path
f = #provider.fetch path
Article.parse f unless f.nil?
end
end
end
The whole workflow is too big for me to paste it in but you can check the project out on Github. I appreciate your help, thank you.
The solution for the problem is as simple as initializing #status inside the call function.
class Server
attr_accessor :status, :mimetype, :body, :provider
def initialize *args, &block
- #status, #mimetype = 200, "text/html"
provider = args[0][:with].nil? ? :filesystem : args[0][:with]
#provider = Blogrite.const_get(provider.capitalize).new
# p "Server is running with #{#provider.class}."
end
def call env
begin
- article = go env['PATH_INFO'].delete("/")
+ #status, #mimetype = 200, "text/html"
+ article = go env['PATH_INFO'].delete("/")
rescue Blogrite::Article::NoBodyError
#status = 404
end
That way the rack instance – that is called only once – stays out of the request's way. Every call function should have its own defaults, not the server class.
Thanks to #rubenfonseca for helping me out.