I have a question regarding two async servers that run on the same event loop. when I close one connection from client side, I see that the second server stops as well.
Here is my code:
async def http_server(addr: str, port: int, verbose: int):
runner = aiohttp.AppRunner(await init_app())
await runner.setup()
site = web.TCPSite(runner, str(addr), port)
await site.start()
async def main(port: int, addr: str, verbose: int, restapiport: int):
# HTTP server
await http_server(str(addr), restapiport, verbose)
print(f'Serving RPC on {addr}:{restapiport} ...')
# TCP server for messaging
server = await asyncio.start_server(handle_client, str(addr), port)
addr = server.sockets[0].getsockname()
print(f'Serving MBE on {addr} CSID={os.environ["CSID"]}')
async with server:
await server.serve_forever()
When I close one connection from client side I get the following Exception (which is ok):
Task exception was never retrieved
future: <Task finished coro=<handle_client() done, defined at /opt/xenv.py:19>
exception=ConnectionResetError(104, 'Connection reset by peer')>
Traceback (most recent call last):
File "/opt/xenv.py", line 41, in handle_client
data = await reader.readexactly(msg_headers.X_MSG_TCP_DATA_BUF_SIZE)
File "/usr/local/lib/python3.7/asyncio/streams.py", line 679, in readexactly
await self._wait_for_data('readexactly')
File "/usr/local/lib/python3.7/asyncio/streams.py", line 473, in _wait_for_data
await self._waiter
File "/usr/local/lib/python3.7/asyncio/selector_events.py", line 814, in _
_read_ready__data_received
data = self._sock.recv(self.max_size)
ConnectionResetError: [Errno 104] Connection reset by peer
it seems that ConnectionResetError exception somehow impacts the other asynchronous tasks. How can I handle this exception without having an impact on the other async task?
here is netstat before the exception:
root#5901ff922714:/opt# netstat
Active Internet connections (w/o servers)
Proto Recv-Q Send-Q Local Address Foreign Address State
tcp 0 0 5901ff922714:34833 symulator_pm-b_1.:33330 ESTABLISHED
tcp 0 0 5901ff922714:34271 5901ff922714:25010 ESTABLISHED
tcp 0 0 5901ff922714:36695 5901ff922714:33330 ESTABLISHED
tcp 8192 0 5901ff922714:25010 5901ff922714:34271 ESTABLISHED
tcp 49152 0 5901ff922714:33330 5901ff922714:36695 ESTABLISHED
tcp 0 0 5901ff922714:40831 symulator_pm-b_1.:25011 ESTABLISHED
Active UNIX domain sockets (w/o servers)
Proto RefCnt Flags Type State I-Node Path
unix 3 [ ] STREAM CONNECTED 396830805
unix 3 [ ] STREAM CONNECTED 396830724
unix 3 [ ] STREAM CONNECTED 396830804
unix 3 [ ] STREAM CONNECTED 396830725
unix 2 [ ] DGRAM 396819365
here is netstat after the exception:
root#5901ff922714:/opt# netstat
Active Internet connections (w/o servers)
Proto Recv-Q Send-Q Local Address Foreign Address State
Active UNIX domain sockets (w/o servers)
Proto RefCnt Flags Type State I-Node Path
unix 3 [ ] STREAM CONNECTED 396830805
unix 3 [ ] STREAM CONNECTED 396830724
unix 3 [ ] STREAM CONNECTED 396830804
unix 3 [ ] STREAM CONNECTED 396830725
any help would be much appreciated
Related
I'm trying to write a function that can connect to a server using a specific network interface so that it's consistently routed through that interface's gateway. This is on a macOS system that has one or more VPN connections.
Here's a proof-of-concept test function I've written:
void connectionTest(const char *hostname, int portNumber, const char *interface) {
struct hostent *serverHostname = gethostbyname(hostname);
if (serverHostname == NULL) {
printf("error: no such host\n");
return;
}
int socketDesc = socket(AF_INET, SOCK_STREAM, 0);
int interfaceIndex = if_nametoindex(interface);
if (interfaceIndex == 0) {
printf("Error: no such interface\n");
close(socketDesc);
return;
}
// Set the socket to specifically use the specified interface:
setsockopt(socketDesc, IPPROTO_IP, IP_BOUND_IF, &interfaceIndex, sizeof(interfaceIndex));
struct sockaddr_in servAddr;
bzero((char *)&servAddr, sizeof(servAddr));
servAddr.sin_family = AF_INET;
bcopy((char *)serverHostname->h_addr, (char *)&servAddr.sin_addr.s_addr, serverHostname->h_length);
servAddr.sin_port = htons(portNumber);
if (connect(socketDesc, (struct sockaddr *)&servAddr, sizeof(servAddr)) < 0) {
printf("connect failed, errno: %d", errno);
close(socketDesc);
return;
}
printf("connection succeeded\n");
close(socketDesc);
}
This function will successfully connect so long as the interface is one of the utun interfaces created by the VPNs, or a physical interface that is not used by the VPNs. But if I try to use the physical interface that is used by the VPNs, the function fails with errno 51: Network is unreachable.
For a more specific example, consider a system with the following network interfaces:
en0: Ethernet connection
en1: Wi-Fi connection
utun10: VPN connection 1, connected via en0
utun11: VPN connection 2, also connected via en0
If I call my function with something like:
connectionTest("api.ipify.org", 80, "en1");
connectionTest("api.ipify.org", 80, "utun10");
connectionTest("api.ipify.org", 80, "utun11");
... it will succeed. However, this is what produces the "network unreachable" error:
connectionTest("api.ipify.org", 80, "en0");
Is there some way to have the function work in the case of en0? (Preferably without changing the system's routing table just for this one connection?)
Edit:
It looks like the system doesn't know how to route packets through en0 when the VPN is up, unless it has a non-default route for en0.
I tried using the route command to check which route in the table would be used for a specific interface, and I get the following:
$ route get -ifscope en0 1.1.1.1
route: writing to routing socket: not in table
Only -ifscope en0 produces that error. However, the route table indicates there is a default route for en0. Here is the routing table when only ethernet and the VPN are connected (so no Wi-Fi or second VPN):
$ netstat -rn
Routing tables
Internet:
Destination Gateway Flags Refs Use Netif Expire
0/1 10.16.0.1 UGSc 165 0 utun10
default 192.168.20.1 UGSc 0 0 en0
10.16/16 10.16.0.8 UGSc 3 0 utun10
10.16.0.8 10.16.0.8 UH 2 0 utun10
127 127.0.0.1 UCS 0 0 lo0
127.0.0.1 127.0.0.1 UH 7 7108160 lo0
128.0/1 10.16.0.1 UGSc 40 0 utun10
169.254 link#8 UCS 1 0 en0 !
192.168.20 link#8 UCS 9 0 en0 !
192.168.20.1/32 link#8 UCS 2 0 en0 !
224.0.0/4 link#22 UmCS 0 0 utun10
224.0.0/4 link#8 UmCSI 1 0 en0 !
224.0.0.251 1:0:5e:0:0:fb UHmLWI 0 0 en0
255.255.255.255/32 link#22 UCS 0 0 utun10
255.255.255.255/32 link#8 UCSI 0 0 en0 !
There's clearly a default route listed for en0 pointing to its gateway, 192.168.20.1. Why isn't the packet being routed? If I create a static route for 1.1.1.1/32 or even 1/8 it will work. But so long as en0 only has a default route, it won't work. It's like the default route has been disabled somehow.
Edit 2:
If I add a new route to the table using:
$ route add -ifscope en0 0/0 192.168.20.1
so that the routing table now includes the following entry:
Destination Gateway Flags Refs Use Netif Expire
default 192.168.20.1 UGScI 1 0 en0
alongside all of the above entries, so there are now two default entries, then the connection works. Why is it necessary for there to be an interface-specific default route in order for this to work?
Once you added the routing table to your question, your problem became obvious.
It is the routing table that determines to which gateway a packet is sent. The routing table tells the sending host to which gateway the packet is sent. It does that by comparing the destination address to the routes in the routing table. The most-specific (longest match) route is used. A default route is the least-specific (shortest match) route, and it is used as the route of last resort when there are no more-specific routes in the routing table.
Based on the routing table you provided, any packet with a destination address from 1.0.0.0 to 126.255.255.255 (0.0.0.0/8 and 127.0.0.0/8 are exceptions as unusable ranges) will match the 0/1 routing table entry rather than the default route (0/0), and any packet with a destination address from 128.0.0.0 to 223.255.255.255 (224.0.0.0/4 is multicast, and 240.0.0.0/4 is unusable) will match the 128/1 routing table entry rather than the default route (0/0), because the route length of 1 is more specific than the default route length of 0. That means any packets destined to an address in those ranges (combined, all addresses destined for a different network) will be sent to the gateway (10.16.0.1) referenced by the routing table entries for the 0/1 and 128/1 routes.
To solve your problem, you need to remove the 0/1 and 128/1 routing table entries and replace them with one or more entries that are restricted to the networks which the tunnel can reach. With that, the entries not matching the tunnel route(s) or other more specific routing table entries will use the default route.
Code
import trio
from trio import socket
async def listen(host, port):
while True:
fullmsg = ""
sock = socket.socket()
await sock.bind((host, port))
sock.listen()
print(f'Awaiting Receive On {host}:{port}')
conn, addr = await sock.accept()
print(f'Connection Received From {addr[0]}:{addr[1]}')
while True:
try:
msg = await conn.recv(8)
if len(msg.decode().strip()) > 0:
print(f'Received {len(msg.strip())} bytes')
fullmsg += msg.decode().strip()
else:
break
except Exception as e:
print(f'DEBUG: {e}')
sock.shutdown(0)
sock.close()
print(fullmsg)
# function that runs the listen function:
async def create():
async with trio.open_nursery() as nursery:
nursery.start_soon(listen, '127.0.0.1', 6969)
# To run the program
trio.run(create)
I want to run the function over and over again everytime it receives a message of length 0 or when the connection gets closed by the client but when the function completes the first iteration of the first while loop, it gives an OSError saying the port is already in use. I close and shutdown my sockets by the ending of the loop but I still don't know where the program is erroring out.
Output Of The Program
Awaiting Receive On 127.0.0.1:6969
Connection Received From 127.0.0.1:37122
Received 8 bytes
Received 5 bytes
Hello, World!
Traceback (most recent call last):
File "./ape.py", line 68, in <module>
trio.run(create)
File "/usr/local/lib/python3.8/dist-packages/trio/_core/_run.py", line 1804, in run
raise runner.main_task_outcome.error
File "./ape.py", line 59, in create
nursery.start_soon(listen, '127.0.0.1', 6969)
File "/usr/local/lib/python3.8/dist-packages/trio/_core/_run.py", line 730, in __aexit__
raise combined_error_from_nursery
File "./ape.py", line 15, in listen
await sock.bind((host, port))
File "/usr/local/lib/python3.8/dist-packages/trio/_socket.py", line 473, in bind
return self._sock.bind(address)
OSError: [Errno 98] Address already in use
Like others said in their comments, the problem is that on Unix-y platforms, you have to set the SO_REUSEADDR socket option if you want to be able to close a listening socket and then immediately open a new one bound to the same port.
Note that on Windows, though, you should never set the SO_REUSEADDR option, because on Windows, the behavior you want is enabled by default, and SO_REUSEADDR is redefined to be a "turn off security" option.
trio.socket is very low-level and exposes all these details, so if you want to deal with them yourself it lets you do that. But most users will be better off using the higher level helpers like trio.serve_tcp, which will handle a lot of these details automatically.
I am trying to gather information from a TCP connection made to my web server to assist in our troubleshooting efforts, similar to http://speedguide.net/analyzer.php tool.
We have a simple PHP server script test page that users connect that returns their private IP to an AJAX call waiting for the response.
I would like to either build on that or prefer using Ruby. I played with the PacketFu lib and get all the information I think I need, however, I'm having trouble with the recipe to combine it all:
listen on port x,
parse the packet
respond back to client.
Using Ruby's TCPServer I can easily handle 1 and 3. With Packetfu, 2.
I've coded with PHP in the past but only HTML-based, no sockets. And I'm not really all that familiar with Ruby sockets either.
Though the packet stream and client.accept don't seem to play nice. The packets aren't always IP or TCP meeting the Packetfu::Packet.is_ip? or is_tcp?.
Could someone point me in the right direction or give me some practical example of how I might combine the two, or adjust my thinking on how I would accomplish this task?
This is the Playground code:
require 'socket'
require 'json'
require 'packetfu'
iface = ARGV[0] || "eno1"
server = TCPServer.open(31337)
cap = PacketFu::Capture.new(:iface => iface, :start => true, :promisc => true)
loop {
cap.stream.each do |p|
pkt = PacketFu::Packet.parse(p)
if pkt.is_ip? || pkt.is_tcp?
if pkt.tcp_dport == 31337
print "Source Addr: #{pkt.ip_saddr}\n"
print "Source Port: #{pkt.tcp_src}\n"
print "Destination Addr: #{pkt.ip_daddr}\n"
print "Destination Port: #{pkt.tcp_dport}\n"
print "TCP Options: #{pkt.tcp_options.inspect}\n"
print "TCP Win: #{pkt.tcp_win}\n"
print "TCP SYN?: #{pkt.tcp_flags.syn}\n"
print "TCP ACK?: #{pkt.tcp_flags.ack}\n"
print "TCP FLAGS ALL: #{pkt.tcp_flags.inspect}\n"
print "TTL: #{pkt.ip_ttl}\n"
print "IP FRAG: #{pkt.ip_frag}\n"
end
end
client = server.accept # Wait for a client to connect
h = { ipaddress: client.peeraddr[2] }
client.puts h.to_json
client.close
end
}
This is the output:
Source Addr: 172.20.0.15
Source Port: 41165
Destination Addr: 172.20.0.10
Destination Port: 31337
TCP Options: "NOP,NOP,TS:216432150;57946250"
TCP Win: 229
TCP SYN?: 0
TCP ACK?: 1
TCP FLAGS ALL: #<struct PacketFu::TcpFlags urg=0, ack=1, psh=0, rst=0, syn=0, fin=0>
TTL: 61
IP FRAG: 16384
This is the browser response:
{"ipaddress":"172.20.0.15"}
I'm trying to test out the failure recovery behavior of ZeroMQ ( via pyzmq ) when using DEALER and ROUTER sockets. Here's my code:
import sys, zmq
import threading
import time, gc
import socket
def tprint(msg):
"""like print, but won't get newlines confused with multiple threads"""
sys.stdout.write(msg + '\n')
sys.stdout.flush()
class ClientWorker(threading.Thread):
def __init__(self, id, ports):
self.id = id
self.ports = ports
super(ClientWorker, self).__init__()
def run(self):
context = zmq.Context()
socket = context.socket(zmq.DEALER)
for port in self.ports:
socket.connect("tcp://localhost:%d" % port)
tprint("client %d started" % (self.id))
for ia in xrange(self.id*100,self.id*100+100):
socket.send_string('request %d' % (ia))
time.sleep(1)
socket.close()
context.term()
class ServerWorker(threading.Thread):
def __init__(self, port, maxReq=None):
self.port = port
self.maxReq = maxReq
super(ServerWorker, self).__init__()
def run(self):
context = zmq.Context()
socket = context.socket(zmq.ROUTER)
socket.bind("tcp://127.0.0.1:%d" % (self.port))
tprint("server started on port %d" % (self.port))
numReq = 0
while True:
ident, msg = socket.recv_multipart()
print self.port, ident, msg
numReq += 1
if self.maxReq and numReq >= self.maxReq:
tprint("server on port %d exiting" % (self.port))
break
socket.unbind("tcp://127.0.0.1:%d" % (self.port))
socket.close()
context.term()
def main():
ports = [5555,5556,5557]
servers = [ServerWorker(port,10 if port==5555 else None) for port in ports]
for s in servers: s.start()
for ia in xrange(1,6):
w = ClientWorker(ia, ports)
w.start()
servers[0].join()
servers[0] = None
gc.collect()
time.sleep(30)
tprint("restarting server")
s = ServerWorker(port)
s.start()
if __name__ == "__main__":
main()
The behavior I observe is as follows:
the server at 5555 will print out 10 items that it receives at which point it exits
the client workers will NOT detect this failure, and will continue sending items to that server
when I attempt to re-.bind() a new server thread to port 5555, I get the "Address in use" error
this despite my closing the socket, calling context.term(), attempting to gc the server object, etc.
Three questions:
Am I correct to expect that the DEALER sockets should be able to detect the failure of one of the servers and redistribute the work to the remaining servers? I suspect that perhaps the reason why it can't detect the failure is the same reason that the socket on port 5555 remains open?
Any ideas about the "Address in use" error?
Am I correct to expect that when I reconnect the server to port 5555, the clients will be able to detect the reconnection and resume sending messages to the server in a round-robin way taking into account the new server?
Am I correct to expect that the DEALER sockets should be able to detect the failure of one of the servers and redistribute the work to the remaining servers?
No, this isn't how DEALERS work. DEALERs that connect load-balance across their peers, whether or not they are there. That means that messages are still queued to worker 5555, even while it's down. Those messages will be delivered immediately when worker 5555 returns.
Any ideas about the "Address in use" error?
This is caused by the fact that port when you start the resumed worker is ports[-1], not ports[0], so it's binding to a port that's still in use by one of your workers, not the one that stopped.
Am I correct to expect that when I reconnect the server to port 5555, the clients will be able to detect the reconnection and resume sending messages to the server in a round-robin way taking into account the new server?
Yes, messages will resume being delivered to 5555 when it comes back, but I think you aren't quite right about which messages will be delivered there.
With some minor adjustments to your script, I get the output:
server started on port 5555
server started on port 5556
server started on port 5557
client 1 started
client 2 started
client 3 started
client 4 started
client 5 started
5555 00800041a7 request 100
5555 00800041a8 request 200
5555 00800041a9 request 300
5555 00800041aa request 400
5555 00800041ab request 500
5556 0060b7acd9 request 101
5556 0060b7acdb request 301
5556 0060b7acdc request 401
5556 0060b7acdd request 501
5556 0060b7acda request 201
5557 004431b782 request 102
5557 004431b784 request 302
5557 004431b783 request 202
5557 004431b785 request 402
5557 004431b786 request 502
5555 00800041a7 request 103
5555 00800041a9 request 303
5555 00800041ab request 503
5555 00800041a8 request 203
5555 00800041aa request 403
server on port 5555 exiting
5556 0060b7acd9 request 104
5556 0060b7acda request 204
5556 0060b7acdd request 504
5556 0060b7acdb request 304
5556 0060b7acdc request 404
5557 004431b782 request 105
5557 004431b786 request 505
5557 004431b783 request 205
5557 004431b784 request 305
5557 004431b785 request 405
5556 0060b7acd9 request 107 <- note jump from 405 to 107
5556 0060b7acdc request 407
5556 0060b7acdd request 507
5556 0060b7acda request 207
5556 0060b7acdb request 307
restarting server on 5555
server started on port 5555
5557 004431b786 request 508
5557 004431b782 request 108
5557 004431b785 request 408
5557 004431b783 request 208
5557 004431b784 request 308
5555 0041c8aac3 request 506 <- here are the X06 messages on the new 5555 worker
5555 0041c8aac4 request 306
5555 0041c8aac5 request 406
5555 0041c8aac6 request 106
5555 0041c8aac7 request 206
5555 0041c8aac7 request 209
5555 0041c8aac4 request 309
5555 0041c8aac3 request 509
5555 0041c8aac5 request 409
5555 0041c8aac6 request 109
5556 0060b7acdd request 510
5556 0060b7acdb request 310
5556 0060b7acda request 210
5556 0060b7acdc request 410
5556 0060b7acd9 request 110
5557 004431b784 request 311
5557 004431b786 request 511
...
Messages 106-506 were sent to 5555 and redelivered later. They were not re-routed to another worker when 5555 wasn't there to receive them.
You can use client_socket.hwm = N to limit how many messages may be pending on a worker before the client should start excluding it from round-robin, but you can't make it zero.
The version of your script that I used:
from binascii import hexlify
import threading
import socket
import sys
import time
import zmq
def tprint(msg):
"""like print, but won't get newlines confused with multiple threads"""
sys.stdout.write(msg + '\n')
sys.stdout.flush()
class ClientWorker(threading.Thread):
def __init__(self, id, ports):
self.id = id
self.ports = ports
super(ClientWorker, self).__init__()
def run(self):
context = zmq.Context.instance()
socket = context.socket(zmq.DEALER)
socket.hwm = 1 # limit messages sent to dead workers
for port in self.ports:
socket.connect("tcp://localhost:%d" % port)
tprint("client %d started" % (self.id))
for ia in xrange(self.id*100,self.id*100+100):
socket.send_string('request %d' % (ia))
time.sleep(1)
socket.close()
context.term()
class ServerWorker(threading.Thread):
def __init__(self, port, maxReq=None):
self.port = port
self.maxReq = maxReq
super(ServerWorker, self).__init__()
def run(self):
context = zmq.Context.instance()
socket = context.socket(zmq.ROUTER)
tprint("server started on port %d" % (self.port))
socket.bind("tcp://127.0.0.1:%d" % (self.port))
numReq = 0
while True:
ident, msg = socket.recv_multipart()
print self.port, hexlify(ident), msg
numReq += 1
if self.maxReq and numReq >= self.maxReq:
tprint("server on port %d exiting" % (self.port))
break
socket.close()
context.term()
def main():
ports = [5555,5556,5557]
servers = [ServerWorker(port,10 if port==5555 else None) for port in ports]
for s in servers: s.start()
for ia in xrange(1,6):
w = ClientWorker(ia, ports)
w.start()
servers[0].join()
time.sleep(10)
port = ports[0]
tprint("restarting server on %i" % port)
s = ServerWorker(port)
s.start()
if __name__ == "__main__":
ctx = zmq.Context.instance()
try:
main()
finally:
ctx.term()
1.From client:
root#amsys-LIFEBOOK-AH502:/home/amsys# radtest -t chap usr password 127.0.0.1 0 testing123
This is how,the way i sended a packet access-request packet from the client (here,loop back only).
2.From server.
the server responds to client as shown as below:
Ready to process requests.
Ignoring request to auth address * port 1812 as server default from unknown client 127.0.0.1 port 34962 proto udp
3.server to client
Sending Access-Request of id 67 from 0.0.0.0 port 47852 to 127.0.0.1 port 1812
User-Name = 'usr'
User-Password = 'password'
NAS-IP-Address = 127.0.1.1
NAS-Port = 0
Message-Authenticator = 0x00
radclient: no response from server for ID 67 socket 3
if anybody would aware about this thing,please give your prompt response and pleased me.thanking you.!