PyTorch Dataloader hangs when num_workers > 0. The code hangs with only about 500 M GPU memory usage.
System info: NVIDIA-SMI 418.56 Driver Version: 418.56 CUDA Version: 10.1.
The same issue appears with pytorch1.5 or pytorch1.6, codes are run in anaconda envs.
Note that this error appears when I run my script in terminal as python main.py, but when I
debug the same code on Pycharm or VScode, or when I run the same code (in terminal) on
other machines, everything goes fine.
Any idea about the reason for this?
Here is the trace when I ctrl c the code in terminal:
File "train.py", line 226, in main
train_domain_adaptation(model, source_loader, target_loader, val_loader,
File "/home/zhangyu/codes/person_seg/IntraDA/ADVENT/advent/domain_adaptation/train_UDA.py", line 326, in train_domain_adaptation
train_advent(model, trainloader, targetloader, val_loader, cfg, group=group, fk_loader=fk_loader)
File "/home/zhangyu/codes/person_seg/IntraDA/ADVENT/advent/domain_adaptation/train_UDA.py", line 114, in train_advent
_, (images_source, labels, src_names, voc_ids, _) = trainloader_iter.__next__()
File "/home/zhangyu/anaconda3/envs/pt16/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 363, in __next__
data = self._next_data()
File "/home/zhangyu/anaconda3/envs/pt16/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 974, in _next_data
idx, data = self._get_data()
File "/home/zhangyu/anaconda3/envs/pt16/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 931, in _get_data
success, data = self._try_get_data()
File "/home/zhangyu/anaconda3/envs/pt16/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 779, in _try_get_data
data = self._data_queue.get(timeout=timeout)
File "/home/zhangyu/anaconda3/envs/pt16/lib/python3.8/queue.py", line 179, in get
self.not_empty.wait(remaining)
File "/home/zhangyu/anaconda3/envs/pt16/lib/python3.8/threading.py", line 306, in wait
gotit = waiter.acquire(True, timeout)
KeyboardInterrupt
Exception in thread Thread-2:
Traceback (most recent call last):
File "/home/zhangyu/anaconda3/envs/pt16/lib/python3.8/threading.py", line 932, in _bootstrap_inner
self.run()
File "/home/zhangyu/anaconda3/envs/pt16/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "/home/zhangyu/anaconda3/envs/pt16/lib/python3.8/site-packages/torch/utils/data/_utils/pin_memory.py", line 25, in _pin_memory_loop
r = in_queue.get(timeout=MP_STATUS_CHECK_INTERVAL)
File "/home/zhangyu/anaconda3/envs/pt16/lib/python3.8/multiprocessing/queues.py", line 116, in get
return _ForkingPickler.loads(res)
File "/home/zhangyu/anaconda3/envs/pt16/lib/python3.8/site-packages/torch/multiprocessing/reductions.py", line 282, in rebuild_storage_fd
fd = df.detach()
File "/home/zhangyu/anaconda3/envs/pt16/lib/python3.8/multiprocessing/resource_sharer.py", line 57, in detach
with _resource_sharer.get_connection(self._id) as conn:
File "/home/zhangyu/anaconda3/envs/pt16/lib/python3.8/multiprocessing/resource_sharer.py", line 87, in get_connection
c = Client(address, authkey=process.current_process().authkey)
File "/home/zhangyu/anaconda3/envs/pt16/lib/python3.8/multiprocessing/connection.py", line 508, in Client
answer_challenge(c, authkey)
File "/home/zhangyu/anaconda3/envs/pt16/lib/python3.8/multiprocessing/connection.py", line 752, in answer_challenge
message = connection.recv_bytes(256) # reject large message
File "/home/zhangyu/anaconda3/envs/pt16/lib/python3.8/multiprocessing/connection.py", line 216, in recv_bytes
buf = self._recv_bytes(maxlength)
File "/home/zhangyu/anaconda3/envs/pt16/lib/python3.8/multiprocessing/connection.py", line 414, in _recv_bytes
buf = self._recv(4)
File "/home/zhangyu/anaconda3/envs/pt16/lib/python3.8/multiprocessing/connection.py", line 379, in _recv
chunk = read(handle, remaining)
ConnectionResetError: [Errno 104] Connection reset by peer
Related
I can't figure out what's wrong in my code:
import threading
from telegram.ext import ApplicationBuilder
import asyncio
import time
botApp = ApplicationBuilder().token(token="******:******").build()
def main():
* computation stuff *
while True
asyncio.run(send_msg()) * send result of computention 2 times*
time.sleep(5)
async def send_msg():
print("sending")
await botApp.bot.sendMessage(
chat_id="********",
text='example',
parse_mode= 'markdown'
)
if __name__ == '__main__':
x = threading.Thread(target=main()).start
botApp.run_polling()
that raise this error:
Traceback (most recent call last):
File "/Library/Python/3.8/site-packages/anyio/streams/tls.py", line 130, in _call_sslobject_method
result = func(*args)
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/ssl.py", line 888, in read
** v = self._sslobj.read(len)
ssl.SSLWantReadError: The operation did not complete (read) (_ssl.c:2607)
During handling of the above exception, another exception occurred:
**
Traceback (most recent call last):
File "/Library/Python/3.8/site-packages/telegram/request/_baserequest.py", line 274, in _request_wrapper
code, payload = await self.do_request(
File "/Library/Python/3.8/site-packages/telegram/request/_httpxrequest.py", line 199, in do_request
res = await self._client.request(
File "/Library/Python/3.8/site-packages/httpx/_client.py", line 1533, in request
return await self.send(request, auth=auth, follow_redirects=follow_redirects)
File "/Library/Python/3.8/site-packages/httpx/_client.py", line 1620, in send
response = await self._send_handling_auth(
File "/Library/Python/3.8/site-packages/httpx/_client.py", line 1648, in _send_handling_auth
response = await self._send_handling_redirects(
File "/Library/Python/3.8/site-packages/httpx/_client.py", line 1685, in _send_handling_redirects
response = await self._send_single_request(request)
File "/Library/Python/3.8/site-packages/httpx/_client.py", line 1722, in _send_single_request
response = await transport.handle_async_request(request)
File "/Library/Python/3.8/site-packages/httpx/_transports/default.py", line 353, in handle_async_request
resp = await self._pool.handle_async_request(req)
File "/Library/Python/3.8/site-packages/httpcore/_async/connection_pool.py", line 253, in handle_async_request
raise exc
File "/Library/Python/3.8/site-packages/httpcore/_async/connection_pool.py", line 237, in handle_async_request
response = await connection.handle_async_request(request)
File "/Library/Python/3.8/site-packages/httpcore/_async/connection.py", line 90, in handle_async_request
return await self._connection.handle_async_request(request)
File "/Library/Python/3.8/site-packages/httpcore/_async/http2.py", line 144, in handle_async_request
raise exc
File "/Library/Python/3.8/site-packages/httpcore/_async/http2.py", line 112, in handle_async_request
status, headers = await self._receive_response(
File "/Library/Python/3.8/site-packages/httpcore/_async/http2.py", line 229, in _receive_response
event = await self._receive_stream_event(request, stream_id)
File "/Library/Python/3.8/site-packages/httpcore/_async/http2.py", line 260, in _receive_stream_event
await self._receive_events(request, stream_id)
File "/Library/Python/3.8/site-packages/httpcore/_async/http2.py", line 281, in _receive_events
events = await self._read_incoming_data(request)
File "/Library/Python/3.8/site-packages/httpcore/_async/http2.py", line 341, in _read_incoming_data
raise exc
File "/Library/Python/3.8/site-packages/httpcore/_async/http2.py", line 327, in _read_incoming_data
data = await self._network_stream.read(self.READ_NUM_BYTES, timeout)
File "/Library/Python/3.8/site-packages/httpcore/backends/asyncio.py", line 34, in read
return await self._stream.receive(max_bytes=max_bytes)
File "/Library/Python/3.8/site-packages/anyio/streams/tls.py", line 195, in receive
data = await self._call_sslobject_method(self._ssl_object.read, max_bytes)
File "/Library/Python/3.8/site-packages/anyio/streams/tls.py", line 137, in _call_sslobject_method
data = await self.transport_stream.receive()
File "/Library/Python/3.8/site-packages/anyio/_backends/_asyncio.py", line 1264, in receive
self._transport.resume_reading()
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/asyncio/selector_events.py", line 798, in resume_reading
self._add_reader(self._sock_fd, self._read_ready)
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/asyncio/selector_events.py", line 744, in _add_reader
self._loop._add_reader(fd, callback, *args)
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/asyncio/selector_events.py", line 256, in _add_reader
self._check_closed()
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/asyncio/base_events.py", line 508, in _check_closed
** raise RuntimeError('Event loop is closed')
RuntimeError: Event loop is closed
The above exception was the direct cause of the following exception:**
Traceback (most recent call last):
File "./test.py", line 26, in <module>
threading.Thread(target=main())
File "./test.py", line 12, in main
asyncio.run(send_msg())
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/asyncio/runners.py", line 43, in run
return loop.run_until_complete(main)
File "/Library/Developer/CommandLineTools/Library/Frameworks/Python3.framework/Versions/3.8/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "./test.py", line 18, in send_msg
await botApp.bot.sendMessage(
File "/Library/Python/3.8/site-packages/telegram/ext/_extbot.py", line 2598, in send_message
return await super().send_message(
File "/Library/Python/3.8/site-packages/telegram/_bot.py", line 331, in decorator
result = await func(*args, **kwargs) # skipcq: PYL-E1102
File "/Library/Python/3.8/site-packages/telegram/_bot.py", line 760, in send_message
return await self._send_message(
File "/Library/Python/3.8/site-packages/telegram/ext/_extbot.py", line 488, in _send_message
result = await super()._send_message(
File "/Library/Python/3.8/site-packages/telegram/_bot.py", line 512, in _send_message
result = await self._post(
File "/Library/Python/3.8/site-packages/telegram/_bot.py", line 419, in _post
return await self._do_post(
File "/Library/Python/3.8/site-packages/telegram/ext/_extbot.py", line 306, in _do_post
return await super()._do_post(
File "/Library/Python/3.8/site-packages/telegram/_bot.py", line 450, in _do_post
return await request.post(
File "/Library/Python/3.8/site-packages/telegram/request/_baserequest.py", line 165, in post
result = await self._request_wrapper(
File "/Library/Python/3.8/site-packages/telegram/request/_baserequest.py", line 290, in _request_wrapper
**raise NetworkError(f"Unknown error in HTTP implementation: {repr(exc)}") from exc
telegram.error.NetworkError: Unknown error in HTTP implementation: RuntimeError('Event loop is closed')**
Is there anyone who can help me?
I wanted to create a main function to do some computational stuff every 5 seconds and advise me sending a message using a telegram bot... For this reason I created a thread to run the main function while the main thread run the telegram API
I am trying this simple code and below error comes while getting uid
url = 'http://localhost:8071'
db = "db8aaa_19sep22"
username = "admin"
password = "Admin#347"
common = xmlrpc.client.ServerProxy('{}/xmlrpc/2/common'.format(url))
common.version()
uid = common.authenticate(db, username, password, {})
Getting below error
File "/home/oem/odoo/odoo-14/odoo/addons/base/models/ir_http.py", line 237, in _dispatch
result = request.dispatch()
File "/home/oem/odoo/odoo-14/odoo/http.py", line 684, in dispatch
result = self._call_function(**self.params)
File "/home/oem/odoo/odoo-14/odoo/http.py", line 360, in _call_function
return checked_call(self.db, *args, **kwargs)
File "/home/oem/odoo/odoo-14/odoo/service/model.py", line 94, in wrapper
return f(dbname, *args, **kwargs)
File "/home/oem/odoo/odoo-14/odoo/http.py", line 348, in checked_call
result = self.endpoint(*a, **kw)
File "/home/oem/odoo/odoo-14/odoo/http.py", line 913, in __call__
return self.method(*args, **kw)
File "/home/oem/odoo/odoo-14/odoo/http.py", line 532, in response_wrap
response = f(*args, **kw)
File "/home/oem/odoo/odoo-14/odoo/addons/web/controllers/main.py", line 1393, in call_button
action = self._call_kw(model, method, args, kwargs)
File "/home/oem/odoo/odoo-14/odoo/addons/web/controllers/main.py", line 1381, in _call_kw
return call_kw(request.env[model], method, args, kwargs)
File "/home/oem/odoo/odoo-14/odoo/api.py", line 396, in call_kw
result = _call_kw_multi(method, model, args, kwargs)
File "/home/oem/odoo/odoo-14/odoo/api.py", line 383, in _call_kw_multi
result = method(recs, *args, **kwargs)
File "/home/oem/odoo/14foraaa/migration_scripts/model/leads.py", line 55, in get_leads_rpc
uid = common.authenticate(db, username, password, {})
File "/usr/lib/python3.7/xmlrpc/client.py", line 1112, in __call__
return self.__send(self.__name, args)
File "/usr/lib/python3.7/xmlrpc/client.py", line 1452, in __request
verbose=self.__verbose
File "/usr/lib/python3.7/xmlrpc/client.py", line 1154, in request
return self.single_request(host, handler, request_body, verbose)
File "/usr/lib/python3.7/xmlrpc/client.py", line 1170, in single_request
return self.parse_response(resp)
File "/usr/lib/python3.7/xmlrpc/client.py", line 1342, in parse_response
return u.close()
File "/usr/lib/python3.7/xmlrpc/client.py", line 656, in close
raise Fault(**self._stack[0])
Exception
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/home/oem/odoo/odoo-14/odoo/http.py", line 640, in _handle_exception
return super(JsonRequest, self)._handle_exception(exception)
File "/home/oem/odoo/odoo-14/odoo/http.py", line 316, in _handle_exception
raise exception.with_traceback(None) from new_cause
xmlrpc.client.Fault: <Fault 1: 'Traceback (most recent call last):\n File "/opt/odoo/openerp/service/wsgi_server.py", line 75, in xmlrpc_return\n result = openerp.http.dispatch_rpc(service, method, params)\n File "/opt/odoo/openerp/http.py", line 115, in dispatch_rpc\n result = dispatch(method, params)\n File "/opt/odoo/openerp/service/common.py", line 26, in dispatch\n return fn(*params)\n File "/opt/odoo/openerp/service/common.py", line 38, in exp_authenticate\n return res_users.authenticate(db, login, password, user_agent_env)\n File "/opt/odoo/openerp/addons/user_logs/users.py", line 26, in authenticate\n uid = super(res_users, self).authenticate(db, login, password, user_agent_env)\n File "/opt/odoo/openerp/addons/base/res/res_users.py", line 464, in authenticate\n uid = self._login(db, login, password)\n File "/opt/odoo/openerp/addons/base/res/res_users.py", line 427, in _login\n self.check_credentials(cr, user_id, password)\n File "/opt/odoo/openerp/api.py", line 268, in wrapper\n return old_api(self, *args, **kwargs)\n File "/opt/odoo/openerp/api.py", line 372, in old_api\n result = method(recs, *args, **kwargs)\n File "/opt/odoo/openerp/addons/otp_auth/models/res_users.py", line 24, in check_credentials\n totp = request.session.get(\'otploginobj\')\n File "/usr/local/lib/python2.7/dist-packages/werkzeug/local.py", line 338, in __getattr__\n return getattr(self._get_current_object(), name)\n File "/usr/local/lib/python2.7/dist-packages/werkzeug/local.py", line 297, in _get_current_object\n return self.__local()\n File "/usr/local/lib/python2.7/dist-packages/werkzeug/local.py", line 132, in _lookup\n raise RuntimeError(\'object unbound\')\nRuntimeError: object unbound\n'>```
I run this script:
The script runs in a loop on an host_ips, connects to each of them and runs a bat file in the command
from socket import *
from pypsexec.client import Client
class Windows(object):
def __init__(self, ip, username, password):
self.ip = ip
self.username = username
self.password = password
self.cmd_exe = "C:\\Windows\\System32\\cmd.exe"
self.windows_client = self.connect_to_windows_client()
def connect_to_windows_client(self):
windows_client = self.windows_connect(self.ip, self.username, self.password)
if windows_client is not None:
return windows_client
self.log.error("Connection error to {}".format(self.ip))
#staticmethod
def windows_connect(address, username, pwd):
try:
windows_client = Client(address, username=username, password=pwd, encrypt=False)
windows_client.connect()
windows_client.create_service()
return windows_client
except Exception:
print("Your Username and Password of " + getfqdn(address) + " are wrong.")
return False
xtenders = ['host_ip1', 'host_ip2']
def main():
for x in xtenders:
print x
x = Windows(x, "user_name", "password")
result = x.windows_client.run_executable('C:\\file_bat_directory\\file.bat')
x.windows_client.remove_service()
x.windows_client.disconnect()
print result
if __name__ == '__main__':
main()
Sometimes the script run ok, but sometimes it is stuck when he connect to machine, it is print the host_ip, waits a few minutes and exit with the following error:
'host_ip1'
Exception in thread msg_worker-'host_ip1':445:
Traceback (most recent call last):
File "/usr/lib64/python2.7/threading.py", line 812, in __bootstrap_inner
self.run()
File "/usr/lib64/python2.7/threading.py", line 765, in run
self.__target(*self.__args, **self.__kwargs)
File "/home/amirh/.local/lib/python2.7/site-packages/smbprotocol/connection.py", line 1199, in _process_message_thread
self.disconnect(False)
File "/home/amirh/.local/lib/python2.7/site-packages/smbprotocol/connection.py", line 799, in disconnect
self.transport.close()
File "/home/amirh/.local/lib/python2.7/site-packages/smbprotocol/transport.py", line 89, in close
self._sock.shutdown(socket.SHUT_RDWR)
File "/usr/lib64/python2.7/socket.py", line 224, in meth
return getattr(self._sock,name)(*args)
error: [Errno 107] Transport endpoint is not connected
Traceback (most recent call last):
File "file_name.py", line 44, in <module>
main()
File "file_name.py", line 39, in main
result = x.windows_client.run_executable('C:\\file_bat_directory\\file.bat')
File "/home/amirh/.local/lib/python2.7/site-packages/pypsexec/client.py", line 395, in run_executable
main_pipe.write(start_msg.pack(), 0)
File "/home/amirh/.local/lib/python2.7/site-packages/smbprotocol/open.py", line 1382, in write
return self._write_response(request, wait)
File "/home/amirh/.local/lib/python2.7/site-packages/smbprotocol/open.py", line 1388, in _write_response
response = self.connection.receive(request, wait=wait)
File "/home/amirh/.local/lib/python2.7/site-packages/smbprotocol/connection.py", line 859, in receive
self._check_worker_running() # The worker may have failed while waiting for the response, check again
File "/home/amirh/.local/lib/python2.7/site-packages/smbprotocol/connection.py", line 1004, in _check_worker_running
self.disconnect(False)
File "/home/amirh/.local/lib/python2.7/site-packages/smbprotocol/connection.py", line 799, in disconnect
self.transport.close()
File "/home/amirh/.local/lib/python2.7/site-packages/smbprotocol/transport.py", line 89, in close
self._sock.shutdown(socket.SHUT_RDWR)
File "/usr/lib64/python2.7/socket.py", line 224, in meth
return getattr(self._sock,name)(*args)
socket.error: [Errno 107] Transport endpoint is not connected
How can I fix this?
I am trying to convert my training data for spaCy train using spaCy convert. My data looks (after some preprocessing with pandas) like this:
1 Hii hii PRON _ NounClass=9|Num=Sing _ _ _ _
2 si si VERB _ _ _ _ _ _
3 mara mara NOUN _ NounClass=10|Num=Plur _ _ _ _
4 ya_kwanza ya_kwanza NUM _ _ _ _ _ _
5 kwa kwa ADP _ _ _ _ _ _
6 uongozi uongozi NOUN _ NounClass=11|Num=Sing _ _ _ _
I used the following command in the Terminal:
PS C:\Users\...\pythonProject1> python -m spacy convert C:\Users\...\pythonProject1\my_dataframe_ready.conllu C:\Users\...\pythonProject1\train
and get the following output:
ℹ Grouping every 1 sentences into a document.
⚠ To generate better training data, you may want to group sentences into
documents with `-n 10`.
Traceback (most recent call last):
File "C:\Users\...\miniconda3\envs\pythonProject1\lib\runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "C:\Users\...\miniconda3\envs\pythonProject1\lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "C:\Users\...\miniconda3\envs\pythonProject1\lib\site-packages\spacy\__main__.py", line 4, in <module>
setup_cli()
File "C:\Users\...\miniconda3\envs\pythonProject1\lib\site-packages\spacy\cli\_util.py", line 71, in setup_cli
command(prog_name=COMMAND)
File "C:\Users\...\miniconda3\envs\pythonProject1\lib\site-packages\click\core.py", line 829, in __call__
return self.main(*args, **kwargs)
File "C:\Users\...\miniconda3\envs\pythonProject1\lib\site-packages\click\core.py", line 782, in main
rv = self.invoke(ctx)
File "C:\Users\...\miniconda3\envs\pythonProject1\lib\site-packages\click\core.py", line 1259, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "C:\Users\...\miniconda3\envs\pythonProject1\lib\site-packages\click\core.py", line 1066, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "C:\Users\...\miniconda3\envs\pythonProject1\lib\site-packages\click\core.py", line 610, in invoke
return callback(*args, **kwargs)
File "C:\Users\...\miniconda3\envs\pythonProject1\lib\site-packages\typer\main.py", line 500, in wrapper
return callback(**use_params) # type: ignore
File "C:\Users\...\miniconda3\envs\pythonProject1\lib\site-packages\spacy\cli\convert.py", line 89, in convert_cli
msg=msg,
File "C:\Users\...\miniconda3\envs\pythonProject1\lib\site-packages\spacy\cli\convert.py", line 140, in convert
db = DocBin(docs=docs, store_user_data=True)
File "C:\Users\...\miniconda3\envs\pythonProject1\lib\site-packages\spacy\tokens\_serialize.py", line 86, in __init__
for doc in docs:
File "C:\Users\...\miniconda3\envs\pythonProject1\lib\site-packages\spacy\training\converters\conllu_to_docs.py", line 38, i
n conllu_to_docs
for sent_doc in sent_docs:
File "C:\Users\...\miniconda3\envs\pythonProject1\lib\site-packages\spacy\training\converters\conllu_to_docs.py", line 85, i
n read_conllx
ner_map=ner_map,
File "C:\Users\...\miniconda3\envs\pythonProject1\lib\site-packages\spacy\training\converters\conllu_to_docs.py", line 209,
in conllu_sentence_to_doc
heads=heads,
File "spacy\tokens\doc.pyx", line 366, in spacy.tokens.doc.Doc.__init__
File "spacy\morphology.pyx", line 49, in spacy.morphology.Morphology.add
File "spacy\morphology.pyx", line 153, in spacy.morphology.Morphology.feats_to_dict
ValueError: need more than 1 value to unpack
Is there something still wrong with my Data? I actually have no idea what this Error should be telling me.
Based on the line your error is occurring on, it looks like you have a malformed feature list somewhere. A feature list looks like alpha=yes|beta=no. It seems like you might have something that looks like alpha=yes|beta, which is invalid.
I think the underscore by itself is a special case and should be valid, but maybe you have some other kind of filler?
You can debug this by modifying the conllu_sentence_to_doc function in conllu_to_docs.py to print the morphs value before calling doc = Doc(...).
I am trying to make requests to https websites with auth proxy, but i am receiving the error below. As they mention in aiohttp documentation, there are no https proxy support, so i wonder if there is any solution to this problem. Here is also my code.
async def get_site(https_url, proxy):
async with aiohttp.ClientSession() as session:
async with session.get(https_url,
proxy=proxy) as resp:
text = await resp.text()
print(text)
loop = asyncio.get_event_loop()
loop.run_until_complete(get_site("https://www.python.org/", "http://user:passw#proxy_url:port"))
loop.run_until_complete(asyncio.sleep(0))
loop.close()
That's the error i am getting:
Traceback (most recent call last):
File "C:\Users\Adam\Anaconda3\envs\test\lib\site-packages\aiohttp\connector.py", line 936, in _wrap_create_connection
return await self._loop.create_connection(*args, **kwargs) # type: ignore # noqa
File "C:\Users\Adam\Anaconda3\envs\test\lib\asyncio\base_events.py", line 1050, in create_connection
transport, protocol = await self._create_connection_transport(
File "C:\Users\Adam\Anaconda3\envs\test\lib\asyncio\base_events.py", line 1080, in _create_connection_transport
await waiter
File "C:\Users\Adam\Anaconda3\envs\test\lib\asyncio\proactor_events.py", line 395, in _loop_writing
self._write_fut = self._loop._proactor.send(self._sock, data)
File "C:\Users\Adam\Anaconda3\envs\test\lib\asyncio\windows_events.py", line 525, in send
self._register_with_iocp(conn)
File "C:\Users\Adam\Anaconda3\envs\test\lib\asyncio\windows_events.py", line 714, in _register_with_iocp
_overlapped.CreateIoCompletionPort(obj.fileno(), self._iocp, 0, 0)
OSError: [WinError 87] The parameter is incorrect
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:/Users/Adam/Desktop/test/test.py", line 22, in <module>
loop.run_until_complete(get_site("https://www.python.org/", "proxy_here"))
File "C:\Users\Adam\Anaconda3\envs\test\lib\asyncio\base_events.py", line 616, in run_until_complete
return future.result()
File "C:/Users/Adam/Desktop/test/test.py", line 15, in waht
async with session.get("https://www.python.org/",
File "C:\Users\Adam\Anaconda3\envs\test\lib\site-packages\aiohttp\client.py", line 1012, in __aenter__
self._resp = await self._coro
File "C:\Users\Adam\Anaconda3\envs\test\lib\site-packages\aiohttp\client.py", line 480, in _request
conn = await self._connector.connect(
File "C:\Users\Adam\Anaconda3\envs\test\lib\site-packages\aiohttp\connector.py", line 523, in connect
proto = await self._create_connection(req, traces, timeout)
File "C:\Users\Adam\Anaconda3\envs\test\lib\site-packages\aiohttp\connector.py", line 855, in _create_connection
_, proto = await self._create_proxy_connection(
File "C:\Users\Adam\Anaconda3\envs\test\lib\site-packages\aiohttp\connector.py", line 1093, in _create_proxy_connection
transport, proto = await self._wrap_create_connection(
File "C:\Users\Adam\Anaconda3\envs\test\lib\site-packages\aiohttp\connector.py", line 943, in _wrap_create_connection
raise client_error(req.connection_key, exc) from exc
aiohttp.client_exceptions.ClientConnectorError: Cannot connect to host www.python.org:443 ssl:None [The parameter is incorrect]
Process finished with exit code 1