I have a fastapi application where I use sqlalchemy and stored procedures.
Now I want to test my endpoints like in the documentation
import pytest
from fastapi.testclient import TestClient
from fastapi import FastAPI
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from ..dependencies import get_db
import cx_Oracle
host = 'xxxx'
port = 1111
sid = 'FUU'
user = 'bar'
password = 'fuubar'
sid = cx_Oracle.makedsn(host, port, sid=sid)
database_url = 'oracle://{user}:{password}#{sid}'.format(
user=user,
password=password,
sid=sid,
)
engine = create_engine(database_url, connect_args={"check_same_thread": False})
TestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
app = FastAPI()
init_router(app)
#pytest.fixture()
def session():
db = TestingSessionLocal()
try:
yield db
finally:
db.close()
#pytest.fixture()
def client(session):
# Dependency override
def override_get_db():
try:
yield session
finally:
session.close()
app.dependency_overrides[get_db] = override_get_db
yield TestClient(app)
def test_index(client):
res = client.get("/")
assert res.text
assert res.status_code == 200
def test_search_course_by_verid_exist():
response = client.get(
'search', params={"search_query": "1111", "semester": "S2022"})
# course exist
assert response.status_code == 200
I've tried it with creating a new app and/or importing it via getting the app from the main.py
from ..main import app
The method is in my courses router.
#router.get("/search", status_code=status.HTTP_200_OK)
async def search_course(
response: Response,
search_query: Union[str, None] = None,
semester: Union[int, None] = None,
db: Session = Depends(get_db),
):
.....
return response
The index test already failes by returning assert 400 == 200. For the 2nd (test_search_course_by_verid_exist) I'll get
AttributeError: 'function' object has no attribute 'get'
My main has some middleware settings like
app.add_middleware(
SessionMiddleware, secret_key="fastAPI"
) # , max_age=300 this should match Login action timeout in token-settings of a realm
app.add_middleware(
TrustedHostMiddleware,
allowed_hosts=settings.ALLOWED_HOSTS,
)
# MIDDLEWARE
#app.middleware("http")
async def check_route(request: Request, call_next):
....
I'm clueless what I'm missing or if things are just different with cx_Oracle
I've tried changing the testclient from fastapi to the starlette one. I've tried not overriding the db and just import the original db settings (which are basically the same). But nothing works.
I'm not sure if this is the proper way to test FastAPI application, https://fastapi.tiangolo.com/tutorial/testing/
Why you didn't declare client as :
client = TestClient(app)
?
Idk if this was the root problem. But naming my fixtures solved the problem and the db connection is working.
conftest.py
#pytest.fixture(name="db_session", scope="session")
def db_session(app: FastAPI) -> Generator[TestingSessionLocal, Any, None]:
Also created the app fixture
#pytest.fixture(name="app", scope="session")
def app() -> Generator[FastAPI, Any, None]:
"""
Create a fresh database on each test case.
"""
_app = start_application()
yield _app
I downloaded bot which copy messages from telegram group and sending to my discord channel.
But when I am trying to send a message in a group, I am getting this error.
2022-12-31 08:25:19,877 - telegram.ext.dispatcher - ERROR - No error handlers are registered, logging exception.
Traceback (most recent call last):
File "C:\Python310\lib\site-packages\telegram\ext\dispatcher.py", line 557, in process_update
handler.handle_update(update, self, check, context)
File "C:\Python310\lib\site-packages\telegram\ext\handler.py", line 199, in handle_update
return self.callback(update, context)
File "C:\Discord-Telegram-Bot-main 1\main.py", line 46, in getTgAnnouncement
textUpdate = update.channel_post.text
AttributeError: 'NoneType' object has no attribute 'text'
I am using python 3.10
Python-telegram-bot 13.15
Below i will add the code
import requests
import json
import asyncio
import os
import json
import logging
from dotenv import load_dotenv
import discord
from discord.ext import commands
from telegram import Update
from telegram.ext import Updater,CallbackContext,MessageHandler,Filters
from telegram.utils import helpers
from telegram.ext.dispatcher import run_async
load_dotenv('token.env')
discordToken = os.getenv('DCTOKEN')# discord bot token
telegramToken = os.getenv('TGTOKEN')# telegram bot token
discordChannelId = os.getenv('DCCID')# discord announcement channel id
loop = asyncio.get_event_loop()
#--------------------------------------------------------------------------------------------------------------
bot = commands.Bot(
command_prefix="k!",
case_insensitive=True,
intents=discord.Intents.all(),
help_command=None
)
async def sendDcAnnouncement(textUpdate,nonTextUpdate):
announcementChannel = bot.get_channel(int(discordChannelId))
if textUpdate != None and nonTextUpdate != None:
await announcementChannel.send(textUpdate,file=discord.File(nonTextUpdate))
os.remove(nonTextUpdate)
elif textUpdate == None:
await announcementChannel.send(file=discord.File(nonTextUpdate))
os.remove(nonTextUpdate)
elif nonTextUpdate == None:
await announcementChannel.send(textUpdate)
def getTgAnnouncement(update: Update, context: CallbackContext):
textUpdate = None
nonTextUpdate = None
updateType = helpers.effective_message_type(update)
if updateType == 'text':
textUpdate = update.channel_post.text
else:
textUpdate = update.channel_post.caption
if updateType == 'photo':
nonTextUpdate = update.channel_post.photo[-1].get_file()['file_path']
elif updateType == 'video':
nonTextUpdate = update.channel_post.video.get_file()['file_path']
elif updateType == 'document':
nonTextUpdate = update.channel_post.document.get_file()['file_path']
elif updateType == 'voice':
nonTextUpdate = update.channel_post.voice.get_file()['file_path']
loop.create_task(sendDcAnnouncement(textUpdate,nonTextUpdate))
#bot.event
async def on_ready():
print(f'logged in as {bot.user}')
updater = Updater(token=telegramToken,use_context=True)
dispatcher = updater.dispatcher
logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
level=logging.INFO)
getTgAnnouncement_handler = MessageHandler((~Filters.command),getTgAnnouncement)
dispatcher.add_handler(getTgAnnouncement_handler)
updater.start_polling()
bot.run(discordToken)
#--------------------------------------------------------------------------------------------------------------
I expect that this bot will send messages and that I will not get these errors.
MessageHandler handles all updates that have one of the fields message, edited_message, channel_post or edited_channel_post. So in getTGAnnouncement, it may very well be the case that update.channel_post is None.
If you want that handler to only handle channel_post update, please use Filters.update.channel_post.
As a side note: I see that you're using python-telegram-bot v13.x. Support for that version is ending soon in favor of the new v20.x.
Disclaimer: I'm currently the maintainer of python-telegram-bot.
The relevant parts of my code are as follows:
#app.on_event("startup")
async def startup():
es = Translator()
app.state.es_conn = es.connect()
#app.on_event('shutdown')
async def shutdown_event():
es = Translator()
await es.close(app.state.es_conn)
#app.post('/query')
async def query(req: Request):
es = Translator()
es.set_conn(req.app.state.es_conn)
return await es.search()
from elasticsearch import AsyncElasticsearch, AsyncTransport
class Translator(object):
def __init__(self):
self.__es = None
def connect(self):
return AsyncElasticsearch(
hosts='http://192.168.0.2:9200/',
transport_class=AsyncTransport)
def set_conn(self, conn):
self.__es = conn
async def close(self, conn):
await conn.close()
self.__es = None
async def search(self):
return await self.__es.search(index=index, body=body)
It runs in the following environment:
PyPy 7.3.7
fastapi 0.73.0
Elasticsearch 7.10.1
My question are:
Although the code can work normally, how can I detect whether it uses connection pool?
How to use connection pool correctly when asynchronous?
Can anyone help me? Thank you!
We want to use django-channels for our websockets but we need to authenticate as well. We have a rest api running with django-rest-framework and there we use tokens to authenticate a user, but the same functionality does not seem to be built into django-channels.
For Django-Channels 2 you can write custom authentication middleware
https://gist.github.com/rluts/22e05ed8f53f97bdd02eafdf38f3d60a
token_auth.py:
from channels.auth import AuthMiddlewareStack
from rest_framework.authtoken.models import Token
from django.contrib.auth.models import AnonymousUser
class TokenAuthMiddleware:
"""
Token authorization middleware for Django Channels 2
"""
def __init__(self, inner):
self.inner = inner
def __call__(self, scope):
headers = dict(scope['headers'])
if b'authorization' in headers:
try:
token_name, token_key = headers[b'authorization'].decode().split()
if token_name == 'Token':
token = Token.objects.get(key=token_key)
scope['user'] = token.user
except Token.DoesNotExist:
scope['user'] = AnonymousUser()
return self.inner(scope)
TokenAuthMiddlewareStack = lambda inner: TokenAuthMiddleware(AuthMiddlewareStack(inner))
routing.py:
from django.urls import path
from channels.http import AsgiHandler
from channels.routing import ProtocolTypeRouter, URLRouter
from channels.auth import AuthMiddlewareStack
from yourapp.consumers import SocketCostumer
from yourapp.token_auth import TokenAuthMiddlewareStack
application = ProtocolTypeRouter({
"websocket": TokenAuthMiddlewareStack(
URLRouter([
path("socket/", SocketCostumer),
]),
),
})
If you are using Django Channels 3 you can use this code:
https://gist.github.com/AliRn76/1fb99688315bedb2bf32fc4af0e50157
middleware.py
from django.contrib.auth.models import AnonymousUser
from rest_framework.authtoken.models import Token
from channels.db import database_sync_to_async
from channels.middleware import BaseMiddleware
#database_sync_to_async
def get_user(token_key):
try:
token = Token.objects.get(key=token_key)
return token.user
except Token.DoesNotExist:
return AnonymousUser()
class TokenAuthMiddleware(BaseMiddleware):
def __init__(self, inner):
super().__init__(inner)
async def __call__(self, scope, receive, send):
try:
token_key = (dict((x.split('=') for x in scope['query_string'].decode().split("&")))).get('token', None)
except ValueError:
token_key = None
scope['user'] = AnonymousUser() if token_key is None else await get_user(token_key)
return await super().__call__(scope, receive, send)
routing.py
from channels.security.websocket import AllowedHostsOriginValidator
from channels.routing import ProtocolTypeRouter, URLRouter
from .middleware import TokenAuthMiddleware
from main.consumers import MainConsumer
from django.conf.urls import url
application = ProtocolTypeRouter({
'websocket': AllowedHostsOriginValidator(
TokenAuthMiddleware(
URLRouter(
[
url(r"^main/$", MainConsumer.as_asgi()),
]
)
)
)
})
This answer is valid for channels 1.
You can find all information in this github issue:
https://github.com/django/channels/issues/510#issuecomment-288677354
I will summarise the discussion here.
copy this mixin into your project:
https://gist.github.com/leonardoo/9574251b3c7eefccd84fc38905110ce4
apply the decorator to ws_connect
the token is received in the app via an earlier authentication request to the /auth-token view in django-rest-framework. We use a querystring to send the token back to django-channels. If you're not using django-rest-framework you can consume the querystring in your own way. Read the mixin for how to get to it.
After using the mixin, and the correct token is used with the upgrade / connect request, the message will have a user like in the example below.
As you can see, we have has_permission() implemented on the User model, so it can just check its instance. If there is no token or the token is invalid, there will be no user on the message.
# get_group, get_group_category and get_id are specific to the way we named
# things in our implementation but I've included them for completeness.
# We use the URL `wss://www.website.com/ws/app_1234?token=3a5s4er34srd32`
def get_group(message):
return message.content['path'].strip('/').replace('ws/', '', 1)
def get_group_category(group):
partition = group.rpartition('_')
if partition[0]:
return partition[0]
else:
return group
def get_id(group):
return group.rpartition('_')[2]
def accept_connection(message, group):
message.reply_channel.send({'accept': True})
Group(group).add(message.reply_channel)
# here in connect_app we access the user on message
# that has been set by #rest_token_user
def connect_app(message, group):
if message.user.has_permission(pk=get_id(group)):
accept_connection(message, group)
#rest_token_user
def ws_connect(message):
group = get_group(message) # returns 'app_1234'
category = get_group_category(group) # returns 'app'
if category == 'app':
connect_app(message, group)
# sends the message contents to everyone in the same group
def ws_message(message):
Group(get_group(message)).send({'text': message.content['text']})
# removes this connection from its group. In this setup a
# connection wil only ever have one group.
def ws_disconnect(message):
Group(get_group(message)).discard(message.reply_channel)
thanks to github user leonardoo for sharing his mixin.
The following Django-Channels 2 middleware authenticates JWTs generated
by djangorestframework-jwt .
The token can be set via the djangorestframework-jwt http APIs, and it will also be sent for WebSocket connections if JWT_AUTH_COOKIE is defined.
settings.py
JWT_AUTH = {
'JWT_AUTH_COOKIE': 'JWT', # the cookie will also be sent on WebSocket connections
}
routing.py:
from channels.routing import ProtocolTypeRouter, URLRouter
from django.urls import path
from json_token_auth import JsonTokenAuthMiddlewareStack
from yourapp.consumers import SocketCostumer
application = ProtocolTypeRouter({
"websocket": JsonTokenAuthMiddlewareStack(
URLRouter([
path("socket/", SocketCostumer),
]),
),
})
json_token_auth.py
from http import cookies
from channels.auth import AuthMiddlewareStack
from django.contrib.auth.models import AnonymousUser
from django.db import close_old_connections
from rest_framework_jwt.authentication import BaseJSONWebTokenAuthentication
class JsonWebTokenAuthenticationFromScope(BaseJSONWebTokenAuthentication):
"""
Extracts the JWT from a channel scope (instead of an http request)
"""
def get_jwt_value(self, scope):
try:
cookie = next(x for x in scope['headers'] if x[0].decode('utf-8') == 'cookie')[1].decode('utf-8')
return cookies.SimpleCookie(cookie)['JWT'].value
except:
return None
class JsonTokenAuthMiddleware(BaseJSONWebTokenAuthentication):
"""
Token authorization middleware for Django Channels 2
"""
def __init__(self, inner):
self.inner = inner
def __call__(self, scope):
try:
# Close old database connections to prevent usage of timed out connections
close_old_connections()
user, jwt_value = JsonWebTokenAuthenticationFromScope().authenticate(scope)
scope['user'] = user
except:
scope['user'] = AnonymousUser()
return self.inner(scope)
def JsonTokenAuthMiddlewareStack(inner):
return JsonTokenAuthMiddleware(AuthMiddlewareStack(inner))
I believe sending token in query string can expose token even inside HTTPS protocols. To come around such issue I have used the following steps:
Create a token based REST API endpoint which creates temporary session and respond back with this session_key (This session is set to expire in 2 minutes)
login(request,request.user)#Create session with this user
request.session.set_expiry(2*60)#Make this session expire in 2Mins
return Response({'session_key':request.session.session_key})
Use this session_key in query parameter in channels parameter
I understand there is one extra API call but I believe it's much more secure than sending token in URL string.
Edit: This is just another approach to this problem, as discussed in comments, get parameters are exposed only in urls of http protocols, which should be avoided in anyhow.
from rest_framework_simplejwt.tokens import UntypedToken
from rest_framework_simplejwt.exceptions import InvalidToken, TokenError
from jwt import decode as jwt_decode
from urllib.parse import parse_qs
from django.contrib.auth import get_user_model
from channels.db import database_sync_to_async
from django.conf import settings
#database_sync_to_async
def get_user(user_id):
User = get_user_model()
try:
return User.objects.get(id=user_id)
except User.DoesNotExist:
return 'AnonymousUser'
class TokenAuthMiddleware:
def __init__(self, app):
# Store the ASGI application we were passed
self.app = app
async def __call__(self, scope, receive, send):
# Look up user from query string (you should also do things like
# checking if it is a valid user ID, or if scope["user"] is already
# populated).
token = parse_qs(scope["query_string"].decode("utf8"))["token"][0]
print(token)
try:
# This will automatically validate the token and raise an error if token is invalid
is_valid = UntypedToken(token)
except (InvalidToken, TokenError) as e:
# Token is invalid
print(e)
return None
else:
# Then token is valid, decode it
decoded_data = jwt_decode(token, settings.SECRET_KEY, algorithms=["HS256"])
print(decoded_data)
scope['user'] = await get_user(int(decoded_data.get('user_id', None)))
# Return the inner application directly and let it run everything else
return await self.app(scope, receive, send)
Asgi like this
import os
from channels.auth import AuthMiddlewareStack
from channels.routing import ProtocolTypeRouter, URLRouter
from django.core.asgi import get_asgi_application
from django.urls import path
from channelsAPI.routing import websocket_urlpatterns
from channelsAPI.token_auth import TokenAuthMiddleware
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'VirtualCurruncy.settings')
application = ProtocolTypeRouter({
"http": get_asgi_application(),
"websocket": TokenAuthMiddleware(
URLRouter([
path("virtualcoin/", websocket_urlpatterns),
])
),
})
ovveride custom AuthMiddleware
from urllib.parse import parse_qs
from channels.auth import AuthMiddleware
from channels.db import database_sync_to_async
from channels.sessions import CookieMiddleware, SessionMiddleware
from rest_framework.authtoken.models import Token
from django.contrib.auth.models import AnonymousUser
#database_sync_to_async
def get_user(scope):
query_string = parse_qs(scope['query_string'].decode())
token = query_string.get('token')
if not token:
return AnonymousUser()
try:
user = Token.objects.get(key=token[0]).user
except Exception as exception:
return AnonymousUser()
if not user.is_active:
return AnonymousUser()
return user
class TokenAuthMiddleware(AuthMiddleware):
async def resolve_scope(self, scope):
scope['user']._wrapped = await get_user(scope)
def TokenAuthMiddlewareStack(inner):
return CookieMiddleware(SessionMiddleware(TokenAuthMiddleware(inner)))
import the TokenAuthMiddlewareStack middleware in asgi.py
import os
from channels.auth import AuthMiddlewareStack
from channels.routing import ProtocolTypeRouter, URLRouter
from channels.security.websocket import AllowedHostsOriginValidator
from django.core.asgi import get_asgi_application
from chat.api.router_ws import urlpatterns_websocket
from .middleware import TokenAuthMiddlewareStack
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mysite.settings")
# Initialize Django ASGI application early to ensure the AppRegistry
# is populated before importing code that may import ORM models.
application = ProtocolTypeRouter({
"http": get_asgi_application(),
"websocket": AllowedHostsOriginValidator(
TokenAuthMiddlewareStack(
URLRouter(urlpatterns_websocket)
)
),
})
In frontend:new WebSocket(ws://8000/{your_path}?token=${localStorage.getItem('token')})
In Consumer: you can access the requested user as self.scope["user"]
Regarding Channels 1.x
As already pointed out here the mixin by leonardoo is the easiest way:
https://gist.github.com/leonardoo/9574251b3c7eefccd84fc38905110ce4
I think, however, it is somewhat confusing to figure out what the mixin is doing and what not, so I will try to make that clear:
When looking for a way to access message.user using the native django channels decorators you would have to implement it like this:
#channel_session_user_from_http
def ws_connect(message):
print(message.user)
pass
#channel_session_user
def ws_receive(message):
print(message.user)
pass
#channel_session_user
def ws_disconnect(message):
print(message.user)
pass
Channels does that by authenticating the user, creating a http_session and then converting the http_session in a channel_session, which uses the reply channel instead of cookies to identify the client.
All this is done in channel_session_user_from_http.
Have a look at the channels source code for more detail:
https://github.com/django/channels/blob/1.x/channels/sessions.py
leonardoo's decorator rest_token_user does, however, not create a channel session it simply stores the user in the message object in ws_connect. As the token is not sent again in ws_receive and the message object is not available either, in order to get the user in ws_receive and ws_disconnect as well, you would have to store it in the session yourself.
This would be a easy way to do this:
#rest_token_user #Set message.user
#channel_session #Create a channel session
def ws_connect(message):
message.channel_session['userId'] = message.user.id
message.channel_session.save()
pass
#channel_session
def ws_receive(message):
message.user = User.objects.get(id = message.channel_session['userId'])
pass
#channel_session
def ws_disconnect(message):
message.user = User.objects.get(id = message.channel_session['userId'])
pass
As Httpretty works on the Python socket layer, even Twisted web requests should be mocked out. But i am seeing some weird behavior on using httpretty. It tries to connect to localhost somehow. Below example shows the difference:
import httpretty
from twisted.internet import reactor
from twisted.internet.defer import Deferred
from twisted.web.client import Agent
from twisted.web.http_headers import Headers
import requests
#httpretty.activate
def main():
httpretty.register_uri(
httpretty.GET, "http://example.com",
body='[{"title": "Test Deal"}]',
content_type="application/json")
agent = Agent(reactor)
d = agent.request(
'GET',
'http://example.com',
Headers({'User-Agent': ['Twisted Web Client Example']}),
None)
def cbError(message):
print 'Async Failed : %s' % message
d.addErrback(cbError)
def cbShutdown(ignored): reactor.stop()
d.addBoth(cbShutdown)
reactor.run()
print 'Response received from Sync: %s' % \
requests.get('http://example.com').status_code
main()
And the response is :
Async Failed : [Failure instance: Traceback (failure with no frames): <class 'twisted.internet.error.ConnectionRefusedError'>: Connection was refused by other side: 111: Connection refused.
]
Response received from Sync: 200
How can i use httpretty with Twisted web client?
You can't. HTTPretty is blocking HTTP client libraries (like requests). It doesn't mock non-blocking sockets.