How to save user data to database instead of a pickle or a json file when trying to post videos on YouTube using Django and data v3 api - django-rest-framework

I'm trying to upload videos to youtube using Django and MSSQL, I want to store the user data to DB so that I can log in from multiple accounts and post videos.
The official documentation provided by youtube implements a file system and after login, all the user data gets saved there, I don't want to store any data in a file as saving files to DB would be a huge risk and not a good practice. So how can I bypass this step and save data directly to DB and retrieve it when I want to post videos to a specific account?
In short, I want to replace the pickle file implementation with storing it in the database.
Here's my code
def youtubeAuthenticate():
os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1"
api_service_name = "youtube"
api_version = "v3"
client_secrets_file = "client_secrets.json"
creds = None
# the file token.pickle stores the user's access and refresh tokens, and is
# created automatically when the authorization flow completes for the first time
if os.path.exists("token.pickle"):
with open("token.pickle", "rb") as token:
creds = pickle.load(token)
# if there are no (valid) credentials availablle, let the user log in.
if not creds or not creds.valid:
if creds and creds.expired and creds.refresh_token:
creds.refresh(Request())
else:
flow = InstalledAppFlow.from_client_secrets_file(client_secrets_file, SCOPES)
creds = flow.run_local_server(port=0)
# save the credentials for the next run
with open("token.pickle", "wb") as token:
pickle.dump(creds, token)
return build(api_service_name, api_version, credentials=creds)
#api_view(['GET','POST'])
def postVideoYT(request):
youtube = youtubeAuthenticate()
print('yt',youtube)
try:
initialize_upload(youtube, request.data)
except HttpError as e:
print("An HTTP error %d occurred:\n%s" % (e.resp.status, e.content))
return Response("Hello")
def initialize_upload(youtube, options):
print('options', options)
print("title", options['title'])
# tags = None
# if options.keywords:
# tags = options.keywords.split(",")
body=dict(
snippet=dict(
title=options['title'],
description=options['description'],
tags=options['keywords'],
categoryId=options['categoryId']
),
status=dict(
privacyStatus=options['privacyStatus']
)
)
# # Call the API's videos.insert method to create and upload the video.
insert_request = youtube.videos().insert(
part=",".join(body.keys()),
body=body,
media_body=MediaFileUpload(options['file'], chunksize=-1, resumable=True)
)
path = pathlib.Path(options['file'])
ext = path.suffix
getSize = os.path.getsize(options['file'])
resumable_upload(insert_request,ext,getSize)
# This method implements an exponential backoff strategy to resume a
# failed upload.
def resumable_upload(insert_request, ext, getSize):
response = None
error = None
retry = 0
while response is None:
try:
print("Uploading file...")
status, response = insert_request.next_chunk()
if response is not None:
respData = response
if 'id' in response:
print("Video id '%s' was successfully uploaded." % response['id'])
else:
exit("The upload failed with an unexpected response: %s" % response)
except HttpError as e:
if e.resp.status in RETRIABLE_STATUS_CODES:
error = "A retriable HTTP error %d occurred:\n%s" % (e.resp.status,
e.content)
else:
raise
except RETRIABLE_EXCEPTIONS as e:
error = "A retriable error occurred: %s" % e
if error is not None:
print(error)
retry += 1
if retry > MAX_RETRIES:
exit("No longer attempting to retry.")
max_sleep = 2 ** retry
sleep_seconds = random.random() * max_sleep
print("Sleeping %f seconds and then retrying..." % sleep_seconds)
time.sleep(sleep_seconds)

Related

Google API to find the storage size of the drive

I'm looking for a Google API to get the size of the drive, but I can't find anything. The code to delete a user's email using the Google API is provided below. Similarly, I need to know the size of the user's drive. Could someone please assist me? Is there a way to get the drive's size via an API? Thanks.
from __future__ import print_function
import os.path
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials
from google_auth_oauthlib.flow import InstalledAppFlow
from googleapiclient.discovery import build
# If modifying these scopes, delete the file token.json.
SCOPES = ['https://www.googleapis.com/auth/admin.directory.group', 'https://www.googleapis.com/auth/admin.directory.user']
def main():
"""Shows basic usage of the Admin SDK Directory API.
Prints the emails and names of the first 10 users in the domain.
"""
creds = None
# The file token.json stores the user's access and refresh tokens, and is
# created automatically when the authorization flow completes for the first
# time.
if os.path.exists('token.json'):
creds = Credentials.from_authorized_user_file('token.json', SCOPES)
# If there are no (valid) credentials available, let the user log in.
if not creds or not creds.valid:
if creds and creds.expired and creds.refresh_token:
creds.refresh(Request())
else:
flow = InstalledAppFlow.from_client_secrets_file(
'credentials.json', SCOPES)
creds = flow.run_local_server(port=0)
# Save the credentials for the next run
with open('token.json', 'w') as token:
token.write(creds.to_json())
service = build('admin', 'directory_v1', credentials=creds)
return service
def test():
# user = service.users().get(userKey="user1#matador.csun.edu").execute()
# members = service.groups().list(domain='my.csun.edu', userKey=user['primaryEmail'], pageToken=None, maxResults=500).execute()
# print(user)
# Call the Admin SDK Directory API
print('Getting the first 10 users in the domain')
results = service.users().list(customer='my_customer', maxResults=10,
orderBy='email').execute()
print(results)
users = results.get('users', [])
if not users:
print('No users in the domain.')
else:
print('Users:')
for user in users:
print(user)
# print(dir(user))
# print(u'{0} ({1})'.format(user['primaryEmail'],
# user['name']['fullName']))
def del_user(user):
try:
service.users().delete(userKey=user).execute()
print("Deleted!")
except:
print("User doesn't exist!")
if __name__ == '__main__':
service = main()
nameExt='23'
# with open('NewGmailInProd/gmailUser'+nameExt+'.txt') as fileToRead:
# with open('NewGmailInProd/test.txt') as fileToRead:
# emails = fileToRead.readlines()
emails = ['user1#matador.csun.edu']
for email in emails:
del_user(email.strip())
The google drive api has an endpoint called about.get This endpoint returns a lot of information about a users drive account try me
One of the things it returns is the users storage quota.
You appear to be trying to go though the Admin SDK Directory API it just gives you access to administer Workspace its not going to give you anything with drive

Pyarrow basic auth: How to prevent `Stream is closed`?

I am new to Arrow Flight and pyarrow (v=6.0.1), and am trying to implement basic auth but I am always getting an error:
OSError: Stream is closed
I have created a minimal reproducing sample, by running the following two files sequentially (representing server and client respectively):
from typing import Dict, Union
from pyarrow.lib import tobytes
from pyarrow.flight import BasicAuth, FlightUnauthenticatedError, ServerAuthHandler, FlightServerBase
from pyarrow._flight import ServerAuthSender, ServerAuthReader
class ServerBasicAuthHandler(ServerAuthHandler):
def __init__(self, creds: Dict[str, str]):
self.creds = {user.encode(): pw.encode() for user, pw in creds.items()}
def authenticate(self, outgoing: ServerAuthSender, incoming: ServerAuthReader):
buf = incoming.read() # this line raises "OSError: Stream is closed"
auth = BasicAuth.deserialize(buf)
if auth.username not in self.creds:
raise FlightUnauthenticatedError("unknown user")
if self.creds[auth.username] != auth.password:
raise FlightUnauthenticatedError("wrong password")
outgoing.write(tobytes(auth.username))
def is_valid(self, token: bytes) -> Union[bytes, str]:
if not token:
raise FlightUnauthenticatedError("no basic auth provided")
if token not in self.creds:
raise FlightUnauthenticatedError("unknown user")
return token
service = FlightServerBase(
location=f"grpc://[::]:50051",
auth_handler=ServerBasicAuthHandler({"user": "pw"}),
)
service.serve()
from pyarrow.flight import FlightClient
client = FlightClient(location=f"grpc://localhost:50051")
client.authenticate_basic_token("user", "pw")
I basically copied the ServerAuthHandler implementation from their tests, so it is proven to work. However, I can't get it to work.
The error message Stream is closed hard to debug. I don't know where it comes from and I can't trace it to anywhere within the pyarrow implementation (neither Pythonside nor C++ side). I can't see where it comes from.
Any help or hints on how to prevent this error would be appreciated.
The example in the OP is mixing up two authentication implementations (which is indeed confusing). The "BasicAuth" object isn't actual HTTP basic authentication that the authenticate_basic_token method implements; this is because contributors have implemented a variety of authentication methods over the years. The actual test is as follows:
header_auth_server_middleware_factory = HeaderAuthServerMiddlewareFactory()
no_op_auth_handler = NoopAuthHandler()
def test_authenticate_basic_token():
"""Test authenticate_basic_token with bearer token and auth headers."""
with HeaderAuthFlightServer(auth_handler=no_op_auth_handler, middleware={
"auth": HeaderAuthServerMiddlewareFactory()
}) as server:
client = FlightClient(('localhost', server.port))
token_pair = client.authenticate_basic_token(b'test', b'password')
assert token_pair[0] == b'authorization'
assert token_pair[1] == b'Bearer token1234'
i.e. we're not using authenticate but rather a "middleware" to do the implementation. A full example looks as follows:
import base64
import pyarrow.flight as flight
class BasicAuthServerMiddlewareFactory(flight.ServerMiddlewareFactory):
def __init__(self, creds):
self.creds = creds
def start_call(self, info, headers):
token = None
for header in headers:
if header.lower() == "authorization":
token = headers[header]
break
if not token:
raise flight.FlightUnauthenticatedError("No credentials supplied")
values = token[0].split(' ', 1)
if values[0] == 'Basic':
decoded = base64.b64decode(values[1])
pair = decoded.decode("utf-8").split(':')
if pair[0] not in self.creds:
raise flight.FlightUnauthenticatedError("No credentials supplied")
if pair[1] != self.creds[pair[0]]:
raise flight.FlightUnauthenticatedError("No credentials supplied")
return BasicAuthServerMiddleware("BearerTokenValue")
raise flight.FlightUnauthenticatedError("No credentials supplied")
class BasicAuthServerMiddleware(flight.ServerMiddleware):
def __init__(self, token):
self.token = token
def sending_headers(self):
return {'authorization': f'Bearer {self.token}'}
class NoOpAuthHandler(flight.ServerAuthHandler):
def authenticate(self, outgoing, incoming):
pass
def is_valid(self, token):
return ""
with flight.FlightServerBase(auth_handler=NoOpAuthHandler(), middleware={
"basic": BasicAuthServerMiddlewareFactory({"test": "password"})
}) as server:
client = flight.connect(('localhost', server.port))
token_pair = client.authenticate_basic_token(b'test', b'password')
print(token_pair)
assert token_pair[0] == b'authorization'
assert token_pair[1] == b'Bearer BearerTokenValue'
I think it's simply due to this not being supported on Windows.
On closer inspection, the test that "proves it work" is being skipped in Windows. The comment refers to this issue. That issue has been fixed though (ostensibly); not anything as to why it wouldn't work with Stream is closed.

"Insufficient permission to access this report." - youtube analytics api

I am trying to run this report:
SCOPES = ['https://www.googleapis.com/auth/yt-analytics.readonly','https://www.googleapis.com/auth/yt-analytics.force-ssl','https://www.googleapis.com/auth/yt-analytics-monetary.readonly','https://www.googleapis.com/auth/youtube.readonly']
API_SERVICE_NAME = 'youtubeAnalytics'
API_VERSION = 'v2'
CLIENT_SECRETS_FILE = '/Users/secret.json'
def initialize_analyticsreporting():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
parents=[tools.argparser])
flags = parser.parse_args([])
flow = client.flow_from_clientsecrets(
CLIENT_SECRETS_FILE, scope=SCOPES,
message=tools.message_if_missing(CLIENT_SECRETS_FILE))
storage = file.Storage('analyticsreporting.dat')
credentials = storage.get()
if credentials is None or credentials.invalid:
credentials = tools.run_flow(flow, storage, flags)
http = credentials.authorize(http=httplib2.Http())
analytics = build('youtubeAnalytics', 'v2', http=http)
return analytics
def execute_api_request(client_library_function, **kwargs):
response = client_library_function(
**kwargs
).execute()
print(response)
youtubeAnalytics = initialize_analyticsreporting()
execute_api_request(
youtubeAnalytics.reports().query,
ids='channel==MINE',
startDate='2020-05-01',
endDate='2020-12-31',
dimensions='video',
metrics='views,likes,dislikes,shares,adImpressions',
maxResults=200,
sort='-views'
)
But I get :
"Insufficient permission to access this report."
I am authenticated with OAuth, I am the content owner but I cant get the adImpressions metric to work.
My end goal is to simply get the impressions of a video using youtube analytics API. I've seen multiple threads on this topic but neither answers the question.
As DalmTo mentioned in the comments:
After adding additional scopes:
SCOPES = ['https://www.googleapis.com/auth/yt-analytics.readonly']
To
SCOPES = ['https://www.googleapis.com/auth/yt-analytics.readonly',
'https://www.googleapis.com/auth/yt-analytics-monetary.readonly',
'https://www.googleapis.com/auth/youtube.readonly']
And reauthenticating - deleting the .dat file, and letting a new one to be created, now I am able to receive the desired metrics.

How to get email message in outlook account using IMAP?

I tried one sample program for getting an email message in outlook account using IMAP. In this account, I have 20 folders its getting all email messages except these folders (contact, calendar, task) not getting data its throwing server error. How to fix this error.
Code
import imaplib
import pprint
import email
import base64
import json
import re
import os
import fileinput
imap_host = 'outlook.office365.com'
imap_user = 'XXXXXXXXXXX'
imap_pass = 'XXXXXXXXXXXXX'
count = 0
file_path = 'geek.txt'
# connect to host using SSL
imap = imaplib.IMAP4_SSL(imap_host,993)
# login to server
l = imap.login(imap_user, imap_pass)
# Get Flags,mailbox_name,delimiter using regex
list_response_pattern = re.compile(r'\((?P<flags>.*?)\) "(?P<delimiter>.*)" (?P<name>.*)')
# Get List of Sync folders
list_data = imap.list()
# Check Local Storage is empty Sync All Folders Details.
print(os.stat(file_path).st_size)
if os.stat(file_path).st_size == 0:
global day
# Iterate folders in Sync folder
for i in list_data[1]:
# Get Folder name
sample = re.findall('"\/"(.*)',i.decode("utf-8"))
# Get Message_ids
try:
print("message")
print(sample[0].lstrip().strip('"'))
data = imap.select(sample[0].lstrip())
search_resp, search_data = imap.search( None, "ALL" )
match = list_response_pattern.match(i.decode("utf-8"))
flags, delimiter, mailbox_name = match.groups()
print("1")
print(mailbox_name)
mailbox_name = mailbox_name.strip('"')
print(mailbox_name)
except Exception as e:
print(e)
continue
# Get Current Status of Folder
current_status = imap.status(
'"{}"'.format(mailbox_name),
'(MESSAGES RECENT UIDNEXT UIDVALIDITY UNSEEN)',
)
print(current_status)
# Get message using UID and Message_id
msg_ids = search_data[ 0 ].split()
print("total count: ",len(msg_ids))
for i in msg_ids:
print("$$$$$$$$$$$$$$$$$$$$")
print("Message Ids: ", i)
count = count + 1
fetch_resp, fetch_UID = imap.fetch( i, 'UID' )
print("Fetch UID: ", fetch_UID)
day = bytes(str(fetch_UID[0].split()[2]).split("'")[1].split(')')[0],'utf-8')
print("ID: ",day)
fetch_resp, fetch_mdg = imap.uid('fetch', day, '(RFC822)')
print(fetch_mdg)
print("$$$$$$$$$$$$$$$$$$$$$")
email_msg = fetch_mdg[0][1]
if email_msg and isinstance(email_msg, str):
try:
email_msg = email.message_from_string(email_msg)
except :
email_msg = None
elif email_msg and isinstance(email_msg, bytes):
try:
email_msg = email.message_from_bytes(email_msg)
except:
email_msg = None
print("*********************************")
print("Count: ",count)
print("UID: ",day)
print(mailbox_name)
print(email_msg['To'])
print(email_msg['From'])
print(email_msg['subject'])
print(email_msg)
print("*********************************")
# Store Folder details in File
status_details = current_status[1][0].decode("utf-8")
status_details = status_details.split('(')[1].split(')')[0].split(' ')
print(status_details)
if len(msg_ids) == 0:
json1 = json.dumps({'total_count':int(status_details[1]),'UID':0,'UIDNext':int(status_details[5]),'UIDValidity':int(status_details[7]), 'Folder name':mailbox_name})
else:
json1 = json.dumps({'total_count':int(status_details[1]),'UID':int(day),'UIDNext':int(status_details[5]),'UIDValidity':int(status_details[7]), 'Folder name':mailbox_name})
file = open(file_path,'a')
file.write(json1)
file.write("\n")
print('hi')
Response
$$$$$$$$$$$$$$$$$$$$
Message Ids: b'3'
Fetch UID: [b'3 (UID 11)']
ID: b'11'
[(b'3 (RFC822 {757}', b'MIME-Version: 1.0\r\nContent-Type: text/plain; charset="us-ascii"\r\nFrom: Microsoft Exchange Server\r\nTo: "\r\nSubject: Retrieval using the IMAP4 protocol failed for the following message:\r\n 11\r\nContent-Transfer-Encoding: quoted-printable\r\n\r\nThe server couldn\'t retrieve the following message:\r\n\r\nSubject: "Test email Sync 3"\r\nFrom: "Imap Testing" ("/O=3DEXCHANGELABS/OU=3DEXCHANGE ADMINISTRATIVE GROUP=\r\n (FYDIBOHF23SPDLT)/CN=3DRECIPIENTS/CN=3DEBF2483D9A0145A59A48B829B12A45E4-MA=\r\nILBOX1")\r\nSent date: 5/6/2020 2:02:59 AM\r\n\r\nThe message hasn\'t been deleted. You might be able to view it using either =\r\nOutlook or Outlook Web App. You can also contact the sender to find out wha=\r\nt the message says.=\r\n'), b' UID 11 FLAGS (\\Seen))']
$$$$$$$$$$$$$$$$$$$$$
Server Error
Subject: Retrieval using the IMAP4 protocol failed for the following message:
7
Content-Transfer-Encoding: quoted-printable
The server couldn't retrieve the following message:
Subject: "Testing"
Sent date: 5/6/2020 2:01:54 AM
The message hasn't been deleted. You might be able to view it using either =
Outlook or Outlook Web App. You can also contact the sender to find out wha=
t the message says.=
I have around 20 folders I iterate one by one get current status of folder and stored in sample file. Its successfully working.but I tried to print email messages some folders (contact,calender,task) its showing this response.

How to find Knowledge base ID (kbid) for QnAMaker?

I am trying to integrate QnAmaker knowledge base with Azure Bot Service.
I am unable to find knowledge base id on QnAMaker portal.
How to find the kbid in QnAPortal?
The Knowledge Base Id can be located in Settings under “Deployment details” in your knowledge base. It is the guid that is nestled between “knowledgebases” and “generateAnswer” in the POST (see image below).
Hope of help!
Hey you can also use python to get this by take a look at the following code.
That is if you wanted to write a program to dynamically get the kb ids.
import http.client, os, urllib.parse, json, time, sys
# Represents the various elements used to create HTTP request path for QnA Maker
operations.
# Replace this with a valid subscription key.
# User host = '<your-resource-name>.cognitiveservices.azure.com'
host = '<your-resource-name>.cognitiveservices.azure.com'
subscription_key = '<QnA-Key>'
get_kb_method = '/qnamaker/v4.0/knowledgebases/'
try:
headers = {
'Ocp-Apim-Subscription-Key': subscription_key,
'Content-Type': 'application/json'
}
conn = http.client.HTTPSConnection(host)
conn.request ("GET", get_kb_method, None, headers)
response = conn.getresponse()
data = response.read().decode("UTF-8")
result = None
if len(data) > 0:
result = json.loads(data)
print
#print(json.dumps(result, sort_keys=True, indent=2))
# Note status code 204 means success.
KB_id = result["knowledgebases"][0]["id"]
print(response.status)
print(KB_id)
except :
print ("Unexpected error:", sys.exc_info()[0])
print ("Unexpected error:", sys.exc_info()[1])

Resources