cronjob
*/30 * * * * /home/ubuntu/web-coin-crawler/venv/bin/python3 /home/ubuntu/web-coin-crawler/webcoincrawler/cron.crontab
cron.py, crontab() Function that crawls on two sites. Save to Django orm as save().
import json
import collections
import crawl_coinmarketcal as coinmarketcal
import crawl_coinscalendar as coinscalendar
import os
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
import django
import datetime
django.setup()
from crawled_data.models import BlogData
def preprocessingDict(dic: dict):
coin_dict = collections.defaultdict(dict)
for key, value in dic.items():
if value['symbol'] in coin_dict[value['date']]:
coin_dict[value['date']][value['symbol']].append([key, value['title'], value['name']])
else:
coin_dict[value['date']][value['symbol']] = [[key, value['title'], value['name']]]
return coin_dict
def crontab():
result = dict()
urls = coinmarketcal.get_urls()
for url in urls:
coinmarketcal.do_crawl(url, result)
urls = coinscalendar.get_urls()
for url in urls:
coinscalendar.do_crawl(url, result)
BlogData(title="COIN_DATA", content=json.dumps(preprocessingDict(result.copy()))).save()
I checked that the cron tab runs every 30 minutes.
However, no data was stored in Django orm.
I changed the Python file itself to run and it was solved.
cron.crontab -> cron
Related
I build Flask app with sqlite3 database and app.route add and app.route save
I have a problem with validators some of them works some does not works
validators.DataRequired() works
URLField() works
but validators.Length(min=1,max=15) does not works at all
from flask_wtf import FlaskForm #I aslo I also tried with Form
from wtforms import BooleanField, StringField, IntegerField, validators,SubmitField
from wtforms.fields.html5 import URLField
class AddRecValidators(FlaskForm): # <---I aslo I also tried with Form
title = StringField('Title:',[validators.DataRequired(), validators.Length(min=1,max=35,message="Title too long max 35 characters")])
authors = StringField('Authors:',[validators.Length(min=1,max=100)])
published_date = IntegerField('Published date:',[validators.Length(min=1,max=4)])
isbn_or_identifier = StringField('ISBN:',[validators.Length(min=1,max=15)])
page_count = IntegerField('Page count:',[ validators.Length(min=1,max=10000)])
language = StringField('Language:',[ validators.Length(min=1,max=3)])
image_links = URLField('Image links:')
submit = SubmitField(label=('Add to library'))
It looks like you're using the wrong validators for the type of input you're validating.
validators.Length() is for strings, see here
For the integers, try using NumberRange
from flask_wtf import FlaskForm
from wtforms import BooleanField, StringField, IntegerField, validators,SubmitField
from wtforms.fields.html5 import URLField
class AddRecValidators(FlaskForm):
title = StringField('Title:',[validators.DataRequired(), validators.Length(min=1,max=35,message="Title too long max 35 characters")])
authors = StringField('Authors:',[validators.Length(min=1,max=100)])
published_date = IntegerField('Published date:',[validators.NumberRange(min=1,max=4)]) # <-- note change to NumberRange
isbn_or_identifier = StringField('ISBN:',[validators.Length(min=1,max=15)])
page_count = IntegerField('Page count:',[ validators.NumberRange(min=1,max=10000)]) # <-- note change to NumberRange
language = StringField('Language:',[ validators.Length(min=1,max=3)])
image_links = URLField('Image links:')
submit = SubmitField(label=('Add to library'))
Also, here are the docs for flask-wtforms validators.
I am trying to deploy model to AKS. I am using AML SDK to register the model in the aml workspace. I am using PipelineModel module to save the model. And I am trying to load the model using PipelineModel.load. My entry script looks like below:
`
import os
import json
import pandas as pd
from azureml.core.model import Model
from pyspark.ml import PipelineModel
from mmlspark import ComputeModelStatistics
def init():
import mmlspark # this is needed to load mmlspark libraries
import logging
# extract and load model
global model, model_path
model_path = Model.get_model_path("{model_name}")
print(model_path)
print(os.stat(model_path))
print(os.path.exists(model_path))
#model_path = os.path.join(os.getenv("AZUREML_MODEL_DIR"), "{model_name}")
logging.basicConfig(level=logging.DEBUG)
#print(model_path)
#with ZipFile(model_path, 'r') as f:
# f.extractall('model')
model = PipelineModel.load(model_path)
#model = PipelineModel.read().load(model_path)
def run(input_json):
try:
output_df = model.transform(pd.read_json(input_json))
evaluator = ComputeModelStatistics().setScoredLabelsCol("prediction").setLabelCol("label").setEvaluationMetric("AUC")
result = evaluator.transform(predictions)
auc = result.select("AUC").collect()[0][0]
result = auc
except Exception as e:
result = str(e)
return json.dumps({{"result": result}})
`
It's giving error like below:
org.apache.hadoop.mapred.InvalidInputException: Input path does not exist: file:/var/azureml-app/azureml-models/lightgbm.model/2/lightgbm.model/metadata\n\tat org.apache.hadoop.mapred.FileInputFormat.singleThreadedListStatus(FileInputFormat.java:287)\n\tat org.apache.hadoop.mapred.FileInputFormat.listStatus(FileInputFormat.java:229)\n\tat org.apache.hadoop.mapred.FileInputFormat.getSplits(FileInputFormat.java:315).
os.path.exists returns true the path fetched from Model.get_model_path.
Am I missing something here?
I am trying to create a tag using ec2.resource method NOT ec2.CLIENT... I can create it with client but trying to figure out with resource. I am getting an error "errorMessage": "'dict' object has no attribute 'create_tags'". I have provided my code. I am comparing certain tag which I am using if statement but when I use create_tag method I get an error.
import boto3
import collections
import sys
ec2 = boto3.resource('ec2',region_name='us-east-2')
def lambda_handler(event, context):
vol = ec2.Volume(id='1256')
for delV in vol.tags:
delV.create_tags(Tags=[{'Key':'Name', 'Value':'Test1'}])
Try this, you don't need to iterate over the existing tags like you are doing;
import boto3
import collections
import sys
ec2 = boto3.resource('ec2',region_name='us-east-2')
def lambda_handler(event, context):
my_list_of_ids = [1256, 1234, 2300]
for volid in my_list_of_ids:
vol = ec2.Volume(id=volid)
for tag in vol.tags:
if tag['Name'] == 'Name' and tag['Value'] == Value:
print("Exists")
vol.create_tags(Tags=[{'Key':'Name', 'Value':'Test1'}])
I have tried running it from host='0.0.0.0' and it is still inaccessible. I can ping my windows machine ip 192.168.1.109 from my virtualmachine, but I can not ping my Ubuntu VirtualMachine ip from ifconfig 10.0.2.15 from my windows side. I am using virtualbox if that helps.
run.py
#!flask/bin/python
from app import app
app.run(host='0.0.0.0',port=5000, debug=True)
init.py
import os
from flask import Flask
from flask.json import JSONEncoder
from flask_sqlalchemy import SQLAlchemy
from flask_login import LoginManager
from flask_mail import Mail
from flask_babel import Babel, lazy_gettext
from config import basedir, ADMINS, MAIL_SERVER, MAIL_PORT, MAIL_USERNAME, \
MAIL_PASSWORD
from .momentjs import momentjs
app = Flask(__name__)
app.config.from_object('config')
db = SQLAlchemy(app)
lm = LoginManager()
lm.init_app(app)
lm.login_view = 'login'
lm.login_message = lazy_gettext('Please log in to access this page.')
mail = Mail(app)
babel = Babel(app)
class CustomJSONEncoder(JSONEncoder):
"""This class adds support for lazy translation texts to Flask's
JSON encoder. This is necessary when flashing translated texts."""
def default(self, obj):
from speaklater import is_lazy_string
if is_lazy_string(obj):
try:
return unicode(obj) # python 2
except NameError:
return str(obj) # python 3
return super(CustomJSONEncoder, self).default(obj)
app.json_encoder = CustomJSONEncoder
if not app.debug and MAIL_SERVER != '':
import logging
from logging.handlers import SMTPHandler
credentials = None
if MAIL_USERNAME or MAIL_PASSWORD:
credentials = (MAIL_USERNAME, MAIL_PASSWORD)
mail_handler = SMTPHandler((MAIL_SERVER, MAIL_PORT),
'no-reply#' + MAIL_SERVER, ADMINS,
'microblog failure', credentials)
mail_handler.setLevel(logging.ERROR)
app.logger.addHandler(mail_handler)
if not app.debug and os.environ.get('HEROKU') is None:
import logging
from logging.handlers import RotatingFileHandler
file_handler = RotatingFileHandler('tmp/microblog.log', 'a',
1 * 1024 * 1024, 10)
file_handler.setLevel(logging.INFO)
file_handler.setFormatter(logging.Formatter(
'%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]'))
app.logger.addHandler(file_handler)
app.logger.setLevel(logging.INFO)
app.logger.info('microblog startup')
if os.environ.get('HEROKU') is not None:
import logging
stream_handler = logging.StreamHandler()
app.logger.addHandler(stream_handler)
app.logger.setLevel(logging.INFO)
app.logger.info('microblog startup')
app.jinja_env.globals['momentjs'] = momentjs
from app import views, models
It worked for me when I changed this line in the main
app.run()
to
app.run(host='192.168.163.128', port=5000)
How do I use Django with the Tornado web server?
it's very simple ( especially with django 1.4) .
1 - just build your django project( and apps ) and make sure it works fine.
2- create a new python file at the root folder ( same dir where you used django-admin.py startproject)
3- then copy the code below , edit the os.environ['DJANGO_SETTINGS_MODULE'] line, and paste it in that new .py file.
import os
import tornado.httpserver
import tornado.ioloop
import tornado.wsgi
import sys
import django.core.handlers.wsgi
#sys.path.append('/home/lawgon/') # path to your project ( if you have it in another dir).
def main():
os.environ['DJANGO_SETTINGS_MODULE'] = 'myProject.settings' # path to your settings module
application = django.core.handlers.wsgi.WSGIHandler()
container = tornado.wsgi.WSGIContainer(application)
http_server = tornado.httpserver.HTTPServer(container)
http_server.listen(8888)
tornado.ioloop.IOLoop.instance().start()
if __name__ == "__main__":
main()
Django 1.6+ it should be like this:
import os
import tornado.httpserver
import tornado.ioloop
import tornado.wsgi
from django.core.wsgi import get_wsgi_application
def main():
os.environ['DJANGO_SETTINGS_MODULE'] = 'myproject.settings' # path to your settings module
application = get_wsgi_application()
container = tornado.wsgi.WSGIContainer(application)
http_server = tornado.httpserver.HTTPServer(container)
http_server.listen(8888)
tornado.ioloop.IOLoop.instance().start()
if __name__ == "__main__":
main()
UPDATE:
I created a minimal working demo which shows how to use the Tornado web server to run nicely with django:
https://github.com/tamasgal/django-tornado
ORIGINAL POST:
Just a remark: The WSGI application workflow has been changed from 1.6 to 1.7. You have to replace the import
import django.core.handlers.wsgi
with
from django.core.wsgi import get_wsgi_application
and change the application initialisation from
application = django.core.handlers.wsgi.WSGIHandler()
to
application = get_wsgi_application()
This is the modified code from the Moayyad Yaghi's answer:
import os
import tornado.httpserver
import tornado.ioloop
import tornado.wsgi
import sys
import django.core.handlers.wsgi
from django.core.wsgi import get_wsgi_application
#sys.path.append('/home/lawgon/') # path to your project ( if you have it in another dir).
def main():
os.environ['DJANGO_SETTINGS_MODULE'] = 'myProject.settings' # path to your settings module
application = django.core.handlers.wsgi.WSGIHandler()
application = get_wsgi_application()
container = tornado.wsgi.WSGIContainer(application)
http_server = tornado.httpserver.HTTPServer(container)
http_server.listen(8888)
tornado.ioloop.IOLoop.instance().start()
if __name__ == "__main__":
main()
There's a project called tornado-proxy that would help you. But I would like to recommend that you use Nginx. In the Nginx config you could now use proxy_pass to direct your calls like this:
location /comet {
proxy_pass http://localhost:8081;
}
location / {
proxy_pass http://localhost:8080;
}
In real world you would connect Django and some production-ready webserver with WSGI. This demo shows how you can run Tornado (and it's webserver) and Django side by side from one python module serving different URL prefixes: https://github.com/bdarnell/django-tornado-demo.
Tornado would block on serving any request directed to Django, though.