How to run playbook api in Ansible v2 with vault - ansible

Here is what I have, I know this works without encryption and I can run
ansible-vault edit common.yml
with
ANSIBLE_VAULT_PASSWORD_FILE=~/.vault_pass.txt
set in the env.
from collections import namedtuple
from ansible.parsing.dataloader import DataLoader
from ansible.vars import VariableManager
from ansible.inventory import Inventory
from ansible.playbook import Playbook
from ansible.executor.playbook_executor import PlaybookExecutor
variable_manager = VariableManager()
loader = DataLoader()
inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list='playbooks/hosts')
playbook_path = 'playbooks/' + PROJECT + '.yml'
Options = namedtuple('Options', ['connection', 'forks', 'become', 'become_method', 'become_user', 'check', 'listhosts', 'listtasks', 'listtags', 'syntax', 'module_path', 'vault_password_file'])
options = Options(connection='ssh', forks=5, become=None, become_method=None, become_user=None, check=False, listhosts=False, listtasks=False, listtags=False, syntax=False, module_path="", vault_password_file=os.environ['ANSIBLE_VAULT_PASSWORD_FILE'])
variable_manager.extra_vars = {'CAP_VERSION': CAP_VERSION, 'cluster': PROJECT + '-' + ENVIRONMENT, 'environ': ENVIRONMENT, 'rpm': rpmSource, 'VRSN': ARTI_BRANCH }
passwords = {}
pbex = PlaybookExecutor(playbooks=[playbook_path], inventory=inventory, variable_manager=variable_manager, loader=loader, options=options, passwords=passwords)
results = pbex.run()
It fails to decrypt the common.yml
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/opt/ansible/ansible/lib/ansible/executor/playbook_executor.py", line 125, in run
all_vars = self._variable_manager.get_vars(loader=self._loader, play=play)
File "/opt/ansible/ansible/lib/ansible/vars/__init__.py", line 304, in get_vars
data = preprocess_vars(loader.load_from_file(vars_file))
File "/opt/ansible/ansible/lib/ansible/parsing/dataloader.py", line 119, in load_from_file
(file_data, show_content) = self._get_file_contents(file_name)
File "/opt/ansible/ansible/lib/ansible/parsing/dataloader.py", line 178, in _get_file_contents
data = self._vault.decrypt(data, filename=b_file_name)
File "/opt/ansible/ansible/lib/ansible/parsing/vault/__init__.py", line 264, in decrypt
raise AnsibleError(msg)
ansible.errors.AnsibleError: Decryption failed on /ansible/playbooks/vars/common.yml

In ansible 2.2.2 (not sure about other versions since the API can change frequently):
You can manually set the password in the python script like so:
loader = DataLoader()
loader.set_vault_password('mypass')
Or you could load the password from your vault password file:
import os
loader = DataLoader()
with open('{}/.vault_pass.txt'.format(os.path.expanduser('~')), 'r') as file:
loader.set_vault_password(file.read().splitlines()[0])
You can skip importing os and just put in your absolute path to the .vault_pass.txt file.
If you are sure your ANSIBLE_VAULT_PASSWORD_FILE is set in env:
import os
loader = DataLoader()
with open(os.environ['ANSIBLE_VAULT_PASSWORD_FILE'], 'r') as file:
loader.set_vault_password(file.read().splitlines()[0])

Related

How to solve ValueError('Invalid async_mode specified') for flask-socketio?

I'm testing a flask-socketio server in bitbucket pipeline. It failed with the following messages:
Traceback (most recent call last):
File "/root/.local/share/virtualenvs/build-3vGKWv3F/lib/python3.7/site-packages/flask_failsafe.py", line 29, in wrapper
return func(*args, **kwargs)
File "/opt/atlassian/pipelines/agent/build/main.py", line 89, in create_app
return cell_data_api.create_app()
File "/opt/atlassian/pipelines/agent/build/cell_data_api/__init__.py", line 30, in create_app
socketio.init_app(app)
File "/root/.local/share/virtualenvs/build-3vGKWv3F/lib/python3.7/site-packages/flask_socketio/__init__.py", line 243, in init_app
self.server = socketio.Server(**self.server_options)
File "/root/.local/share/virtualenvs/build-3vGKWv3F/lib/python3.7/site-packages/socketio/server.py", line 127, in __init__
self.eio = self._engineio_server_class()(**engineio_options)
File "/root/.local/share/virtualenvs/build-3vGKWv3F/lib/python3.7/site-packages/engineio/server.py", line 145, in __init__
raise ValueError('Invalid async_mode specified')
ValueError: Invalid async_mode specified
Traceback (most recent call last):
File "/.pyenv/versions/3.7.4/lib/python3.7/runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "/.pyenv/versions/3.7.4/lib/python3.7/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/opt/atlassian/pipelines/agent/build/main.py", line 117, in <module>
socketio.run(APP, host='0.0.0.0', port=8080, debug=True, use_reloader=True)
File "/root/.local/share/virtualenvs/build-3vGKWv3F/lib/python3.7/site-packages/flask_socketio/__init__.py", line 564, in run
if app.debug and self.server.eio.async_mode != 'threading':
AttributeError: 'NoneType' object has no attribute 'eio'
My main.py file looks like:
import os
from cell_data_api import socketio
# Detect if we are running in App Engine
# Make sure this does NOT start if we are running a Cloud Function
if os.getenv('APP_ENGINE', '') == 'TRUE':
import cell_data_api
APP = cell_data_api.create_app()
if __name__ == '__main__':
from flask_failsafe import failsafe
#failsafe
def create_app():
# note that the import is *inside* this function so that we can catch
# errors that happen at import time
import cell_data_api
# If `entrypoint` is not defined in app.yaml, App Engine will look for an app
# called `app` in `main.py`.
return cell_data_api.create_app()
APP = create_app()
# This is used when running locally only. When deploying to Google App
# Engine, a webserver process such as Gunicorn will serve the app. This
# can be configured by adding an `entrypoint` to app.yaml.
socketio.run(APP, host='0.0.0.0', port=8080, debug=True, use_reloader=True)
Main.py imports from cell_data_api.py, which looks like:
import os
from flask import Flask
from flask_cors import CORS
# import eventlet
from engineio.async_drivers import eventlet
from flask_socketio import SocketIO
socketio = SocketIO(
always_connect=True,
logger=True,
async_mode=eventlet,
cookie=...,
ping_timeout=...
)
def create_app():
# create and configure the app
app = Flask(__name__)
CORS(app)
......
socketio.init_app(app)
# ensure the instance folder exists
try:
os.makedirs(app.instance_path)
except OSError:
pass
return app
My environment is Python 3.7 with installation packages:
[dev-packages]
alembic = "*"
flask_failsafe = "*"
wcwidth = "*"
[packages]
flask = "*"
absl-py = "*"
flask-cors = "*"
grpcio = "*"
transitions = "*"
sqlalchemy-json = "*"
sqlalchemy = "1.3.0"
flask_socketio='*'
eventlet='*'
Unlike the other two questions I found about the same error, I'm not using pyinstaller or cx_Freeze.
The async_mode parameter takes a string as an argument.
Instead of this:
async_mode=eventlet,
Do this:
async_mode='eventlet',

ModuleNotFoundError: No module named 'pycaret.internal'

I am trying to deploy a ML model using Streamlit and Pycaret on Heroku.
When I try deploying the app, I get the following error:
ModuleNotFoundError: No module named 'pycaret.internal'
Traceback:
File "/app/.heroku/python/lib/python3.6/site-packages/streamlit/ScriptRunner.py", line 322, in _run_script
exec(code, module.__dict__)
File "/app/Final.py", line 11, in <module>
tuned_cat=joblib.load('Cat.pkl')
File "/app/.heroku/python/lib/python3.6/site-packages/joblib/numpy_pickle.py", line 585, in load
obj = _unpickle(fobj, filename, mmap_mode)
File "/app/.heroku/python/lib/python3.6/site-packages/joblib/numpy_pickle.py", line 504, in _unpickle
obj = unpickler.load()
File "/app/.heroku/python/lib/python3.6/pickle.py", line 1050, in load
dispatch[key[0]](self)
File "/app/.heroku/python/lib/python3.6/pickle.py", line 1338, in load_global
klass = self.find_class(module, name)
File "/app/.heroku/python/lib/python3.6/pickle.py", line 1388, in find_class
__import__(module, level=0)
I have the following dependencies in the requirements.txt file:
pycaret==1.0.0
streamlit==0.58.0
I had to go with the pycaret version 1.0 because when I chose the latest version the slug size on Heroku would go beyond 500M and it would not deploy. Compiled slug size: 552M is too large (max is 500M).
Link to Github
Final.py:
import streamlit as st
import joblib
from pycaret.classification import *
tuned_cat=joblib.load('Cat.pkl')
import tensorflow as tf
physical_devices = tf.config.list_physical_devices('CPU')
def run():
add_selectbox = st.sidebar.selectbox(
"What would you like to do?",
("Online Prediction","Batch Prediction"))
st.sidebar.info('This app is created to predict if the applicant should be granted a loan or not.')
st.title("Loan Prediction App")
if add_selectbox == 'Online Prediction':
gender = st.selectbox('Gender',['Female','Male'])
married = st.selectbox('Married',['No','Yes'])
depend = st.selectbox('Dependents',['0','1','2','3+'])
edu = st.selectbox('Education',['Graduate','Not Graduate'])
self = st.selectbox('Self Employed',['No','Yes'])
app_inc = st.number_input ('Applicant Income')
co_inc = st.number_input ('Coapplicant Income')
amt = st.number_input ('Loan Amount')
term = st.number_input ('Loan Amount Term')
credit = st.selectbox('Credit History',['0','1'])
prop_are = st.selectbox('Property Area',['Rural','Semiurban','Urban'])
output=""
test_df = pd.DataFrame()
test_df['Gender']= [gender]
test_df['Married']=[married]
test_df['Dependents']=[depend]
test_df['Education']=[edu]
test_df['Self_Employed']=[self]
test_df['ApplicantIncome']=[app_inc]
test_df['CoapplicantIncome']=[co_inc]
test_df['LoanAmount']=[amt]
test_df['Loan_Amount_Term']=[term]
test_df['Credit_History']=[credit]
test_df['Property_Area']=[prop_are]
if st.button("Predict"):
Cat_pred=predict_model(tuned_cat,data=test_df)['Label']
output = Cat_pred.values
if(output==0):
text="Rejected"
st.error(text)
elif(output==1):
text="Approved"
st.success(text)
if add_selectbox == 'Batch Prediction':
file_upload = st.file_uploader("Upload excel file for predictions", type=["xlsx"])
if file_upload is not None:
data = pd.read_excel(file_upload)
st.success('File uploaded successfully!')
Cat_pred=predict_model(tuned_cat,data=data)['Label']
data['Prediction']=Cat_pred
st.write(data)
st.markdown(get_table_download_link(data), unsafe_allow_html=True)
import base64
def get_table_download_link(df):
"""Generates a link allowing the data in a given panda dataframe to be downloaded
in: dataframe
out: href string
"""
csv = df.to_csv(index=False)
b64 = base64.b64encode(
csv.encode()
).decode() # some strings <-> bytes conversions necessary here
return f'Download csv file'
if __name__ == '__main__':
run()

How to integrate gspread with aiohttp

I am trying to use gspread, but I need the library to mesh well with another async library I am using.
After digging through the docs for gspread, I found this function that I can use:
class gspread.Client(auth, session=None)
An instance of this class communicates with Google API.
Parameters:
auth – An OAuth2 credential object. Credential objects are those created by the oauth2client library. https://github.com/google/oauth2client
session – (optional) A session object capable of making HTTP requests while persisting some parameters across requests. Defaults to requests.Session.
Which gives me an optional session parameter. How would I specify the session to use aiohttp?
I wrote a bit of test code, which compiles fine, but running the code crashes.
import aiohttp
import gspread
import random
from oauth2client.service_account import ServiceAccountCredentials
scope = ['https://spreadsheets.google.com/feeds',
'https://www.googleapis.com/auth/drive']
creds = ServiceAccountCredentials.from_json_keyfile_name('client_secret.json', scope)
c = gspread.authorize(creds)
client = gspread.Client(auth=c, session=aiohttp.ClientSession)
sheet = client.open_by_key('1Hkwo9gSpk3NjgPLPkG8kh0zBNw2nxsYWRw0cVdn0JA0')
ws = sheet.get_worksheet(0)
rcount = ws.row_count
msg = ws.cell(random.randint(1,rcount),1).value
print(msg)
The error message I get is below:
Traceback (most recent call last):
File "c:\Users\xxxxx\.vscode\extensions\ms-python.python-2018.9.0\pythonFiles\experimental\ptvsd_launcher.py", line 118, in <module>
vspd.debug(filename, port_num, '', '', run_as)
File "c:\Users\xxxxx\.vscode\extensions\ms-python.python-2018.9.0\pythonFiles\experimental\ptvsd\ptvsd\debugger.py", line 37, in debug
run(address, filename, *args, **kwargs)
File "c:\Users\xxxxx\.vscode\extensions\ms-python.python-2018.9.0\pythonFiles\experimental\ptvsd\ptvsd\_local.py", line 79, in run_file
run(argv, addr, **kwargs)
File "c:\Users\xxxxx\.vscode\extensions\ms-python.python-2018.9.0\pythonFiles\experimental\ptvsd\ptvsd\_local.py", line 140, in _run
_pydevd.main()
File "c:\Users\xxxxx\.vscode\extensions\ms-python.python-2018.9.0\pythonFiles\experimental\ptvsd\ptvsd\_vendored\pydevd\pydevd.py", line 1751, in main
debugger.connect(host, port)
File "c:\Users\xxxxx\.vscode\extensions\ms-python.python-2018.9.0\pythonFiles\experimental\ptvsd\ptvsd\_vendored\pydevd\pydevd.py", line 1107, in run
return self._exec(is_module, entry_point_fn, module_name, file, globals, locals)
File "c:\Users\xxxxx\.vscode\extensions\ms-python.python-2018.9.0\pythonFiles\experimental\ptvsd\ptvsd\_vendored\pydevd\pydevd.py", line 1114, in _exec
pydev_imports.execfile(file, globals, locals) # execute the script
File "c:\Users\xxxxx\.vscode\extensions\ms-python.python-2018.9.0\pythonFiles\experimental\ptvsd\ptvsd\_vendored\pydevd\_pydev_imps\_pydev_execfile.py", line 25, in execfile
exec(compile(contents+"\n", file, 'exec'), glob, loc)
File "c:\Users\xxxxx\Desktop\Coding\Discord Bot\Testing\test.py", line 14, in <module>
ws = sheet.get_worksheet(0)
File "C:\Users\xxxxx\AppData\Local\Programs\Python\Python36\lib\site-packages\gspread\models.py", line 141, in get_worksheet
sheet_data = self.fetch_sheet_metadata()
File "C:\Users\xxxxx\AppData\Local\Programs\Python\Python36\lib\site-packages\gspread\models.py", line 123, in fetch_sheet_metadata
r = self.client.request('get', url, params=params)
File "C:\Users\xxxxx\AppData\Local\Programs\Python\Python36\lib\site-packages\gspread\client.py", line 73, in request
headers=headers
TypeError: get() missing 1 required positional argument: 'url'
PS C:\Users\xxxxx\Desktop\Coding\Discord Bot>
Any ideas?
It's impossible to use aiohttp with current version of gspread (3.0.1). the gspread library uses synchronous calls and aiohttp uses asynchronous calls.
Please, reconsider to use compatible library like requests or httplib2.
For anyone wondering, once I posted this question, I found that someone made an async wrapper for gspread. Check out the library here and show this guy your appreciation. I sure am!

Why is pypandoc giving the error "RuntimeError: Missing format!" in this setup.py script?

I have a setup.py script for a module. This setup.py script features conversion of a Markdown README file to reStructuredText (suitable for PyPI). When an attempt is made to install this module using pip (sudo pip install supermodule), the following error is presented:
Collecting supermodule
Downloading supermodule-2017.1.12.2329.tar.gz
Complete output from command python setup.py egg_info:
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "/tmp/pip-build-GijMOu/supermodule/setup.py", line 43, in <module>
main()
File "/tmp/pip-build-GijMOu/supermodule/setup.py", line 14, in main
long_description = pypandoc.convert("README.md", "rst"),
File "/usr/local/lib/python2.7/dist-packages/pypandoc/__init__.py", line 50, in convert
outputfile=outputfile, filters=filters)
File "/usr/local/lib/python2.7/dist-packages/pypandoc/__init__.py", line 68, in _convert
raise RuntimeError('Missing format!')
RuntimeError: Missing format!
What is going wrong and how can I fix this?
The setup.py script is as follows:
#!/usr/bin/python
# -*- coding: utf-8 -*-
import os
import pypandoc
import setuptools
def main():
setuptools.setup(
name = "supermodule",
version = "2017.01.12.2329",
description = "super module",
long_description = pypandoc.convert("README.md", "rst"),
url = "https://github.com/user/supermodule",
author = "A. Person",
author_email = "ap#sern.ch",
license = "GPLv3",
py_modules = [
"supermodule"
],
install_requires = [
"opencv",
"docopt",
"propyte",
"shijian",
"tonescale"
],
scripts = [
"supermodule_test.py"
],
entry_points = """
[console_scripts]
supermodule = supermodule:supermodule
"""
)
def read(*paths):
with open(os.path.join(*paths), "r") as filename:
return filename.read()
if __name__ == "__main__":
main()

Error 1053 When Starting Window Service Written In Python

I have already looked at and tried the resolutions to this question that others have posted. One user said that to try and change my setup.py file from:
from distutils.core import setup
import py2exe
setup(console=["dev.py"])
to
from distutils.core import setup
import py2exe
setup(service=["dev.py"])
I got the following results:
running py2exe
*** searching for required modules ***
Traceback (most recent call last):
File "C:\Python27\Scripts\distutils-setup.py", line 5, in <module>
setup(service=["C:\Python27\Scripts\dev.py"])
File "C:\Python27\lib\distutils\core.py", line 152, in setup
dist.run_commands()
File "C:\Python27\lib\distutils\dist.py", line 953, in run_commands
self.run_command(cmd)
File "C:\Python27\lib\distutils\dist.py", line 972, in run_command
cmd_obj.run()
File "C:\Python27\lib\site-packages\py2exe\build_exe.py", line 243, in run
self._run()
File "C:\Python27\lib\site-packages\py2exe\build_exe.py", line 296, in _run
self.find_needed_modules(mf, required_files, required_modules)
File "C:\Python27\lib\site-packages\py2exe\build_exe.py", line 1274, in
find_needed_modules
mf.import_hook(mod)
File "C:\Python27\lib\site-packages\py2exe\mf.py", line 719, in import_hook
return Base.import_hook(self,name,caller,fromlist,level)
File "C:\Python27\lib\site-packages\py2exe\mf.py", line 136, in import_hook
q, tail = self.find_head_package(parent, name)
File "C:\Python27\lib\site-packages\py2exe\mf.py", line 204, in find_head_package
raise ImportError, "No module named " + qname
ImportError: No module named dev
Now, when I run py2exe with "console" in my setup script it works fine, but the service doesn't start and I get the error. When I run py2exe with "service" in my setup script py2exe doesn't run and tells me it can't find my module.
I have tried to re-install py2exe to no resolution. I have also tried to change:
def SvcDoRun(self):
servicemanager.LogMsg(servicemanager.EVENTLOG_INFORMATION_TYPE,
servicemanager.PYS_SERVICE_STARTED,
(self._svc_name_,''))
to
def SvcDoRun(self):
self.ReportServiceStatus(win32service.SERVICE_RUNNING)
win32event.WaitForSingleObject(self.hWaitStop, win32event.INFINITE)
Didn't make a difference either. CAN ANYONE HELP ME PLEASE? Here is what I am working on. It monitors a server and spits back a text file every 60 seconds which I use to monitor my servers at any given minute. Any help you guys and gals can give would be great.
import win32serviceutil
import win32service
import win32event
import servicemanager
import socket
import wmi
import _winreg
from time import sleep
import os
class SrvMonSvc (win32serviceutil.ServiceFramework):
_svc_name_ = "SrvMonSvc"
_svc_display_name_ = "Server Monitor"
def __init__(self,args):
win32serviceutil.ServiceFramework.__init__(self,args)
self.hWaitStop = win32event.CreateEvent(None,0,0,None)
socket.setdefaulttimeout(60)
def SvcStop(self):
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
win32event.SetEvent(self.hWaitStop)
def SvcDoRun(self):
servicemanager.LogMsg(servicemanager.EVENTLOG_INFORMATION_TYPE,
servicemanager.PYS_SERVICE_STARTED,
(self._svc_name_,''))
self.main()
def main(self):
host = wmi.WMI(namespace="root/default").StdRegProv
try:
result, api = host.GetStringValue(
hDefKey = _winreg.HKEY_LOCAL_MACHINE,
sSubKeyName = "SOFTWARE\Server Monitor",
sValueName = "API")
if api == None:
raise Exception
else:
pass
except:
exit()
while 1 == 1:
with open("C:/test.txt", "wb") as b:
computer = wmi.WMI(computer="exsan100")
for disk in computer.Win32_LogicalDisk (DriveType=3):
name = disk.caption
size = round(float(disk.Size)/1073741824, 2)
free = round(float(disk.FreeSpace)/1073741824, 2)
used = round(float(size), 2) - round(float(free), 2)
for mem in computer.Win32_OperatingSystem():
a_mem = (int(mem.FreePhysicalMemory)/1024)
for me in computer.Win32_ComputerSystem():
t_mem = (int(me.TotalPhysicalMemory)/1048576)
u_mem = t_mem - a_mem
for cpu in computer.Win32_Processor():
load = cpu.LoadPercentage
print >>b, api
print >>b, name
print >>b, size
print >>b, used
print >>b, t_mem
print >>b, u_mem
print >>b, load
b.close()
date_list = []
stamp = time.strftime("%c",time.localtime(time.time()))
date_list.append(stamp)
name = re.sub(r"[^\w\s]", "",date_list[0])
os.rename("C:/test.txt", ("C:/%s.txt" % name))
try:
sleep(60.00)
except:
exit()
if __name__ == '__main__':
win32serviceutil.HandleCommandLine(SrvMonSvc)
Have you progressed from your original problem. I had similar problem with a python service and found out that it was missing DLLs since the 'System Path' (not the user path) was not complete.
Running pythonservice.exe with -debug from the command prompt was not a problem because it used correct PATH environment variable, but if your service is installed as a System service it's worth checking out if the System Path variable has all the paths for the required DLLs (MSVC, Python, System32). For me it was missing the python DLLs path, after that it worked again.

Resources