I have an automation tool that starts a Ruby on Rails server using the command line:
from subprocess import Popen
devnull = open(os.devnull, 'r+')
self.webserver = Popen(server_cmd, shell=True, stdin=devnull, stdout=devnull,
stderr=devnull, close_fds=True, preexec_fn=os.setsid)
The webserver is a process that contains the server running as a separate process (the way it is supposed to be). So I have the stop_webserver method to kill the process and stop the Rails server:
def stop_webserver(self):
"""
Stop the Rails server, if there is one running or it was created.
Kill the process and all of its children, in order to avoid having a
zoombie processes.
"""
if self.webserver is None:
self.log.info("Server isn't running. Nothing to do.")
return
if self.is_server_running():
# os.killpg(self.webserver.pid, signal.SIGTERM)
self.log.info("PID: %s" % self.webserver.pid)
# self.log.info("PID: %s" % os.getpgid(self.webserver.pid))
time.sleep(10)
os.killpg(self.webserver.pid, signal.SIGKILL)
self.webserver = None
self.log.info("Server was stopped.")
However I'm always getting the following error:
Traceback (most recent call last):
File "unit/front_end_test.py", line 27, in runTest
frontend.stop_webserver()
File "front_end.py", line 184, in stop_webserver
os.killpg(self.webserver.pid, signal.SIGKILL)
OSError: [Errno 1] Operation not permitted
If I try to terminate the process using os.getpgid I get a different error:
Traceback (most recent call last):
File "unit/front_end_test.py", line 27, in runTest
frontend.stop_webserver()
File "front_end.py", line 182, in stop_webserver
self.log.info("PID: %s" % os.getpgid(self.webserver.pid))
OSError: [Errno 3] No such process
However, the server is still running at port 3001. The process was never killed. How can I properly terminate the process?
I solved the problem by forcing a os.system() kill:
def stop_webserver(self):
"""
Stop the Rails server, if there is one running or it was created.
Kill the process and all of its children, in order to avoid having a
zoombie processes.
"""
if self.webserver is None:
self.log.info("Server isn't running. Nothing to do.")
return
if self.is_server_running():
try:
# os.killpg(self.webserver.pid, signal.SIGKILL)
os.killpg(self.webserver.pid, signal.SIGTERM)
self.webserver = None
self.log.info("Server was stopped.")
except OSError, e:
self.log.exception(e)
self.log.info("PID: %s" % self.webserver.pid)
os.system("sudo kill %s" % (self.webserver.pid, ))
Related
Why does my paho-mqtt (1.5.1) on_connect work if connection OK but it is not called when an error. For testing I'm using Linux Lite 4.2 based on Ubuntu 18.04 LTS with Xfcs running in a VM (VBox).
class subscribemqtt:
.....
def on_connect(self, client, userdata, flags, rc):
print ("ZZZZZZZZZZZZZ in on_connect")
connectErrs = {.........}
self.connectRc = rc
self.connectReason = connectErrs[str(rc)]
print ("$$$$$$$$$$$$$", self.connectRc, self.connectReason)
return
def subscribe(self, arguments):
...........
self.client = paho.Client(self.config['CLIENT_ID'])
self.client.on_message = self.on_subscribe
self.client.on_connect = self.on_connect
print ("#############", self.on_connect)
print ("XXXXXXXXXXXX calling self.client.connect(...)
self.client.connect(self.config['HOST'],self.config['PORT'])
print ("YYYYYYYYYYYYY calling self.client.loop_start()")
self.client.loop_start()
print ("AAAAAAAAAAAAA", self.connected)
while not self.connected:
time.sleep(0.1)
print ("BBBBBBBBBBBBB", self.connected, self.connectRc)
When all the parameters are correct, on_connect gets called:
############# <bound method subscribemqtt.on_connect of <__main__.subscribemqtt object at 0x7f32065a6ac8>>
XXXXXXXXXXXX calling self.client.connect(self.config['HOST'],self.config['PORT']
YYYYYYYYYYYYY calling self.client.loop_start()
AAAAAAAAAAAAA**ZZZZZZZZZZZZZ in on_connect**
False$$$$$$$$$$$$$
0 Connection successful
BBBBBBBBBBBBB True 0
When I set the host address to an invalid address (to create an error to test my error handling) I get:
subscribemqtt.subscribe:topic= Immersion Dummy
############# <bound method subscribemqtt.on_connect of <__main__.subscribemqtt object at 0x7ffb942ae0b8>>
XXXXXXXXXXXX calling self.client.connect(self.config['HOST'],self.config['PORT']
Traceback (most recent call last):
File "/home/linuxlite/Desktop/EMS/sendroutines/subscribemqtt.py", line 275, in <module>
(retcode, reason, topic) = subscribeObj.subscribe([None, topic])
File "/home/linuxlite/Desktop/EMS/sendroutines/subscribemqtt.py", line 191, in subscribe
self.client.connect(self.config['HOST'],self.config['PORT'])
File "/usr/local/lib/python3.6/dist-packages/paho/mqtt/client.py", line 941, in connect
return self.reconnect()
File "/usr/local/lib/python3.6/dist-packages/paho/mqtt/client.py", line 1075, in reconnect
sock = self._create_socket_connection()
File "/usr/local/lib/python3.6/dist-packages/paho/mqtt/client.py", line 3546, in _create_socket_connection
return socket.create_connection(addr, source_address=source, timeout=self._keepalive)
File "/usr/lib/python3.6/socket.py", line 704, in create_connection
for res in getaddrinfo(host, port, 0, SOCK_STREAM):
File "/usr/lib/python3.6/socket.py", line 745, in getaddrinfo
for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
socket.gaierror: [Errno -2] Name or service not known
>>>
Thanks for reading.
Alan
PS. I just tried:
try:
self.client.connect(self.config['HOST'],self.config['PORT'])
except:
print ("**************** Error exception calling self.client.connect")
And that works but my understanding is that on_connect should be called for errors.
From the docs:
on_connect()
on_connect(client, userdata, flags, rc)
Called when the broker responds to our connection request.
The important part there is "when the broker responds". But in the example you have shown the hostname provided can not be resolved, so broker never responds because it is never actually contacted.
on_connect() will be called if connection succeeds or if it fails because the username/password is wrong or a unavailable protocol version (e.g. requesting MQTTv5 from a broker that only supports v3)
I am new to multiprocessing in python.I am extracting some features from a list of 70,000 URLs. I have them from 2 different files. After the feature extraction process I pass the result to a list and then to a CSV file.
The code runs but then stops with the error.I tried to catch the error but it produced another one.
Python version = 3.5
from feature_extractor import Feature_extraction
import pandas as pd
from pandas.core.frame import DataFrame
import sys
from multiprocessing.dummy import Pool as ThreadPool
import threading as thread
from multiprocessing import Process,Manager,Array
import time
class main():
lst = None
def __init__(self):
manager = Manager()
self.lst = manager.list()
self.dostuff()
self.read_lst()
def feature_extraction(self,url):
if self.lst is None:
self.lst = []
features = Feature_extraction(url)
self.lst.append(features.get_features())
print(len(self.lst))
def Pool(self,url):
pool = ThreadPool(8)
results = pool.map(self.feature_extraction, url)
def dostuff(self):
df = pd.read_csv('verified_online.csv',encoding='latin-1')
df['label'] = df['phish_id'] * 0
mal_urls = df['url']
df2 = pd.read_csv('new.csv')
df2['label'] = df['phish_id']/df['phish_id']
ben_urls = df2['urls']
t = Process(target=self.Pool,args=(mal_urls,))
t2 = Process(target=self.Pool,args=(ben_urls,))
t.start()
t2.start()
t.join()
t2.join
def read_lst(self):
nw_df = DataFrame(list(self.lst))
nw_df.columns = ['Redirect count','ssl_classification','url_length','hostname_length','subdomain_count','at_sign_in_url','exe_extension_in_request_url','exe_extension_in_landing_url',
'ip_as_domain_name','no_of_slashes_in requst_url','no_of_slashes_in_landing_url','no_of_dots_in_request_url','no_of_dots_in_landing_url','tld_value','age_of_domain',
'age_of_last_modified','content_length','same_landing_and_request_ip','same_landing_and_request_url']
frames = [df['label'],df2['label']]
new_df = pd.concat(frames)
new_df = new_df.reset_index()
nw_df['label'] = new_df['label']
nw_df.to_csv('dataset.csv', sep=',', encoding='latin-1')
if __name__ == '__main__':
start_time = time.clock()
try:
main()
except BrokenPipeError:
print("broken pipe....")
pass
print (time.clock() - start_time, "seconds")
Error Traceback
Process Process-3:
Traceback (most recent call last):
File "F:\Continuum\Anaconda3\lib\multiprocessing\connection.py", line 312, in _recv_bytes
nread, err = ov.GetOverlappedResult(True)
BrokenPipeError: [WinError 109] The pipe has been ended
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "F:\Continuum\Anaconda3\lib\multiprocessing\process.py", line 249, in _bootstrap
self.run()
File "F:\Continuum\Anaconda3\lib\multiprocessing\process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "H:\Projects\newoproject\src\main.py", line 33, in Pool
results = pool.map(self.feature_extraction, url)
File "F:\Continuum\Anaconda3\lib\multiprocessing\pool.py", line 260, in map
return self._map_async(func, iterable, mapstar, chunksize).get()
File "F:\Continuum\Anaconda3\lib\multiprocessing\pool.py", line 608, in get
raise self._value
File "F:\Continuum\Anaconda3\lib\multiprocessing\pool.py", line 119, in worker
result = (True, func(*args, **kwds))
File "F:\Continuum\Anaconda3\lib\multiprocessing\pool.py", line 44, in mapstar
return list(map(*args))
File "H:\Projects\newoproject\src\main.py", line 26, in feature_extraction
self.lst.append(features.get_features())
File "<string>", line 2, in append
File "F:\Continuum\Anaconda3\lib\multiprocessing\managers.py", line 717, in _callmethod
kind, result = conn.recv()
File "F:\Continuum\Anaconda3\lib\multiprocessing\connection.py", line 250, in recv
buf = self._recv_bytes()
File "F:\Continuum\Anaconda3\lib\multiprocessing\connection.py", line 321, in _recv_bytes
raise EOFError
EOFError
My response is late and does not address the posted problem directly; but hopefully will provide a clue to others who encounter similar errors.
Errors that I encountered:
BrokenPipeError
WinError 109 The pipe has been ended &
WinError 232 The pipe is being closed
Observed with Python 36 on Windows 7, when:
(1) the same async function was submitted multiple times, each time with a different instance of a multiprocessing data store, a Queue in my case (multiprocessing.Manager().Queue())
AND
(2) the references to the Queues were saved in short-life local variables in the enveloping function.
The errors were occurring despite the fact that the Queues, shared with the successfully spawned and executing async-functions, had items and would still be in active use (put() & get()) at the time of exception.
The error consistently occurred when the same async_func was called the 2nd time with a 2nd instance of the Queue. Immediately after apply_async() of the function, the connection to the 1st Queue supplied to the async_func the 1st time, would get broken.
The issue got resolved when the references to the Queues were saved in non-overlapping (like a Queue-list) & longer-life variables (like variables returned to functions higher in the call-stack) in the enveloping function.
I'm trying to run a Python script that connects to a MySQL database through PyMySQL. The script is effectively:
import pymysql
cnx = pymysql.connect(read_default_file = "/directory/my.cnf", cursorclass = pymysql.cursors.DictCursor)
cursor = cnx.cursor()
# Do stuff.
When I run the script in the interpreter, I don't get any errors, but when I try to run it from the command line, I get the following error:
Traceback (most recent call last):
File "s02_prepare_data_RNN.py", line 264, in <module>
(omniture, urls, years, global_regions) = get_omniture_data("omniture_results")
File "s02_prepare_data_RNN.py", line 76, in get_omniture_data
sso_to_accountid = get_sso_accountids()
File "s02_prepare_data_RNN.py", line 31, in get_sso_accountids
cursor.execute(query)
File "/home/rdu/malcorn/.local/lib/python2.6/site-packages/pymysql/cursors.py", line 134, in execute
result = self._query(query)
File "/home/rdu/malcorn/.local/lib/python2.6/site-packages/pymysql/cursors.py", line 282, in _query
conn.query(q)
File "/home/rdu/malcorn/.local/lib/python2.6/site-packages/pymysql/connections.py", line 768, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "/home/rdu/malcorn/.local/lib/python2.6/site-packages/pymysql/connections.py", line 929, in _read_query_result
result.read()
File "/home/rdu/malcorn/.local/lib/python2.6/site-packages/pymysql/connections.py", line 1125, in read
first_packet = self.connection._read_packet()
File "/home/rdu/malcorn/.local/lib/python2.6/site-packages/pymysql/connections.py", line 893, in _read_packet
packet.check_error()
File "/home/rdu/malcorn/.local/lib/python2.6/site-packages/pymysql/connections.py", line 369, in check_error
err.raise_mysql_exception(self._data)
File "/home/rdu/malcorn/.local/lib/python2.6/site-packages/pymysql/err.py", line 120, in raise_mysql_exception
_check_mysql_exception(errinfo)
File "/home/rdu/malcorn/.local/lib/python2.6/site-packages/pymysql/err.py", line 115, in _check_mysql_exception
raise InternalError(errno, errorvalue)
pymysql.err.InternalError: (1046, u'No database selected')
Modifying my code from:
import pymysql
cnx = pymysql.connect(read_default_file = "/directory/my.cnf", cursorclass = pymysql.cursors.DictCursor)
cursor = cnx.cursor()
# Do stuff.
to:
import pymysql
cnx = pymysql.connect(read_default_file = "/directory/my.cnf", cursorclass = pymysql.cursors.DictCursor)
cursor = cnx.cursor()
if __name__ == "__main__":
# Do stuff.
fixed the error. I had the thought that Python might be trying to execute the queries before the connection was established, so I tried putting the main part of my program under if __name__ == "__main__": and that fixed it. I'm still not 100% what is going on, though. I had assumed the code would wait for the connection to be established before proceeding to the following lines, but this fix suggests that's not the case.
It's also worth noting that I was only getting the error when running the original script from the command line on a server that has Python 2.6. When I ran the original script from the command line on my local machine that has Python 2.7, I did not get the error.
Anyway, if __name__ == "__main__": is good Python style, so I'll make sure to use it in the future.
Sometimes this may simply happen when the database URI is not built correctly. For me, the error occurred when my pymysql connection string (URL) looked like this:
mysql+pymysql://:#localhost/
To fix it, it needed to look like:
mysql+pymysql://<my-database-user-name>:#localhost/<my-database-name> # or replace localhost with your particular host name
If you don't select the database in connection, you must add this line.
connection = pymysql.connect(host=DB_host, user=DB_user, password=DB_password) # NO DB_NAME SPECIFED
cursor = connection.cursor()
cursor.execute(f'CREATE DATABASE IF NOT EXISTS {DB_NAME}') # ADD THIS LINE!
connection.select_db(DB_NAME)
I'm having some problems accessing the Windows 7 Registry with the _winreg.QueryValueEx function in the Python 2.7.3 _winreg module.
I am running the python process as Administrator, and can create new keys and values like this:
import _winreg as wreg
key = wreg.CreateKey(wreg.HKEY_LOCAL_MACHINE, "Software\\TestCompany\\TestProject")
# Create new subkey
wreg.SetValue(key, 'NewSubkey', wreg.REG_SZ, 'testsubkey')
print wreg.QueryValue(key, 'NewSubKey')
# prints 'testsubkey'
# Create new value
wreg.SetValueEx(key, 'ValueName', 0, wreg.REG_SZ, 'testvalue')
print wreg.QueryValueEx(key,'ValueName')
# prints (u'testvalue', 1)
key.Close()
However, when I re-open the same key and try to set the value, it gives me an Access is denied error:
key = wreg.OpenKey(wreg.HKEY_LOCAL_MACHINE, "Software\\TestCompany\\TestProject",wreg.KEY_SET_VALUE)
wreg.SetValue(key, 'NewSubkey', wreg.REG_SZ, 'subkey_changed')
print wreg.QueryValue(key, 'NewSubkey')
# prints 'subkey_changed'
wreg.SetValueEx(key, 'ValueName', 0, wreg.REG_SZ, 'value_changed')
Traceback (most recent call last):
File "<pyshell#43>", line 1, in <module>
wreg.SetValueEx(key, 'ValueName', 0, wreg.REG_SZ, 'value_changed')
WindowsError: [Error 5] Access is denied
print wreg.QueryValueEx(key, 'ValueName')
# still prints: (u'testvalue', 1)
key.Close()
Interestingly, running as Administrator, I cannot open with KEY_WRITE or KEY_ALL_ACCESS access rights:
>>> key = wreg.OpenKey(wreg.HKEY_LOCAL_MACHINE, "Software\\TestCompany\\TestProject",wreg.KEY_WRITE)
Traceback (most recent call last):
File "<pyshell#47>", line 1, in <module>
key = wreg.OpenKey(wreg.HKEY_LOCAL_MACHINE, "Software\\TestCompany\\TestProject",wreg.KEY_WRITE)
WindowsError: [Error 5] Access is denied
>>> key = wreg.OpenKey(wreg.HKEY_LOCAL_MACHINE, "Software\\TestCompany\\TestProject",wreg.KEY_ALL_ACCESS)
Traceback (most recent call last):
File "<pyshell#48>", line 1, in <module>
key = wreg.OpenKey(wreg.HKEY_LOCAL_MACHINE, "Software\\TestCompany\\TestProject",wreg.KEY_ALL_ACCESS)
WindowsError: [Error 5] Access is denied
I solved the problem by doing:
key = wreg.OpenKey(wreg.HKEY_LOCAL_MACHINE, "Software\\TestCompany\\TestProject",0, wreg.KEY_ALL_ACCESS)
I have already looked at and tried the resolutions to this question that others have posted. One user said that to try and change my setup.py file from:
from distutils.core import setup
import py2exe
setup(console=["dev.py"])
to
from distutils.core import setup
import py2exe
setup(service=["dev.py"])
I got the following results:
running py2exe
*** searching for required modules ***
Traceback (most recent call last):
File "C:\Python27\Scripts\distutils-setup.py", line 5, in <module>
setup(service=["C:\Python27\Scripts\dev.py"])
File "C:\Python27\lib\distutils\core.py", line 152, in setup
dist.run_commands()
File "C:\Python27\lib\distutils\dist.py", line 953, in run_commands
self.run_command(cmd)
File "C:\Python27\lib\distutils\dist.py", line 972, in run_command
cmd_obj.run()
File "C:\Python27\lib\site-packages\py2exe\build_exe.py", line 243, in run
self._run()
File "C:\Python27\lib\site-packages\py2exe\build_exe.py", line 296, in _run
self.find_needed_modules(mf, required_files, required_modules)
File "C:\Python27\lib\site-packages\py2exe\build_exe.py", line 1274, in
find_needed_modules
mf.import_hook(mod)
File "C:\Python27\lib\site-packages\py2exe\mf.py", line 719, in import_hook
return Base.import_hook(self,name,caller,fromlist,level)
File "C:\Python27\lib\site-packages\py2exe\mf.py", line 136, in import_hook
q, tail = self.find_head_package(parent, name)
File "C:\Python27\lib\site-packages\py2exe\mf.py", line 204, in find_head_package
raise ImportError, "No module named " + qname
ImportError: No module named dev
Now, when I run py2exe with "console" in my setup script it works fine, but the service doesn't start and I get the error. When I run py2exe with "service" in my setup script py2exe doesn't run and tells me it can't find my module.
I have tried to re-install py2exe to no resolution. I have also tried to change:
def SvcDoRun(self):
servicemanager.LogMsg(servicemanager.EVENTLOG_INFORMATION_TYPE,
servicemanager.PYS_SERVICE_STARTED,
(self._svc_name_,''))
to
def SvcDoRun(self):
self.ReportServiceStatus(win32service.SERVICE_RUNNING)
win32event.WaitForSingleObject(self.hWaitStop, win32event.INFINITE)
Didn't make a difference either. CAN ANYONE HELP ME PLEASE? Here is what I am working on. It monitors a server and spits back a text file every 60 seconds which I use to monitor my servers at any given minute. Any help you guys and gals can give would be great.
import win32serviceutil
import win32service
import win32event
import servicemanager
import socket
import wmi
import _winreg
from time import sleep
import os
class SrvMonSvc (win32serviceutil.ServiceFramework):
_svc_name_ = "SrvMonSvc"
_svc_display_name_ = "Server Monitor"
def __init__(self,args):
win32serviceutil.ServiceFramework.__init__(self,args)
self.hWaitStop = win32event.CreateEvent(None,0,0,None)
socket.setdefaulttimeout(60)
def SvcStop(self):
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
win32event.SetEvent(self.hWaitStop)
def SvcDoRun(self):
servicemanager.LogMsg(servicemanager.EVENTLOG_INFORMATION_TYPE,
servicemanager.PYS_SERVICE_STARTED,
(self._svc_name_,''))
self.main()
def main(self):
host = wmi.WMI(namespace="root/default").StdRegProv
try:
result, api = host.GetStringValue(
hDefKey = _winreg.HKEY_LOCAL_MACHINE,
sSubKeyName = "SOFTWARE\Server Monitor",
sValueName = "API")
if api == None:
raise Exception
else:
pass
except:
exit()
while 1 == 1:
with open("C:/test.txt", "wb") as b:
computer = wmi.WMI(computer="exsan100")
for disk in computer.Win32_LogicalDisk (DriveType=3):
name = disk.caption
size = round(float(disk.Size)/1073741824, 2)
free = round(float(disk.FreeSpace)/1073741824, 2)
used = round(float(size), 2) - round(float(free), 2)
for mem in computer.Win32_OperatingSystem():
a_mem = (int(mem.FreePhysicalMemory)/1024)
for me in computer.Win32_ComputerSystem():
t_mem = (int(me.TotalPhysicalMemory)/1048576)
u_mem = t_mem - a_mem
for cpu in computer.Win32_Processor():
load = cpu.LoadPercentage
print >>b, api
print >>b, name
print >>b, size
print >>b, used
print >>b, t_mem
print >>b, u_mem
print >>b, load
b.close()
date_list = []
stamp = time.strftime("%c",time.localtime(time.time()))
date_list.append(stamp)
name = re.sub(r"[^\w\s]", "",date_list[0])
os.rename("C:/test.txt", ("C:/%s.txt" % name))
try:
sleep(60.00)
except:
exit()
if __name__ == '__main__':
win32serviceutil.HandleCommandLine(SrvMonSvc)
Have you progressed from your original problem. I had similar problem with a python service and found out that it was missing DLLs since the 'System Path' (not the user path) was not complete.
Running pythonservice.exe with -debug from the command prompt was not a problem because it used correct PATH environment variable, but if your service is installed as a System service it's worth checking out if the System Path variable has all the paths for the required DLLs (MSVC, Python, System32). For me it was missing the python DLLs path, after that it worked again.