I have a GUI application which launches some commands using subprocess and then shows the progress of this commands by reading from subprocess.Popen.stdout and using wx.ProgressDialog. I've written the app under Linux and it works flawlessly there, but I'm now doing some testing under windows and it seems that trying to update the progress dialog causes the app to hang. There are no error messages or anything, so it's difficult for me to figure out what's happening. Below is a simplified code:
The subprocess is launched in separate thread by this method in main thread:
def onOk(self,event):
""" Starts processing """
self.infotxt.Clear()
args = self.getArgs()
self.stringholder = args['outfile']
if (args):
cmd = self.buildCmd(args, True)
if (cmd):
# Make sure the output directory is writable.
if not self.isWritable(args['outfile']):
print "Cannot write to %s. Make sure you have write permission or select a different output directory." %os.path.dirname(args['outfile'])
else:
try:
self.thread = threading.Thread(target=self.runCmd,args=(cmd,))
self.thread.setDaemon(True)
self.thread.start()
except Exception:
sys.stderr.write('Error starting thread')
And here's the runCmd method:
def runCmd(self, cmd):
""" Runs a command line provided as a list of arguments """
temp = []
aborted = False
dlg = None
for i in cmd:
temp.extend(i.split(' '))
# Use wx.MutexGuiEnter()/MutexGuiLeave() for anything that accesses GUI from another thread
wx.MutexGuiEnter()
max = 100
stl = wx.PD_CAN_ABORT | wx.PD_APP_MODAL | wx.PD_ELAPSED_TIME | wx.PD_REMAINING_TIME
dlg = wx.ProgressDialog("Please wait", "Processing...", maximum = max, parent = self.frame, style=stl)
wx.MutexGuiLeave()
# This is for windows to not display the black command line window when executing the command
if os.name == 'nt':
si = subprocess.STARTUPINFO()
si.dwFlags |= subprocess.STARTF_USESHOWWINDOW
si.wShowWindow = subprocess.SW_HIDE
else:
si = None
try:
proc = subprocess.Popen(temp, shell=False, bufsize=1, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
except Exception:
sys.stderr.write('Error executing a command. ')
# Progress dialog
count = 0
while True:
line=proc.stdout.readline()
count += 1
wx.MutexGuiEnter()
if dlg.Update(count) == (True, False):
print line.rstrip()
wx.MutexGuiLeave()
if not line: break
else:
print "Processing cancelled."
aborted = True
wx.MutexGuiLeave()
proc.kill()
break
wx.MutexGuiEnter()
dlg.Destroy()
wx.GetApp().GetTopWindow().Raise()
wx.MutexGuiLeave()
if aborted:
if os.path.exists(self.stringholder):
os.remove(self.stringholder)
dlg.Destroy()
proc.wait()
Again this works fine under Linux, but freezes on Windows. If I remove dlg.Update() line it also works fine. The subprocess output is printed out in main window and ProgressDialog is shown, just progressbar doesn't move. What am I missing?
Try not using wx.MutexGuiEnter and wx.MutexGuiLeave. You can handle updating the GUI from another thread using wx.CallAfter. I have never seen anyone using those mutexes in wx application before and not even in tutorials or examples of using threads.
Related
I am trying to show an animated "working" prompt while running some python code. I've been searching for a way to do so but the solutions i've found are not quite what i want (if i recall correctly tqdm and alive-progress require a for loop with a defined number of iterations) and I'd like to find a way to code it myself.
The closest I've gotten to making it is using asyncio as follows:
async def main():
dummy_task = asyncio.create_task(dummy_search())
bar_task = asyncio.create_task(progress())
test = await dummy_task
bar_task.cancel()
where dummy task can be any async task and bar_task is:
FLUSH_LINE = "\033[K"
async def progress(mode=""):
def integers():
n = 0
while True:
yield n
n += 1
progress_indicator = ["-", "\\", "|", "/"]
message = "working"
message_len = len(message)
message += "-" * message_len
try:
if not mode:
for i in (n for n in integers()):
await asyncio.sleep(0.05)
message = message[1:] + message[0]
print(f"{FLUSH_LINE}{progress_indicator[i % 4]} [{message[:message_len]}]", end="\r")
finally:
print(f"{FLUSH_LINE}")
The only problem with this approach is that asyncio does not actually run tasks in parallel, so if the dummy_task does not use await at any point, the bar_task will not run until the dummy task is complete, and it won't show the working prompt in the terminal.
How should I go about trying to run both tasks in parallel? Do I need to use multiprocessing? If so, would both tasks write to the same terminal by default?
Thanks in advance.
I'm running speedtest.exe from a Windows 10 command line (cmd.) When running, the test displays partial data. I.e. A changing display of how fast the download is at any given second.
Once complete, it only shows the totals.
But I'm trying to track a latency issue, where the download starts very slow and then speeds up until it reaches it's maximum speed level...so I want to capture all of those changes, even if they're messy.
I would use the script command in Linux, but the Linux port of speedtest doesn't show the changing data rate. Just dots.
I've tried a number of different methods including ...
import subprocess
import sys
from time import sleep
cmd = command = r"C:\Program Files (x86)\ookla-speedtest-1.1.1-win64\speedtest"
#===============================================================================
# with open("test.log", "wb") as f:
# process = subprocess.Popen(command, stdout=subprocess.PIPE)
# for c in iter(lambda: process.stdout.read(1), b""):
# sys.stdout.buffer.write(c)
# f.buffer.write(c)
#===============================================================================
#===============================================================================
# process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
# while True:
# # while process.stdout.readable():
# line = process.stdout.readline()
#
# if not line:
# break
#
# print(line.strip())
#===============================================================================
process = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
while True:
out = process.stdout.read(1)
out = str(out).strip("b").strip("'")
if out == '' and process.poll() != None:
break
if out != '':
sys.stdout.write(out)
sys.stdout.flush()
sleep(0.1)
None of them capture the transient data, just the final output (even with buffering disabled.) Is there a way to do this with either the Python subprocess, or even just by adding output capture to the command line itself?
I needed to write a solution to write data on and then print RFID labels en-masse, each generated as .png images from a template python script and data taken from a database or excel file.
To print the program simply calls the relative system utility (CUPS on unix systems) using subprocess.check_call(print_cmd) passing the image file (saved on a ram-mounted file system for minimal disk usage)
Now, it also needs to run on Windows systems, but there is not really a decent system utility for that, and solutions under a similar question command line tool for print picture? don't account for print-job completion or if the job results in an error, the margins are all screwed and the image is always rotated 90 degrees for some reason.
How can I sanely print an image using a command or a script in Windows and wait for it to complete successfully or return an error if the job results in an error?
Possibly with no dependencies
If you can install dependencies, there are many programs that offer a solution out-of-the-box.
The only sane way i could find to solve this issue with no dependencies is by creating a powershell script to account for this
[CmdletBinding()]
param (
[string] $file = $(throw "parameter is mandatory"),
[string] $printer = "EXACT PRINTER NAME HERE"
)
$ERR = "UserIntervention|Error|Jammed"
$status = (Get-Printer -Name $printer).PrinterStatus.ToString()
if ($status -match $ERR){ exit 1 }
# https://stackoverflow.com/a/20402656/17350905
# only sends the print job to the printer
rundll32 C:\Windows\System32\shimgvw.dll,ImageView_PrintTo $file $printer
# wait until printer is in printing status
do {
$status = (Get-Printer -Name $printer).PrinterStatus.ToString()
if ($status -match $ERR){ exit 1 }
Start-Sleep -Milliseconds 100
} until ( $status -eq "Printing" )
# wait until printing is done
do {
$status = (Get-Printer -Name $printer).PrinterStatus.ToString()
if ($status -match $ERR){ exit 1 }
Start-Sleep -Milliseconds 100
} until ( $status -eq "Normal" )
I would then need to slightly modify the print subprocess call to
powershell -File "path\to\print.ps1" "C:\absolute\path\to\file.png"
Then there are a couple of necessary setup steps:
(discaimer, I don't use windows in english so i don't know how the english thigs are supposed to be called. i will use cursive for those)
create an example image, right click and then select Print
from the print dialog that opens then set up all the default options you want, like orientation, margins, paper type, etc etc for the specific printer you're gonna use.
Go to printer settings, under tools then edit Printer Status Monitoring
edit monitoring frequency to "only during print jobs". it should be disabled by default
in the next tab, modify polling frequency to the minimum available, 100ms during print jobs (you can use a lower one for the while not printing option
Assuming the following:
only your program is running this script
theres always only 1 printing job at a time for a given printer
the printer drivers were not written by a monkey and they actually report the current, correct printer status
This little hack will allow to print an image from a command and await job completion, with error management; and uses only windows preinstalled software
Further optimization could be done by keeping powershell subprocess active and only passing it scripts in the & "path\to\print.ps1" "C:\absolute\path\to\file.png" format, waiting for standard output to report an OK or a KO; but only if mass printing is required.
Having had to work on this again, just wanted to add a simpler solution in "pure" python using the pywin32 package
import time
import subprocess
from typing import List
try:
import win32print as wprint
PRINTERS: List[str] = [p[2] for p in wprint.EnumPrinters(wprint.PRINTER_ENUM_LOCAL)]
PRINTER_DEFAULT = wprint.GetDefaultPrinter()
WIN32_SUPPORTED = True
except:
print("[!!] an error occured while retrieving printers")
# you could throw an exception or whatever
# bla bla do other stuff
if "WIN32_SUPPORTED" in globals():
__printImg_win32(file, printer_name)
def __printImg_win32(file: str, printer: str = ""):
if not printer:
printer = PRINTER_DEFAULT
# verify prerequisites here
# i still do prefer to print calling rundll32 directly,
# because of the default printer settings shenaningans
# and also because i've reliably used it to spool millions of jobs
subprocess.check_call(
[
"C:\\Windows\\System32\\rundll32",
"C:\\Windows\\System32\\shimgvw.dll,ImageView_PrintTo",
file,
printer,
]
)
__monitorJob_win32(printer)
pass
def __monitorJob_win32(printer: str, timeout=16.0):
p = wprint.OpenPrinter(printer)
# wait for job to be sheduled
t0 = time.time()
while (time.time()-t0) < timeout:
ptrr = wprint.GetPrinter(p, 2)
# unsure about those flags, but definitively not errors.
# it seems they are "moving paper forward"
if ptrr["Status"] != 0 and ptrr["Status"] not in [1024,1048576]:
raise Error("Printer is in error (status %d)!" % ptrr["Status"])
if ptrr["cJobs"] > 0:
break
time.sleep(0.1)
else:
raise Error("Printer timeout sheduling job!")
# await job completion
t0 = time.time()
while (time.time()-t0) < timeout:
ptrr = wprint.GetPrinter(p, 2)
if ptrr["Status"] != 0 and ptrr["Status"] not in [1024,1048576]:
raise Error("Printer is in error (status %d)!" % ptrr["Status"])
if ptrr["cJobs"] == 0 and ptrr["Status"] == 0:
break
time.sleep(0.1)
else:
raise Error("Printer timeout waiting for completion!")
wprint.ClosePrinter(p)
return
useful additional resources
Print image files using python
Catch events from printer in Windows
pywin32's win32print "documentation"
I am using tesseract to perform OCR on screengrabs. I have an app using a tkinter window leveraging self.after in the initialization of my class to perform constant image scrapes and update label, etc values in the tkinter window. I have searched for multiple days and can't find any specific examples how to leverage CREATE_NO_WINDOW with Python3.6 on a Windows platform calling tesseract with pytesseract.
This is related to this question:
How can I hide the console window when I run tesseract with pytesser
I have only been programming Python for 2 weeks and don't understand what/how to perform the steps in the above question. I opened up the pytesseract.py file and reviewed and found the proc = subprocess.Popen(command, stderr=subproces.PIPE) line but when I tried editing it I got a bunch of errors that I couldn't figure out.
#!/usr/bin/env python
'''
Python-tesseract. For more information: https://github.com/madmaze/pytesseract
'''
try:
import Image
except ImportError:
from PIL import Image
import os
import sys
import subprocess
import tempfile
import shlex
# CHANGE THIS IF TESSERACT IS NOT IN YOUR PATH, OR IS NAMED DIFFERENTLY
tesseract_cmd = 'tesseract'
__all__ = ['image_to_string']
def run_tesseract(input_filename, output_filename_base, lang=None, boxes=False,
config=None):
'''
runs the command:
`tesseract_cmd` `input_filename` `output_filename_base`
returns the exit status of tesseract, as well as tesseract's stderr output
'''
command = [tesseract_cmd, input_filename, output_filename_base]
if lang is not None:
command += ['-l', lang]
if boxes:
command += ['batch.nochop', 'makebox']
if config:
command += shlex.split(config)
proc = subprocess.Popen(command, stderr=subprocess.PIPE)
status = proc.wait()
error_string = proc.stderr.read()
proc.stderr.close()
return status, error_string
def cleanup(filename):
''' tries to remove the given filename. Ignores non-existent files '''
try:
os.remove(filename)
except OSError:
pass
def get_errors(error_string):
'''
returns all lines in the error_string that start with the string "error"
'''
error_string = error_string.decode('utf-8')
lines = error_string.splitlines()
error_lines = tuple(line for line in lines if line.find(u'Error') >= 0)
if len(error_lines) > 0:
return u'\n'.join(error_lines)
else:
return error_string.strip()
def tempnam():
''' returns a temporary file-name '''
tmpfile = tempfile.NamedTemporaryFile(prefix="tess_")
return tmpfile.name
class TesseractError(Exception):
def __init__(self, status, message):
self.status = status
self.message = message
self.args = (status, message)
def image_to_string(image, lang=None, boxes=False, config=None):
'''
Runs tesseract on the specified image. First, the image is written to disk,
and then the tesseract command is run on the image. Tesseract's result is
read, and the temporary files are erased.
Also supports boxes and config:
if boxes=True
"batch.nochop makebox" gets added to the tesseract call
if config is set, the config gets appended to the command.
ex: config="-psm 6"
'''
if len(image.split()) == 4:
# In case we have 4 channels, lets discard the Alpha.
# Kind of a hack, should fix in the future some time.
r, g, b, a = image.split()
image = Image.merge("RGB", (r, g, b))
input_file_name = '%s.bmp' % tempnam()
output_file_name_base = tempnam()
if not boxes:
output_file_name = '%s.txt' % output_file_name_base
else:
output_file_name = '%s.box' % output_file_name_base
try:
image.save(input_file_name)
status, error_string = run_tesseract(input_file_name,
output_file_name_base,
lang=lang,
boxes=boxes,
config=config)
if status:
errors = get_errors(error_string)
raise TesseractError(status, errors)
f = open(output_file_name, 'rb')
try:
return f.read().decode('utf-8').strip()
finally:
f.close()
finally:
cleanup(input_file_name)
cleanup(output_file_name)
def main():
if len(sys.argv) == 2:
filename = sys.argv[1]
try:
image = Image.open(filename)
if len(image.split()) == 4:
# In case we have 4 channels, lets discard the Alpha.
# Kind of a hack, should fix in the future some time.
r, g, b, a = image.split()
image = Image.merge("RGB", (r, g, b))
except IOError:
sys.stderr.write('ERROR: Could not open file "%s"\n' % filename)
exit(1)
print(image_to_string(image))
elif len(sys.argv) == 4 and sys.argv[1] == '-l':
lang = sys.argv[2]
filename = sys.argv[3]
try:
image = Image.open(filename)
except IOError:
sys.stderr.write('ERROR: Could not open file "%s"\n' % filename)
exit(1)
print(image_to_string(image, lang=lang))
else:
sys.stderr.write('Usage: python pytesseract.py [-l lang] input_file\n')
exit(2)
if __name__ == '__main__':
main()
The code I am leveraging is similar to the example in the similar question:
def get_string(img_path):
# Read image with opencv
img = cv2.imread(img_path)
# Convert to gray
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# Apply dilation and erosion to remove some noise
kernel = np.ones((1, 1), np.uint8)
img = cv2.dilate(img, kernel, iterations=1)
img = cv2.erode(img, kernel, iterations=1)
# Write image after removed noise
cv2.imwrite(src_path + "removed_noise.png", img)
# Apply threshold to get image with only black and white
# Write the image after apply opencv to do some ...
cv2.imwrite(src_path + "thres.png", img)
# Recognize text with tesseract for python
result = pytesseract.image_to_string(Image.open(src_path + "thres.png"))
return result
When it gets to the following line, there is a flash of a black console window for less than a second and then it closes when it runs the command.
result = pytesseract.image_to_string(Image.open(src_path + "thres.png"))
Here is the picture of the console window:
Program Files (x86)_Tesseract
Here is what is suggested from the other question:
You're currently working in IDLE, in which case I don't think it
really matters if a console window pops up. If you're planning to
develop a GUI app with this library, then you'll need to modify the
subprocess.Popen call in pytesser.py to hide the console. I'd first
try the CREATE_NO_WINDOW process creation flag. – eryksun
I would greatly appreciate any help for how to modify the subprocess.Popen call in the pytesseract.py library file using CREATE_NO_WINDOW. I am also not sure of the difference between pytesseract.py and pytesser.py library files. I would leave a comment on the other question to ask for clarification but I can't until I have more reputation on this site.
I did more research and decided to learn more about subprocess.Popen:
Documentation for subprocess
I also referenced the following articles:
using python subprocess.popen..can't prevent exe stopped working prompt
I changed the original line of code in pytesseract.py:
proc = subprocess.Popen(command, stderr=subprocess.PIPE)
to the following:
proc = subprocess.Popen(command, stderr=subprocess.PIPE, creationflags = CREATE_NO_WINDOW)
I ran the code and got the following error:
Exception in Tkinter callback Traceback (most recent call last):
File
"C:\Users\Steve\AppData\Local\Programs\Python\Python36-32\lib\tkinter__init__.py",
line 1699, in call
return self.func(*args) File "C:\Users\Steve\Documents\Stocks\QuickOrder\QuickOrderGUI.py", line
403, in gather_data
update_cash_button() File "C:\Users\Steve\Documents\Stocks\QuickOrder\QuickOrderGUI.py", line
208, in update_cash_button
currentCash = get_string(src_path + "cash.png") File "C:\Users\Steve\Documents\Stocks\QuickOrder\QuickOrderGUI.py", line
150, in get_string
result = pytesseract.image_to_string(Image.open(src_path + "thres.png")) File
"C:\Users\Steve\AppData\Local\Programs\Python\Python36-32\lib\site-packages\pytesseract\pytesseract.py",
line 125, in image_to_string
config=config) File "C:\Users\Steve\AppData\Local\Programs\Python\Python36-32\lib\site-packages\pytesseract\pytesseract.py",
line 49, in run_tesseract
proc = subprocess.Popen(command, stderr=subprocess.PIPE, creationflags = CREATE_NO_WINDOW) NameError: name 'CREATE_NO_WINDOW'
is not defined
I then defined the CREATE_NO_WINDOW variable:
#Assignment of the value of CREATE_NO_WINDOW
CREATE_NO_WINDOW = 0x08000000
I got the value of 0x08000000 from the above linked article. After adding the definition I ran the application and I didn't get any more console window popups.
I'm following Gray Hat Python book and have copied their code for the process snapshots using PyDBG. When I run the script I get no errors and expected output however my program is not actually reverting to the snapshot. When I go in debug it seems like values are in the snapshot variables as if it is storing snapshot info but I don't really know enough to say for sure.
Here is code:
from pydbg import *
from pydbg.defines import *
import threading
import time
import sys
class snapshotter(object):
def __init__(self,exe_path):
self.exe_path = exe_path
self.pid = None
self.dbg = None
self.running = True
pydbg_thread = threading.Thread(target=self.start_debugger)
pydbg_thread.setDaemon(0)
pydbg_thread.start()
while self.pid == None:
time.sleep(1)
monitor_thread = threading.Thread(target=self.monitor_debugger)
monitor_thread.setDaemon(0)
monitor_thread.start()
def monitor_debugger(self):
while self.running == True:
input = raw_input("Enter: 'snap','restore' or 'quit'")
input = input.lower().strip()
if input == "quit":
print "[*] Exiting the snapshotter."
self.running = False
self.dbg.terminate_process()
elif input == "snap":
print "[*] Suspending all threads."
self.dbg.suspend_all_threads()
print "[*] Obtaining snapshot."
self.dbg.process_snapshot()
print "[*] Resuming operation."
self.dbg.resume_all_threads()
elif input == "restore":
print "[*] Suspending all threads."
self.dbg.suspend_all_threads()
print "[*] Restoring snapshot."
self.dbg.process_restore()
print "[*] Resuming operation."
self.dbg.resume_all_threads()
def start_debugger(self):
self.dbg = pydbg()
pid = self.dbg.load(self.exe_path)
self.pid = self.dbg.pid
self.dbg.run()
exe_path = "C:\\WINDOWS\\System32\\calc.exe"
snapshotter(exe_path)
Your code worked for me on Python 2.7 on XP SP1 (Virtual machine). Which version of Windows are you running? Also, could it be your pydbg install? I'm using a binary I found at:
http://www.lfd.uci.edu/~gohlke/pythonlibs/#pydbg