I am using Python 3.7.3 in macOS system in Anaconda environment. Tensorflow (1.14.0), Matplotlib (3.1.0) and other modules were installed there and everything worked fine. I wrote the following codes and run it.
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
def add_layer(inputs, inputs_size, outputs_size,activation_function = None):
with tf.name_scope('layer'):
with tf.name_scope('weight'):
Weights = tf.Variable(tf.random.normal([inputs_size, outputs_size]))
with tf.name_scope('biase'):
biases = tf.Variable(tf.zeros([1,outputs_size])+0.1)
with tf.name_scope('wx_plus_b'):
Wx_plus_b = tf.matmul(inputs, Weights) + biases
if activation_function == None:outputs = Wx_plus_b
else: outputs = activation_function(Wx_plus_b)
return outputs
'''
multiple lines omitted here
'''
writer = tf.compat.v1.summary.FileWriter("logs/",sess.graph)
I can see a local file with name
"events.out.tfevents.1561289962.Botaos-MacBook-Pro.local"
generated in "logs/" folder. I opened terminal and cd to that folder with Anaconda environment activated. Then I typed
"python -m tensorboard.main --logdir=‘logs/‘ --host localhost --port 6006"
and got response
TensorBoard 1.14.0 at http://localhost:6006/ (Press CTRL+C to quit)
Then no matter I use safari or chrome to open "http://localhost:6006/", there's always nothing shown except "No dashboards are active for the current data set."
enter image description here
Actually I also tried other commends such as
python -m tensorboard.main --logd logs --host localhost --port 6006
python -m tensorboard.main --logd logs --host localhost --port 6006
But there's no difference.
The original codes as follows:
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
def add_layer(inputs, inputs_size, outputs_size,activation_function = None):
with tf.name_scope('layer'):
with tf.name_scope('weight'):
Weights = tf.Variable(tf.random.normal([inputs_size, outputs_size]))
with tf.name_scope('biase'):
biases = tf.Variable(tf.zeros([1,outputs_size])+0.1)
with tf.name_scope('wx_plus_b'):
Wx_plus_b = tf.matmul(inputs, Weights) + biases
if activation_function == None:outputs = Wx_plus_b
else: outputs = activation_function(Wx_plus_b)
return outputs
x_data = np.linspace(-1,1,300,dtype = np.float32)[:,np.newaxis]
noise = np.random.normal(0,0.05,x_data.shape).astype(np.float32)
y_data = np.square(x_data) - 0.5 + noise
with tf.name_scope('inputs'):
xs = tf.compat.v1.placeholder(tf.float32,[None,1],name='x_in')
ys = tf.compat.v1.placeholder(tf.float32,[None,1],name='y_in')
l1 = add_layer(xs, 1, 10, tf.nn.relu)
prediction = add_layer(l1, 10, 1, None)
with tf.name_scope('loss'):
loss = tf.reduce_mean(tf.reduce_sum(tf.square(prediction - ys),reduction_indices=[1])) #no need to do tf.sum() as in link. #tf.reduce_mean()
with tf.name_scope('train'):
train_step = tf.compat.v1.train.GradientDescentOptimizer(0.1).minimize(loss)
sess = tf.compat.v1.Session()
writer = tf.compat.v1.summary.FileWriter("logs/",sess.graph)
sess.run(tf.compat.v1.global_variables_initializer())
I think the problem is that you are trying to find the logs directory while you are already in the logs directory.
try executing: tensorboard --logdir logs
from the directory containing the logs dir.
Related
I am trying to input a dataset from Kaggle into this notebook from the Tensorflow docs in order to train a CycleGAN model. My current approach is to download the folders into my notebook and loop through the paths of each image and use cv2.imread(path) to add the uint8 image data to a list. But this doesn't work and I know my current approach is wrong because the code provided by google requires a Prefetch dataset.
Here's my current code (excluding the opencv part)
import os
# specify the img directory path
art_path = "/content/abstract-art-gallery/Abstract_gallery/Abstract_gallery/"
land_path = "/content/landscape-pictures/"
def grab_path(folder, i_count=100):
res = []
for file in range(i_count):
if os.listdir(folder)[0].endswith(('.jpg', '.png', 'jpeg')):
img_path = folder + os.listdir(folder)[0]
res.append(img_path)
return res
art_path, land_path = grab_path(art_path), grab_path(land_path)
print(art_path)
print(land_path)
The error in the code comes here:
train_horses = train_horses.cache().map(
preprocess_image_train, num_parallel_calls=AUTOTUNE).shuffle(
BUFFER_SIZE).batch(BATCH_SIZE)
Is there a simpler approach to this problem?
import pathlib
import tensorflow as tf
import numpy as np
#tf.autograph.experimental.do_not_convert
def read_image(path):
image_string = tf.io.read_file(path)
image = DataUtils.decode_image(image_string,(image_size))
return image
AUTO = tf.data.experimental.AUTOTUNE
paths = np.array([x for x in pathlib.Path(IMAGE_PATHS_DIR).rglob('*.jpg')])
dataset = tf.data.Dataset.from_tensor_slices((paths.astype(str)))
dataset = dataset.map(self.read_image)
dataset = dataset.shuffle(2048)
dataset = dataset.prefetch(AUTOTUNE)
I am trying to deploy my tensorflow model in a tensorflow serving using tensorflow saved model. Input for my tf model is a string value and i have defined my signature as below
prediction_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={'input_path': tensor_info_input},
outputs={'output_prediction': tensor_info_output},
method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME))
builder.add_meta_graph_and_variables(
sess, [tf.saved_model.tag_constants.SERVING],
signature_def_map={
'predict_images':
prediction_signature,
})
My objective is to read a image from the path specified in signature definition. How do i get the input_path from input definition and convert the tensor into actual string value to read the path
I think you need to add some "reading operation" in your model file.
Like this:
def inference(image_path_string):
# read image from a image path string
image_file = tf.read_file(image_path_string)
image_raw = tf.cond(tf.image.is_jpeg(image_file),
lambda: tf.image.decode_jpeg(image_file, channels= 3 if rgb, else 1),
lambda: tf.image.decode_bmp(image_file))
# image preprocessing code
...
...
# your prediction model code
...
...
You can place the below mentioned code in a Python file say, Client.py. And for inference, you can run the below mentioned command in the Terminal
python Client.py --image abc/0.png --model mnist --signature_name predict
This will take the path of the image, "abc/0.png" and will convert it to Numeric Values, and will run the inference.
Code for Client.py is mentioned below. The below code is for MNIST Images. You can reshape your images accordingly:
from __future__ import print_function
import argparse
import time
import numpy as np
from scipy.misc import imread
import grpc
from tensorflow.contrib.util import make_tensor_proto
from tensorflow_serving.apis import predict_pb2
from tensorflow_serving.apis import prediction_service_pb2_grpc
def run(host, port, image, model, signature_name):
channel = grpc.insecure_channel('{host}:{port}'.format(host=host, port=port))
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
# Read an image
data = imread(image)
data = data.astype(np.float32)
print(data)
start = time.time()
# Call classification model to make prediction on the image
request = predict_pb2.PredictRequest()
request.model_spec.name = model
request.model_spec.signature_name = signature_name
request.inputs['image'].CopyFrom(make_tensor_proto(data, shape=[1, 28, 28, 1]))
result = stub.Predict(request, 10.0)
end = time.time()
time_diff = end - start
# Reference:
# How to access nested values
# https://stackoverflow.com/questions/44785847/how-to-retrieve-float-val-from-a-predictresponse-object
print(result)
print('time elapased: {}'.format(time_diff))
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--host', help='Tensorflow server host name', default='localhost', type=str)
parser.add_argument('--port', help='Tensorflow server port number', default=8500, type=int)
parser.add_argument('--image', help='input image', type=str)
parser.add_argument('--model', help='model name', type=str)
parser.add_argument('--signature_name', help='Signature name of saved TF model',
default='serving_default', type=str)
args = parser.parse_args()
run(args.host, args.port, args.image, args.model, args.signature_name)
For more information, you can refer this beautiful article on Tensorflow Serving,
https://medium.com/#yuu.ishikawa/serving-pre-modeled-and-custom-tensorflow-estimator-with-tensorflow-serving-12833b4be421
To Whom It May Concern,
The code below is being run in a Docker container based on jupyter's data science notebook;
however, I've install Java 8 and h2o (version 3.20.0.7), as well as exposed the necessary ports. The docker container is being run on a system using Ubuntu 16.04 and has 32 threads and over 300G of RAM.
h2o is using all the threads and 26.67 Gb of memory. I'm attempted to classify text as either a 0 or a 1 using the code below.
However, despite setting max_runtime_secs to 900 or 15 minutes, the code hadn't finished executing and was still tying up most of the machine resources ~15 hours later. As a side note, it took df_train about 20 minutes to parse. Any thoughts on what's going wrong?
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.metrics import accuracy_score, recall_score, precision_score, f1_score
df = pd.read_csv('Data.csv')[['Text', 'Classification']]
vectorizer = CountVectorizer(analyzer='word',token_pattern=r'\w{1,}',
ngram_range=(1, 3), stop_words = 'english')
x_train_vec = vectorizer.fit_transform(df['Text'])
y_train = df['Classification']
import h2o
from h2o.automl import H2OAutoML
h2o.init()
df_train = h2o.H2OFrame(x_train_vec.A, header=-1, column_names=vectorizer.get_feature_names())
df_labels = h2o.H2OFrame(y_train.reset_index()[['Classification']])
df_train = df_train.concat(df_labels)
x_train_cn = df_train.columns
y_train_cn = 'Classification'
x_train_cn.remove(y_train_cn)
df_train[y_train_cn] = df_train[y_train_cn].asfactor()
h2o_aml = H2OAutoML(max_runtime_secs = 900, exclude_algos = ["DeepLearning"])
h2o_aml.train(x = x_train_cn , y = y_train_cn, training_frame = df_train)
lb = h2o_aml.leaderboard
y_predict = h2o_aml.leader.predict(df_train.drop('Classification'))
print('accuracy: {}'.format(accuracy_score(y_pred=y_predict, y_true=y_train)))
print('precision: {}'.format(precision_score(y_pred=y_predict, y_true=y_train)))
print('recall: {}'.format(recall_score(y_pred=y_predict, y_true=y_train)))
print('f1: {}\n'.format(f1_score(y_pred=y_predict, y_true=y_train)))
This is a bug that has been fixed on master. If you want, you can try out the fix now on the nightly release, otherwise, it will be fixed in the next stable release of H2O, 3.22.
I am using tesseract to perform OCR on screengrabs. I have an app using a tkinter window leveraging self.after in the initialization of my class to perform constant image scrapes and update label, etc values in the tkinter window. I have searched for multiple days and can't find any specific examples how to leverage CREATE_NO_WINDOW with Python3.6 on a Windows platform calling tesseract with pytesseract.
This is related to this question:
How can I hide the console window when I run tesseract with pytesser
I have only been programming Python for 2 weeks and don't understand what/how to perform the steps in the above question. I opened up the pytesseract.py file and reviewed and found the proc = subprocess.Popen(command, stderr=subproces.PIPE) line but when I tried editing it I got a bunch of errors that I couldn't figure out.
#!/usr/bin/env python
'''
Python-tesseract. For more information: https://github.com/madmaze/pytesseract
'''
try:
import Image
except ImportError:
from PIL import Image
import os
import sys
import subprocess
import tempfile
import shlex
# CHANGE THIS IF TESSERACT IS NOT IN YOUR PATH, OR IS NAMED DIFFERENTLY
tesseract_cmd = 'tesseract'
__all__ = ['image_to_string']
def run_tesseract(input_filename, output_filename_base, lang=None, boxes=False,
config=None):
'''
runs the command:
`tesseract_cmd` `input_filename` `output_filename_base`
returns the exit status of tesseract, as well as tesseract's stderr output
'''
command = [tesseract_cmd, input_filename, output_filename_base]
if lang is not None:
command += ['-l', lang]
if boxes:
command += ['batch.nochop', 'makebox']
if config:
command += shlex.split(config)
proc = subprocess.Popen(command, stderr=subprocess.PIPE)
status = proc.wait()
error_string = proc.stderr.read()
proc.stderr.close()
return status, error_string
def cleanup(filename):
''' tries to remove the given filename. Ignores non-existent files '''
try:
os.remove(filename)
except OSError:
pass
def get_errors(error_string):
'''
returns all lines in the error_string that start with the string "error"
'''
error_string = error_string.decode('utf-8')
lines = error_string.splitlines()
error_lines = tuple(line for line in lines if line.find(u'Error') >= 0)
if len(error_lines) > 0:
return u'\n'.join(error_lines)
else:
return error_string.strip()
def tempnam():
''' returns a temporary file-name '''
tmpfile = tempfile.NamedTemporaryFile(prefix="tess_")
return tmpfile.name
class TesseractError(Exception):
def __init__(self, status, message):
self.status = status
self.message = message
self.args = (status, message)
def image_to_string(image, lang=None, boxes=False, config=None):
'''
Runs tesseract on the specified image. First, the image is written to disk,
and then the tesseract command is run on the image. Tesseract's result is
read, and the temporary files are erased.
Also supports boxes and config:
if boxes=True
"batch.nochop makebox" gets added to the tesseract call
if config is set, the config gets appended to the command.
ex: config="-psm 6"
'''
if len(image.split()) == 4:
# In case we have 4 channels, lets discard the Alpha.
# Kind of a hack, should fix in the future some time.
r, g, b, a = image.split()
image = Image.merge("RGB", (r, g, b))
input_file_name = '%s.bmp' % tempnam()
output_file_name_base = tempnam()
if not boxes:
output_file_name = '%s.txt' % output_file_name_base
else:
output_file_name = '%s.box' % output_file_name_base
try:
image.save(input_file_name)
status, error_string = run_tesseract(input_file_name,
output_file_name_base,
lang=lang,
boxes=boxes,
config=config)
if status:
errors = get_errors(error_string)
raise TesseractError(status, errors)
f = open(output_file_name, 'rb')
try:
return f.read().decode('utf-8').strip()
finally:
f.close()
finally:
cleanup(input_file_name)
cleanup(output_file_name)
def main():
if len(sys.argv) == 2:
filename = sys.argv[1]
try:
image = Image.open(filename)
if len(image.split()) == 4:
# In case we have 4 channels, lets discard the Alpha.
# Kind of a hack, should fix in the future some time.
r, g, b, a = image.split()
image = Image.merge("RGB", (r, g, b))
except IOError:
sys.stderr.write('ERROR: Could not open file "%s"\n' % filename)
exit(1)
print(image_to_string(image))
elif len(sys.argv) == 4 and sys.argv[1] == '-l':
lang = sys.argv[2]
filename = sys.argv[3]
try:
image = Image.open(filename)
except IOError:
sys.stderr.write('ERROR: Could not open file "%s"\n' % filename)
exit(1)
print(image_to_string(image, lang=lang))
else:
sys.stderr.write('Usage: python pytesseract.py [-l lang] input_file\n')
exit(2)
if __name__ == '__main__':
main()
The code I am leveraging is similar to the example in the similar question:
def get_string(img_path):
# Read image with opencv
img = cv2.imread(img_path)
# Convert to gray
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# Apply dilation and erosion to remove some noise
kernel = np.ones((1, 1), np.uint8)
img = cv2.dilate(img, kernel, iterations=1)
img = cv2.erode(img, kernel, iterations=1)
# Write image after removed noise
cv2.imwrite(src_path + "removed_noise.png", img)
# Apply threshold to get image with only black and white
# Write the image after apply opencv to do some ...
cv2.imwrite(src_path + "thres.png", img)
# Recognize text with tesseract for python
result = pytesseract.image_to_string(Image.open(src_path + "thres.png"))
return result
When it gets to the following line, there is a flash of a black console window for less than a second and then it closes when it runs the command.
result = pytesseract.image_to_string(Image.open(src_path + "thres.png"))
Here is the picture of the console window:
Program Files (x86)_Tesseract
Here is what is suggested from the other question:
You're currently working in IDLE, in which case I don't think it
really matters if a console window pops up. If you're planning to
develop a GUI app with this library, then you'll need to modify the
subprocess.Popen call in pytesser.py to hide the console. I'd first
try the CREATE_NO_WINDOW process creation flag. – eryksun
I would greatly appreciate any help for how to modify the subprocess.Popen call in the pytesseract.py library file using CREATE_NO_WINDOW. I am also not sure of the difference between pytesseract.py and pytesser.py library files. I would leave a comment on the other question to ask for clarification but I can't until I have more reputation on this site.
I did more research and decided to learn more about subprocess.Popen:
Documentation for subprocess
I also referenced the following articles:
using python subprocess.popen..can't prevent exe stopped working prompt
I changed the original line of code in pytesseract.py:
proc = subprocess.Popen(command, stderr=subprocess.PIPE)
to the following:
proc = subprocess.Popen(command, stderr=subprocess.PIPE, creationflags = CREATE_NO_WINDOW)
I ran the code and got the following error:
Exception in Tkinter callback Traceback (most recent call last):
File
"C:\Users\Steve\AppData\Local\Programs\Python\Python36-32\lib\tkinter__init__.py",
line 1699, in call
return self.func(*args) File "C:\Users\Steve\Documents\Stocks\QuickOrder\QuickOrderGUI.py", line
403, in gather_data
update_cash_button() File "C:\Users\Steve\Documents\Stocks\QuickOrder\QuickOrderGUI.py", line
208, in update_cash_button
currentCash = get_string(src_path + "cash.png") File "C:\Users\Steve\Documents\Stocks\QuickOrder\QuickOrderGUI.py", line
150, in get_string
result = pytesseract.image_to_string(Image.open(src_path + "thres.png")) File
"C:\Users\Steve\AppData\Local\Programs\Python\Python36-32\lib\site-packages\pytesseract\pytesseract.py",
line 125, in image_to_string
config=config) File "C:\Users\Steve\AppData\Local\Programs\Python\Python36-32\lib\site-packages\pytesseract\pytesseract.py",
line 49, in run_tesseract
proc = subprocess.Popen(command, stderr=subprocess.PIPE, creationflags = CREATE_NO_WINDOW) NameError: name 'CREATE_NO_WINDOW'
is not defined
I then defined the CREATE_NO_WINDOW variable:
#Assignment of the value of CREATE_NO_WINDOW
CREATE_NO_WINDOW = 0x08000000
I got the value of 0x08000000 from the above linked article. After adding the definition I ran the application and I didn't get any more console window popups.
I'm running Jupyter 5.0.0 notebook with Python 3.5.3 on Windows 10. The following example code fails to run:
from concurrent.futures import as_completed, ProcessPoolExecutor
import time
import numpy as np
def do_work(idx1, idx2):
time.sleep(0.2)
return np.mean([idx1, idx2])
with ProcessPoolExecutor(max_workers=4) as executor:
futures = set()
for idx in range(32):
future = winprocess.submit(
executor, do_work, idx, idx * 2
)
futures.add(future)
for future in as_completed(futures):
print(future.result())
... and throws BrokenProcessPool: A process in the process pool was terminated abruptly while the future was running or pending.
The code works perfectly fine on Ubuntu 14.04.
I've understand that Windows doesn't have os.fork, thus multiprocessing is handled differently, and doesn't always play nice with interactive mode and Jupyter.
What are some workarounds to make ProcessPoolExecutor work in this case?
There are some similar questions, but they relate to multiprocessing.Pool:
multiprocessing.Pool in jupyter notebook works on linux but not windows
Closer inspection shows that a Jupyter notebook can run external python modules which is parallelized using ProcessPoolExecutor. So, a solution is to do the parallelizable part of your code in a module and call it from the Jupyter notebook.
That said, this can be generalized as a utility. The following can be stored as a module, say, winprocess.py and imported by jupyter.
import inspect
import types
def execute_source(callback_imports, callback_name, callback_source, args):
for callback_import in callback_imports:
exec(callback_import, globals())
exec('import time' + "\n" + callback_source)
callback = locals()[callback_name]
return callback(*args)
def submit(executor, callback, *args):
callback_source = inspect.getsource(callback)
callback_imports = list(imports(callback.__globals__))
callback_name = callback.__name__
future = executor.submit(
execute_source,
callback_imports, callback_name, callback_source, args
)
return future
def imports(callback_globals):
for name, val in list(callback_globals.items()):
if isinstance(val, types.ModuleType) and val.__name__ != 'builtins' and val.__name__ != __name__:
import_line = 'import ' + val.__name__
if val.__name__ != name:
import_line += ' as ' + name
yield import_line
Here is how you would use this:
from concurrent.futures import as_completed, ProcessPoolExecutor
import time
import numpy as np
import winprocess
def do_work(idx1, idx2):
time.sleep(0.2)
return np.mean([idx1, idx2])
with ProcessPoolExecutor(max_workers=4) as executor:
futures = set()
for idx in range(32):
future = winprocess.submit(
executor, do_work, idx, idx * 2
)
futures.add(future)
for future in as_completed(futures):
print(future.result())
Notice that executor has been changed with winprocess and the original executor is passed to the submit function as a parameter.
What happens here is that the notebook function code and imports are serialized and passed to the module for execution. The code is not executed until it is safely in a new process, thus does not trip up with trying to make a new process based on the jupyter notebook itself.
Imports are handled in such a way as to maintain aliases. The import magic can be removed if you make sure to import everything needed for the function being executed inside the function itself.
Also, this solution only works if you pass all necessary variables as arguments to the function. The function should be static so to speak, but I think that's a requirement of ProcessPoolExecutor as well. Finally, make sure you don't execute other functions defined elsewhere in the notebook. Only external modules will be imported, thus other notebook functions won't be included.