I am trying to input a dataset from Kaggle into this notebook from the Tensorflow docs in order to train a CycleGAN model. My current approach is to download the folders into my notebook and loop through the paths of each image and use cv2.imread(path) to add the uint8 image data to a list. But this doesn't work and I know my current approach is wrong because the code provided by google requires a Prefetch dataset.
Here's my current code (excluding the opencv part)
import os
# specify the img directory path
art_path = "/content/abstract-art-gallery/Abstract_gallery/Abstract_gallery/"
land_path = "/content/landscape-pictures/"
def grab_path(folder, i_count=100):
res = []
for file in range(i_count):
if os.listdir(folder)[0].endswith(('.jpg', '.png', 'jpeg')):
img_path = folder + os.listdir(folder)[0]
res.append(img_path)
return res
art_path, land_path = grab_path(art_path), grab_path(land_path)
print(art_path)
print(land_path)
The error in the code comes here:
train_horses = train_horses.cache().map(
preprocess_image_train, num_parallel_calls=AUTOTUNE).shuffle(
BUFFER_SIZE).batch(BATCH_SIZE)
Is there a simpler approach to this problem?
import pathlib
import tensorflow as tf
import numpy as np
#tf.autograph.experimental.do_not_convert
def read_image(path):
image_string = tf.io.read_file(path)
image = DataUtils.decode_image(image_string,(image_size))
return image
AUTO = tf.data.experimental.AUTOTUNE
paths = np.array([x for x in pathlib.Path(IMAGE_PATHS_DIR).rglob('*.jpg')])
dataset = tf.data.Dataset.from_tensor_slices((paths.astype(str)))
dataset = dataset.map(self.read_image)
dataset = dataset.shuffle(2048)
dataset = dataset.prefetch(AUTOTUNE)
Related
I am trying to download a batch of files from the following website to my local folder. But the download is very slow. Average size of each file is in the range of 30 - 60 MB. Is there a better way to improve this code so that I can download them fast?
import requests
from os import mkdir
from os.path import isdir
from bs4 import BeautifulSoup
from os import chdir, getcwd
url = "https://opendata.dwd.de/climate_environment/CDC/grids_germany/hourly/radolan/historical/asc/"
years = [str(year) for year in range(2005,2021)]
links = [url + i + "/" for i in years]
Creating an empty list to store list of list:
t_links = []
def get_tarlinks():
for i in links:
#create response object
r = requests.get(i)
#create beautiful object
soup = BeautifulSoup(r.content, 'html5lib')
#find all links on webpage
a_links = soup.find_all('a')
#filter the link sending with .tar
tar_links = [i + link['href'] for link in a_links if link['href'].endswith('.tar')]
t_links.append(tar_links)
return t_links
t_links = get_tarlinks()
src_path = "D:/Sandeep/Thesis/Data/"
Download files to local machine:
for i in t_links:
for j in i:
year,filename = j.split('/')[10:]
r = requests.get(j, allow_redirects=True)
if isdir(src_path+year) == False:
mkdir(src_path+year)
chdir(src_path+year)
open(filename, "wb").write(r.content)
else:
open(filename, "wb").write(r.content)
Note: Please check for the indentation when you copy this code to your IDE. Thanks!
I want to read multiple images in python I'm using this code but when I run it,nothing happens.
Could you tell me what is the problem?
import glob , cv2
import numpy as np
def read_img(img_list , img):
n=cv2.imread(img)
img_list.append(n)
return img_list
path = glob.glob("02291G0AR/*.bmp")
list_ = []
cv_image = [read_img(list_,img) for img in path]
print(cv_image)
"02291G0AR" is the folder where my images are save in. and it's near my code file
Perhaps it's just a matter of the print function not being the adequate one to use. I'd try:
for img in cv_image:
cv2.imshow('image',img)
I am trying to deploy my tensorflow model in a tensorflow serving using tensorflow saved model. Input for my tf model is a string value and i have defined my signature as below
prediction_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={'input_path': tensor_info_input},
outputs={'output_prediction': tensor_info_output},
method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME))
builder.add_meta_graph_and_variables(
sess, [tf.saved_model.tag_constants.SERVING],
signature_def_map={
'predict_images':
prediction_signature,
})
My objective is to read a image from the path specified in signature definition. How do i get the input_path from input definition and convert the tensor into actual string value to read the path
I think you need to add some "reading operation" in your model file.
Like this:
def inference(image_path_string):
# read image from a image path string
image_file = tf.read_file(image_path_string)
image_raw = tf.cond(tf.image.is_jpeg(image_file),
lambda: tf.image.decode_jpeg(image_file, channels= 3 if rgb, else 1),
lambda: tf.image.decode_bmp(image_file))
# image preprocessing code
...
...
# your prediction model code
...
...
You can place the below mentioned code in a Python file say, Client.py. And for inference, you can run the below mentioned command in the Terminal
python Client.py --image abc/0.png --model mnist --signature_name predict
This will take the path of the image, "abc/0.png" and will convert it to Numeric Values, and will run the inference.
Code for Client.py is mentioned below. The below code is for MNIST Images. You can reshape your images accordingly:
from __future__ import print_function
import argparse
import time
import numpy as np
from scipy.misc import imread
import grpc
from tensorflow.contrib.util import make_tensor_proto
from tensorflow_serving.apis import predict_pb2
from tensorflow_serving.apis import prediction_service_pb2_grpc
def run(host, port, image, model, signature_name):
channel = grpc.insecure_channel('{host}:{port}'.format(host=host, port=port))
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
# Read an image
data = imread(image)
data = data.astype(np.float32)
print(data)
start = time.time()
# Call classification model to make prediction on the image
request = predict_pb2.PredictRequest()
request.model_spec.name = model
request.model_spec.signature_name = signature_name
request.inputs['image'].CopyFrom(make_tensor_proto(data, shape=[1, 28, 28, 1]))
result = stub.Predict(request, 10.0)
end = time.time()
time_diff = end - start
# Reference:
# How to access nested values
# https://stackoverflow.com/questions/44785847/how-to-retrieve-float-val-from-a-predictresponse-object
print(result)
print('time elapased: {}'.format(time_diff))
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--host', help='Tensorflow server host name', default='localhost', type=str)
parser.add_argument('--port', help='Tensorflow server port number', default=8500, type=int)
parser.add_argument('--image', help='input image', type=str)
parser.add_argument('--model', help='model name', type=str)
parser.add_argument('--signature_name', help='Signature name of saved TF model',
default='serving_default', type=str)
args = parser.parse_args()
run(args.host, args.port, args.image, args.model, args.signature_name)
For more information, you can refer this beautiful article on Tensorflow Serving,
https://medium.com/#yuu.ishikawa/serving-pre-modeled-and-custom-tensorflow-estimator-with-tensorflow-serving-12833b4be421
what changes should i do in this code so i don't have to save image to disk in step [A] then again read it from disk in step [B]. as showing in code. can anyone help me this with changes in the code or some tips?
import io
import os
import six
from google.cloud import vision
from google.cloud import translate
from google.cloud.vision import types
import json
from wand.image import Image
client = vision.ImageAnnotatorClient()
sample_pdf = Image(filename='CMB72_CMB0720160.pdf[0]', resolution=500)
blank = Image(filename='Untitled.png')
all_ = sample_pdf.clone()
polling_ = sample_pdf.clone()
voters = sample_pdf.clone()
all_.crop(3000,2800,3800,3860)
polling_.crop(870,4330,2900,4500)
voters.crop(1300,4980,2000,5250)
blank.composite(all_,left=0,top=0)
blank.composite(voters,left=0,top=1100)
blank.composite(polling_,left=0,top=1420)
blank.save('CMB72_CMB0720122.jpg')---------------[A]
file_name = 'CMB72_CMB0720122.jpg'-------------|
with io.open(file_name,'rb') as image_file:----|>[B]
content = image_file.read()---------------|
image = types.Image(content= content)
image_context = vision.types.ImageContext(
language_hints=['hi'])
response = client.document_text_detection(image=image)
texts = response.text_annotations
file = open('jin.txt','w+',encoding='utf-8')
file.write(texts[0].description)
file.close()
Use the wand.image.Image.make_blob method.
content = blank.make_blob('JPEG')
I have retrained an InceptionV3 model on my own data and am trying to modify the code from the Tensorflow image classification tutorial here https://www.tensorflow.org/tutorials/image_recognition.
I attempted reading in the directory as a list and looping over it but this didn't work:
load_graph(FLAGS.graph)
filelist = os.listdir(FLAGS.image)
for i in filelist:
# load image
image_data = load_image(i)
I just get an error saying that FLAGS hasn't been defined, so I guess FLAGS has to go together with the load_image function? This is the original program:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import argparse
import sys
import os
import tensorflow as tf
parser = argparse.ArgumentParser()
parser.add_argument(
'--image', required=True, type=str, help='Absolute path to image file.')
parser.add_argument(
'--num_top_predictions',
type=int,
default=5,
help='Display this many predictions.')
parser.add_argument(
'--graph',
required=True,
type=str,
help='Absolute path to graph file (.pb)')
parser.add_argument(
'--labels',
required=True,
type=str,
help='Absolute path to labels file (.txt)')
parser.add_argument(
'--output_layer',
type=str,
default='final_result:0',
help='Name of the result operation')
parser.add_argument(
'--input_layer',
type=str,
default='DecodeJpeg/contents:0',
help='Name of the input operation')
def load_image(filename):
"""Read in the image_data to be classified."""
return tf.gfile.FastGFile(filename, 'rb').read()
def load_labels(filename):
"""Read in labels, one label per line."""
return [line.rstrip() for line in tf.gfile.GFile(filename)]
def load_graph(filename):
"""Unpersists graph from file as default graph."""
with tf.gfile.FastGFile(filename, 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
tf.import_graph_def(graph_def, name='')
def run_graph(image_data, labels, input_layer_name, output_layer_name,
num_top_predictions):
with tf.Session() as sess:
# Feed the image_data as input to the graph.
# predictions will contain a two-dimensional array, where one
# dimension represents the input image count, and the other has
# predictions per class
softmax_tensor = sess.graph.get_tensor_by_name(output_layer_name)
predictions, = sess.run(softmax_tensor, {input_layer_name: image_data})
# Sort to show labels in order of confidence
top_k = predictions.argsort()[-num_top_predictions:][::-1]
for node_id in top_k:
human_string = labels[node_id]
score = predictions[node_id]
print('%s (score = %.5f)' % (human_string, score))
return 0
def main(argv):
"""Runs inference on an image."""
if argv[1:]:
raise ValueError('Unused Command Line Args: %s' % argv[1:])
if not tf.gfile.Exists(FLAGS.image):
tf.logging.fatal('image file does not exist %s', FLAGS.image)
if not tf.gfile.Exists(FLAGS.labels):
tf.logging.fatal('labels file does not exist %s', FLAGS.labels)
if not tf.gfile.Exists(FLAGS.graph):
tf.logging.fatal('graph file does not exist %s', FLAGS.graph)
# load image
image_data = load_image(FLAGS.image)
# load labels
labels = load_labels(FLAGS.labels)
# load graph, which is stored in the default session
load_graph(FLAGS.graph)
run_graph(image_data, labels, FLAGS.input_layer, FLAGS.output_layer,
FLAGS.num_top_predictions)
if __name__ == '__main__':
FLAGS, unparsed = parser.parse_known_args()
tf.app.run(main=main, argv=sys.argv[:1]+unparsed)
Try tf.flags.FLAGS, or at the top, from tf.flags import FLAGS
Try the following,
import os
import tensorflow as tf
# Define this after your imports. This is similar to python argparse except more verbose
FLAGS = tf.app.flags.FLAGS
tf.app.flags.DEFINE_string('image', '/Users/photos',
"""
Define your 'image' folder here
or as an argument to your script
for eg, test.py --image /Users/..
""")
# use listdir to list the images in the target folder
filelist = os.listdir(FLAGS.image)
# now iterate over the objects in the list
for i in filelist:
# load image
image_data = load_image(i)
This should work. Hope it helps.
Thanks for the help given, The FLAGS come from the argparser module and not the TensorFlow flags module, and FLAGS may have to be called from within a function. I eventually solved this by making a separate function so I think that's what is happening:
def get_image_list(path):
return glob.glob(path + '*.jpg')
Then further down calling a loop:
filelist = get_image_list(FLAGS.image)
for i in filelist:
image_data = load_image(i)
run_graph(image_data, labels, FLAGS.input_layer, FLAGS.output_layer,
FLAGS.num_top_predictions)