I am trying to create a function to re-project rasters in a dataset. However I get an error.
RasterioIOError: Attempt to create new tiff file 'E:/MapsForTesting/Reprojected' failed: Permission denied
How do I allow my computer or rasterio to write files? I can't seem to find the answer.
Here is my code:
```
###############################################################################################
import os
import rasterio
from rasterio.warp import calculate_default_transform , reproject, Resampling
#get working directory
os.chdir('E:\MapsForTesting')
#open unprojected raster
dstCrs = 'EPSG:32620' # the EPSG code for UTM Zone 20
def reproject(infile, outfilepath):
#kill = rasterio.open(infile) """Attempt to solve issue does not work"""
with rasterio.open(infile) as srs: #if input is halifax.jpg or raw .tif will recieve north up error
srs_crs = srs.crs
transform, width, height = calculate_default_transform(
srs.crs, dstCrs, srs.width, srs.height, *srs.bounds) #calculate transform
kwargs = srs.meta.copy()
kwargs.update({
'crs': dstCrs,
'transform': transform,
'width': width,
'height': height
})
#kill.close()# close the rasterio dataset """Attempt to solve issue does not work"""
#os.remove(infile)# delete the file """Attempt to solve issue does not work"""
with rasterio.open(outfilepath.format(), 'w', **kwargs) as dst:
for i in range(1, srs.count + 1):
reproject(
source = rasterio.band(srs, i),
destination = rasterio.band(dst, i),
srs_transform = srs.transform,
srs_crs = src.crs,
dst_transform = transform,
dst_crs = dst_crs,
resampling = Resampling.nearest)
dst.write(reproject)
return(outfilepath)
reproject('E:\MapsForTesting\halifaxCRS1.tif','E:\MapsForTesting\Reprojected')
################################################################################################# `
I have validated that my function would work if I overwrite the same file however the output is not desirable and I would like the code to write a new file instead.
Related
I'm trying to downsample a point cloud. I have 2 data formats for different parts of my data.
The .bin files cause no problems, but when I'm trying to downsample the .e57 files I encounter a strange problem.
Here's what I do:
import numpy as np
import open3d
pointfile = "path/to/file.e57"
pcd_data = np.fromfile(point_file, dtype=np.float32)
pcd_data = velo_data.reshape(-1, 4)
pcd_points = velo_data[:, :3]
pcd = open3d.geometry.PointCloud()
pcd.points = open3d.utility.Vector3dVector(pcd_points)
pcd_down = pcd.voxel_down_sample(voxel_size=0.8)
res = np.asarray(pcd_down.points)
It works fine for .bin, but when i try the .e57 I get the error:
RuntimeError: [Open3D ERROR] [VoxelDownSample] voxel_size is too small.
No matter if I use voxel_size of 0.005, 0.8, 100, 5000 or 1000000000000000.
I tried the earlier open3d Version:
pcd_down = open3d.geometry.voxel_down_sample(voxel_size=0.8)
and at least it throws no error, but my downsampled pointcloud then contains 0 points (from ~350 000).
As the file should be structured in points with 4 features, the file seems to be read correctly (this works for any of my files), as the reshape works just fine.
Any ideas?
Still have no clue about the original error, but I succesfully worked around the problem by using pye57:
https://github.com/davidcaron/pye57
together with this solution to a possibly occuring problem:
https://github.com/davidcaron/pye57/issues/6#issuecomment-803894677
With this code
import numpy as np
import open3d
import pye57
point_file = "path/to/file.e57"
e57 = pye57.E57(point_file)
data = e57.read_scan_raw(0)
assert isinstance(data["cartesianX"], np.ndarray)
assert isinstance(data["cartesianY"], np.ndarray)
assert isinstance(data["cartesianZ"], np.ndarray)
x = np.array(data["cartesianX"])
y = np.array(data["cartesianY"])
z = np.array(data["cartesianZ"])
pcd_points = np.concatenate((x, y), axis=0)
pcd_points = np.concatenate((pcd_points, z), axis=0)
pcd_points = velo_points.reshape(-1, 3)
pcd = open3d.geometry.PointCloud()
pcd.points = open3d.utility.Vector3dVector(pcd_points)
pcd_down = pcd.voxel_down_sample(voxel_size=0.0035)
I finally get a downsampled point cloud.
I have a .pfm which is a (.PF/.pf) file. I am trying to visualize it but I am unable to do so. Normally the .pfm files contain the header of the format.
PF/pf
width height
scale=1
But my file has this header.I am unable to visualize it as the image can anyone help me out. Any help is appreciated
Typ=Pic98::TPlane
Lines=750
Columns=1125
FirstLine=0
FirstColumn=0
import re
import numpy as np
file = open("PF file.PF", 'rb')
header = file.readline().rstrip().decode('utf-8')
if header == 'PF':
raise Exception('Only ONE channel image is supported.')
elif header == 'Typ=Pic98::TPlane<float>':
color = False
else:
raise Exception('Not a PFM file.')
dim_match = re.match(r'(^(\w+).(\d+)$)\n(^(\w+).(\d+)\s$)',
file.readline().decode('ascii'))
if dim_match:
width, height = map(int, dim_match.groups())
else:
raise Exception('Malformed PFM header.')
if header == 'Typ=Pic98::TPlane<float>':
scale =1
endian = '>'
else:
scale = -scale
endian = '<'
npImage = np.reshape(npImage, width,height)
npImage = np.flipud(npImage)
if ret_PIL:
img = Image.fromarray(npImage, 'F')
return img
return npImage
file.close()
Updated Answer
I have re-written my answer below in a slightly different, hopefully clearer style.
#!/usr/bin/env python3
import re
import cv2
import numpy as np
from PIL import Image
def readPF(filename):
"""Read named PF file into Numpy array"""
# Slurp entire file into memory as binary 'bytes'
with open(filename, 'rb') as f:
data = f.read()
# Check correct header, return None if incorrect
if not re.match(b'Typ=Pic98::TPlane<float>', data):
return None
# Get Lines and Columns, both must be present, else return None
L = re.search(b'Lines=(\d+)', data)
C = re.search(b'Columns=(\d+)', data)
if not (L and C):
return None
height = int(L.groups()[0])
width = int(C.groups()[0])
print(f'DEBUG: Height={height}, width={width}')
# Take the data from the END of the file in case other header lines added at start
na = np.frombuffer(data[-4*height*width:], dtype=np.dtype('<f4')).reshape((height,width))
# Some debug stuff
min, max, mean = na.min(), na.max(), na.mean()
print(f'DEBUG: min={min}, max={max}, mean={mean}')
return na
################################################################################
# Main
################################################################################
na = readPF('PF file.PF')
################################################################################
# Use either of the following to save the image:
################################################################################
# Save with OpenCV as scaled PNG
u16 = (65535*(na - np.min(na))/np.ptp(na)).astype(np.uint16)
cv2.imwrite('OpenCV.png', u16)
# Convert to PIL Image and save as TIFF
pi = Image.fromarray(na, mode='F')
pi.save('PIL.tif')
Original Answer
Not too sure what I image I should be expecting, but here is a rough idea:
#!/usr/bin/env python3
import re
import cv2
import numpy as np
from PIL import Image
file = open("PF file.PF", 'rb')
header = file.readline().rstrip().decode('utf-8')
if header == 'PF':
raise Exception('Only ONE channel image is supported.')
elif header == 'Typ=Pic98::TPlane<float>':
color = False
else:
raise Exception('Not a PFM file.')
while True:
line = file.readline().decode('ascii')
match = re.match('(\w+)=(\d+)', line)
n, v = match.groups()
if n == 'Lines':
height = int(v)
print(f'Height: {height}')
if n == 'Columns':
width = int(v)
print(f'Width: {width}')
break
# Seek backwards from the end of the file in case any clown has added something to the header
file.seek(-height*width*4,2)
# Read remainder of file into Numpy array of floats and reshape
na = np.fromfile(file, dtype=np.float32).reshape((height,width))
# Some debug stuff
min, max, mean = na.min(), na.max(), na.mean()
print(f'DEBUG: min={min}, max={max}, mean={mean}')
################################################################################
# Use either of the following to save the image:
################################################################################
# Save with OpenCV as scaled PNG
u16 = (65535*(na - np.min(na))/np.ptp(na)).astype(np.uint16)
cv2.imwrite('OpenCV.png', u16)
# Convert to PIL Image and save as TIFF
pi = Image.fromarray(na, mode='F')
pi.save('PIL.tif')
The output is as follows:
Height: 750
Width: 1125
DEBUG: min=0.0, max=127881704.0, mean=1618343.625
Another possibility is to use ImageMagick to make it into a PNG, I get the following, and ImageMagick defaults to little-endian, so if this is correct, your image is little endian.
magick -define quantum:format=floating-point -depth 32 -size 1125x750+80 gray:"PF file.pf" -auto-level image.png
Keywords: Python, ImageMagick, image processing, float, float32, Numpy, PFM
I am using this cvkit tool https://github.com/roboception/cvkit on Mac and Ubuntu. It works very well to visualize '.pfm' files (e.g., disparity or depth maps saved as pfm file).
I want to use tensorflow version 2.4.0-dev20201009 in python 3.7.
My dataset are in the subfolder "data\Images". The label of an image is a float number between 1 and 5 and can read from the allTestData.csv from the subfolder "data".
What is the best way to read the data with validation split of 30 percent? So far I wanted to use
tf.keras.preprocessing.image_dataset_from_directory but this doesn't help me to incorperate the labels correctly, as all my images are in one folder and do not have one-hot encoded vectors as labels. How would you do this in tensorflow?
For the sake of completeness, I planed to use
def create_model():
model = keras.Sequential()
model.add(MobileNetV2(input_shape=(224, 224, 3), include_top=False))
model.trainable = True
model.add(layers.GlobalAveragePooling2D())
model.add(layers.Dense(1024, activation="relu"))
model.add(layers.Dense(1, activation="softmax"))
model.compile(optimizer='adam',
loss=tf.losses.mean_squared_error,
metrics=[tf.metrics.SparseCategoricalAccuracy()])
model.summary()
return model
for training the model. The question is only regarding how to read the training data?
I will answer my own question.
The best way was to write a manual function that reads the labels and images.
Assume that the images are in 'data\Images' and the labels are in a .txt file and the labels are in a .txt file at 'data\train_test_files\All_labels.txt'. Then the following two methods will do the job:
def loadImages(IMG_SIZE):
path = os.path.join(os.getcwd(), 'data\\Images')
training_data=[]
labelMap = getLabelMap()
for img in os.listdir(path):
out_array = np.zeros((350,350, 3), np.float32) #350x350 is the pixel size of the images
try:
img_array = cv2.imread(os.path.join(path, img))
img_array=img_array.astype('float32')
out_array = cv2.normalize(img_array, out_array, 0, 1, cv2.NORM_MINMAX)
out_array = cv2.resize(out_array, (IMG_SIZE, IMG_SIZE)
training_data.append([out_array, float(labelMap[img])])
except Exception as e:
pass
return training_data
def getLabelMap():
map = {}
path = os.getcwd()
path = os.path.join(path, "data\\train_test_files\\All_labels.txt")
f = open(path, "r")
for line in f:
line = line.split() #lines in txt file are of the form 'image_name.jpg 3.2'
map[line[0]] = line[1] #3.2 is the label
f.close()
return map
#call of method:
training_set=[]
training_set = loadImages(244) #I want to have my images resized to 244x244
I've got code along the lines of the following which generates a new image out of some existing images.
from PIL import Image as pyImage
def create_compound_image(back_image_path, fore_image_path, fore_x_position):
back_image_size = get_image_size(back_image_path)
fore_image_size = get_image_size(fore_image_path)
new_image_width = (fore_image_size[0] / 2) + back_image_size[0]
new_image_height = fore_image_size[1] + back_image_size[1]
new_image = create_new_image_canvas(new_image_width, new_image_height)
back_image = pyImage.open(back_image_path)
fore_image = pyImage.open(fore_image_path)
new_image.paste(back_image, (0, 0), mask = None)
new_image.paste(fore_image, (fore_x_position, back_image_size[1]), mask = None)
return new_image
Later in the code, I've got something like this:
from kivy.uix.image import Image
img = Image(source = create_compound_image(...))
If I do the above, I get the message that Image.source only accepts string/unicode.
If I create a StringIO.StringIO() object from the new image, and try to use that as the source, the error message is the same as above. If I use the output of the StringIO object's getvalue() method as the source, the message is that the source must be encoded string without NULL bytes, not str.
What is the proper way to use the output of the create_compound_image() function as the source when creating a kivy Image object?
It seems you want to just combine two images into one, you can actually just create a texture using Texture.create and blit the data to a particular pos using Texture.blit_buffer .
from kivy.core.image import Image
from kivy.graphics import Texture
bkimg = Image(bk_img_path)
frimg = Image(fr_img_path)
new_size = ((frimg.texture.size[0]/2) + bkimg.texture.size[0],
frimg.texture.size[1] + bkimg.texture.size[1])
tex = Texture.create(size=new_size)
tex.blit_buffer(pbuffer=bkimg.texture.pixels, pos=(0, 0), size=bkimg.texture.size)
tex.blit_buffer(pbuffer=frimg.texture.pixels, pos=(fore_x_position, bkimg.texture.size[1]), size=frimg.texture.size)
Now you can use this texture anywhere directly like::
from kivy.uix.image import Image
image = Image()
image.texture = tex
source is a StringProperty and is expecting a path to file. That's why you got errors when you tried to pass PIL.Image object, StringIO object or string representation of image. It's not what framework wants. As for getting image from StringIO, it was discussed before here:
https://groups.google.com/forum/#!topic/kivy-users/l-3FJ2mA3qI
https://github.com/kivy/kivy/issues/684
You can also try much simpler, quick and dirty method - just save your image as a tmp file and read it normal way.
I have a python script with multiple figures that I would like to update during a loop. Some will be images, and others will be line/scatter plots. I am having trouble getting the image to display on the correct figure. (the line and scatter data are showing up on the right figures, but the image seems to always be going on the figure that was created last, eventually I'll be displaying more than one image figure, so I can't just create the image figure last)
Here is roughly the code I have so far, the 3D scatter plot is showing up on Figure 1, but both the image and the line plots are showing up on Figure 3, with Figure 2 blank:
import matplotlib.pyplot as plt
from collections import deque
class Bla():
def __init__( self ):
self.pc_fig = plt.figure(1)
self.pc_ax = self.pc_fig.add_subplot(111, projection='3d')
self.pc_ax.set_xlim3d([0, 50])
self.pc_ax.set_ylim3d([0, 50])
self.pc_ax.set_zlim3d([0, 20])
self.pc_ax.hold(False)
self.vts_fig = plt.figure(2)
self.vts_ax = self.vts_fig.add_subplot(111)
self.em_fig = plt.figure(3)
self.em_ax = self.em_fig.add_subplot(111)
self.em_ax.hold(True)
self.image_data = deque()
self.motion_data = deque()
plt.ion()
plt.show()
def run( self ):
em_prev_xy = ( 0, 0 )
while True:
if len( self.motion_data ) > 0:
data1 = self.motion_data.popleft()
em_xy = data1.get_em()
self.em_ax.plot( [ em_prev_xy[0], em_xy[0] ], [ em_prev_xy[1], em_xy[1] ],'b')
pc = self.get_pc()
pc_index = nonzero(pc>.002)
pc_value = pc[pc_index] * 100
self.pc_ax.scatter(pc_index[0],pc_index[1],pc_index[2],s=pc_value)
self.pc_ax.set_xlim3d([0, 50])
self.pc_ax.set_ylim3d([0, 50])
self.pc_ax.set_zlim3d([0, 20])
plt.pause( 0.0001 ) # This is needed for the display to update
if len( self.image_data ) > 0:
im = self.image_data.popleft()
plt.imshow( im, cmap=plt.cm.gray, axes=self.vts_ax )
plt.pause( 0.0001 )
def main():
bla = Bla()
bla.run()
if __name__ == "__main__":
main()
Basically I have some queues that get populated in a callback when new data arrives, and I want this data to be displayed as it arrives.
I am new to matplotlib, so any help with my image display issue or tips for better ways of using matplotlib to display figures in general will be much appreciated
You are mixing the OO and state machine interfaces. See this answer for an explanation of what is going on.
Replace this line:
plt.imshow( im, cmap=plt.cm.gray, axes=self.vts_ax )
with
the_axes_you_want.imshow(...)
which should fix your image issue.