when i use my vgg16 to extract features from my images i get list of features with lot of zero in it just like that : (0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0.08844849, 0.40013123, 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0.59819716, 0. , 0. , 0. , 0.03238258, 0. , 0. , 0. , 0.32838312, 0.42723358, 0.10626598, 0. , 0.1936072 , 0.71200961, 0.38265556, 0.06889667, 0.3056691 , 0.35969719, 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0.9093371 , 0.17480908, 0. , 0. , 0.07098073, 0.41668156, 0. , 0. , 0. , 0. , 0.93668669, 0. , 0. , 0. , 1.08008015, 0. , 0.24190509, 1.52572215)
and this zeros come from the negative values in the images because the relu activation function cant deal with negative values and the output become zero like u see.
iam asking is this normal to train my LSTM with the feature i extract with the vgg16 with this zeros in it or the training wont be good ?
and if it not good to have lot of zeros in my features what should i do as an alternative solution ?
this is my code :
import pandas as pd
import numpy as np
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Convolution2D, MaxPooling2D, Flatten, Dense, Dropout, GlobalAveragePooling2D
from tensorflow.keras.applications import VGG16
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Conv2D, MaxPooling2D,MaxPool2D
from tensorflow.keras.layers import Dense, Dropout
from tensorflow.keras.layers import Flatten, BatchNormalization
import os, shutil
from keras.preprocessing.image import ImageDataGenerator
import keras
img_width, img_height = 224, 224
conv_base = VGG16(weights='imagenet',
include_top=False,
input_shape=(img_width, img_height, 3))
train_dir = "/content/drive/My Drive/small_data/data/train"
valid_dir = "/content/drive/My Drive/small_data/data/validation"
datagen = ImageDataGenerator(rescale=1./255)
batch_size = 1
#this is the function to extract the features
def extract_features(directory, sample_count):
features = np.zeros(shape=(sample_count, 7, 7, 512)) # Must be equal to the output of the convolutional base
labels = np.zeros(shape=(sample_count,6))
# Preprocess data
generator = datagen.flow_from_directory(directory,
target_size=(img_width,img_height),
batch_size = batch_size,
class_mode='binary')
# Pass data through convolutional base
i = 0
for inputs_batch, labels_batch in generator:
features_batch = conv_base .predict(inputs_batch)
features[i * batch_size: (i + 1) * batch_size] = features_batch
labels[i * batch_size: (i + 1) * batch_size] = labels_batch
i += 1
if i * batch_size >= sample_count:
break
return features, labels
# here i call the function
train_features, train_labels = extract_features(train_dir, 43)
validation_features, validation_labels = extract_features(valid_dir, 28)
# when i print train_features variable i get list of features containing
numbers and zeros like what i wrote
Related
try to build an image binary classification model using keras. Unfortunately, get a same output every time. The probability for each test sample was different, but they all favor one label.
The datasets are balanced. label L(n=250) vs. Label E(n=250): 300 for train, 100 for validate, 100 for test. There is no sample overlap among those groups.
After failing to predict the test dataset, I also used the training dataset for prediction which meant the model would make predictions for the samples that had just been trained. I know it does not make any sense. But it also got same output: Counter({0: 300}).
from keras.layers.core import Dense, Flatten, Dropout
from keras.layers.convolutional import Conv2D, MaxPooling2D, SeparableConv2D
import keras
from keras import layers
from skimage.transform import resize
import math
import os,random
import cv2
import numpy as np
import pandas as pd
from keras.models import Sequential
from sklearn.model_selection import KFold
from sklearn.metrics import confusion_matrix
from collections import Counter
import matplotlib.pyplot as plt
class DataGenerator(keras.utils.Sequence):
def __init__(self, datas, batch_size=32, shuffle=True):
self.batch_size = batch_size
self.datas = datas
self.indexes = np.arange(len(self.datas))
self.shuffle = shuffle
def __len__(self):
return math.ceil(len(self.datas) / float(self.batch_size))
def __getitem__(self, index):
batch_indexs = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
batch_datas = [self.datas[k] for k in batch_indexs]
X, y = self.data_generation(batch_datas)
return X, y
def on_epoch_end(self):
if self.shuffle == True:
np.random.shuffle(self.indexes)
def data_generation(self, batch_datas):
images = []
labels = []
for i, data in enumerate(batch_datas):
image = resize((cv2.imread(data)/255),(128, 128))
image = list(image)
images.append(image)
right = data.rfind("\\",0)
left = data.rfind("\\",0,right)+1
class_name = data[left:right]
if class_name=="e":
labels.append(0)
else:
labels.append(1)
return np.array(images), np.array(labels)
def create_model():
model = Sequential()
model.add(Conv2D(8, kernel_size=(3, 3),
input_shape=(128, 128, 3),
activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(16, kernel_size=(3, 3),
padding="same",
activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(32, kernel_size=(3, 3),
activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(units=64, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(units=1, activation='sigmoid'))
model.compile(optimizer='sgd',
loss='binary_crossentropy',
metrics=['accuracy'])
return model
e_train = []
e_test = []
l_test = []
l_train = []
for file in os.listdir('\e\train')
e_train.append(os.path.join('\e\train',file))
for file in os.listdir('\e\test')
e_test.append(os.path.join('\e\test',file))
for file in os.listdir('\l\train')
l_train.append(os.path.join('\l\train',file))
for file in os.listdir('\l\test')
e_test.append(os.path.join('\l\test',file))
data_tr = e_train + l_train
data_te = e_test + l_test
g_te = DataGenerator(data_te)
kf = KFold(n_splits=4, shuffle=True, random_state=seed)
fold = 1
for train, test in kf.split(data_tr):
model = create_model()
g_tr = DataGenerator(data_tr[train])
g_v = DataGenerator(data_tr[test])
H = model.fit_generator(generator = g_tr, epochs=10,
validation_data = g_v, shuffle=False,
max_queue_size=10,workers=1)
pred = model.predict(g_te, max_queue_size=10, workers=1, verbose=1)
print(pred)
# the probability was different, but the right column of probability were always bigger
# [[0.49817565 0.5018243 ]
# [0.4872172 0.5127828 ]
# [0.48092505 0.519075 ]
predicted_class_indices = [np.argmax(probas) for probas in pred]
print(Counter(predicted_class_indices))
# the output was the same
# Counter({0: 100})
fold = fold + 1
Any thoughts would be appreciated.
Solution:
Instead of solving a binary classification problem, convert it into a multi-class problem with two classes. So, the output of the last layer will have a softmax activation which will provide a probability distribution for the classes. Refer this tutorial wherein you'll understand the changes which need to be made.
Explanation
You should not use the sigmoid activation in the output layer of the model, while using relu activation in the intermediate layers. The vanilla ReLU ( Rectified Linear Unit ) activation is defined as,
Hence, the range of the activation function is [ 0 , infinity ). On the other hand, considering a sigmoid activation,
The range of the sigmoid function is ( 0 , 1 ). So, if a large signal ( > 0 ) is passed through the sigmoid function, the output will be very close to 1 i.e. a fully saturated firing. The output of the relu function can provide a large signal from the intermediate layers, hence making a fully saturated firing ( or 1s ) at the output layer where the sigmoid activation is performed.
If the logits are [ 5.6 , 1.2 , 3.2 , 4.8 ], the output of the sigmoid function is,
[0.9963157 , 0.76852477, 0.96083426, 0.99183744]
and that of softmax is,
[0.6441953 , 0.00790901, 0.05844009, 0.2894557 ]
I have a conditional statement that adds row of binary values from matrix A to matrix B. I want to put this in a loop so that it continues to add rows from matrix A until matrix B is full. Currently matrix B is initialized as 10 by 10 matrix of zeros. Do I need to initialize matrix B differently in order to create this condition or is there a way of doing it as is?
Below is roughly how my code looks so far
from random import sample
import numpy as np
matrixA = np.random.randint(2, size=(10,10))
matrixB = np.zeros((10,10))
x, y = sample(range(1, 10), k=2)
if someCondition:
matrixB = np.append(matrixB, [matrixA[x]], axis=0)
else:
matrixB = np.append(matrixB, [matrixA[y]], axis=0)
you don't need a loop for it. It is really easy to just do it using smart indexing. For example:
import numpy as np
A = np.random.randint(0, 10, size=(20,10))
B = np.empty((10, 10))
print(A)
# Copy till the row that satisfies your conditions. Here I assume it to be 10
B = A[:10, :]
print(B)
When using tf.boolean_mask(), a Value Error is raised. It reads "Number of mask dimensions must be specified, even if some dimensions are None. E.g. shape=[None] is ok, but shape=None is not.
I suspect that something is going wrong when I create my boolean mask s, because when I just create a boolean mask by hand, all works fine. However, I've checked the shape and the dtype of s so far, and couldn't notice anything suspicious. Both seemed to be identical to the shape and type of the boolean mask I created by hand.
Please see a screenshot of the problem.
The following should allow you to reproduce the error on your machine. You need tensorflow, numpy and scipy.
with tf.Session() as sess:
# receive five embedded vectors
v0 = tf.constant([[3.0,1.0,2.,4.,2.]])
v1 = tf.constant([[4.0,0,1.0,4,1.]])
v2 = tf.constant([[1.0,1.0,0.0,4.,8.]])
v3 = tf.constant([[1.,4,2.,5.,2.]])
v4 = tf.constant([[3.,2.,3.,2.,5.]])
# concatenate the five embedded vectors into a matrix
VT = tf.concat([v0,v1,v2,v3,v4],axis=0)
# perform SVD on the concatenated matrix
s, u1, u2 = tf.svd(VT)
e = tf.square(s) # list of eigenvalues
v = u1 # eigenvectors as column vectors
# sample a set
s = tf.py_func(sample_dpp_bin,[e,v],tf.bool)
X = tf.boolean_mask(VT,s)
print(X.eval())
This is the code to generate s. s is a sample from a determinantal point process (for the mathematically interested).
Note that I'm using tf.py_func to wrap this python function:
import tensorflow as tf
import numpy as np
from scipy.linalg import orth
def sample_dpp_bin(e_val,e_vec):
# e_val = np.array of eigenvalues
# e_vec = array of eigenvectors (= column vectors)
eps = 0.01
# sample a set of eigenvectors
ind = (np.random.rand(len(e_val)) <= (e_val)/(1+e_val))
k = sum(ind)
if k == e_val.size:
return np.ones(e_val.size,dtype=bool) # check for full set
if k == 0:
return np.zeros(e_val.size,dtype=bool)
V = e_vec[:,np.array(ind)]
# sample a set of k items
sample = np.zeros(e_val.size,dtype=bool)
for l in range(k-1,-1,-1):
p = np.sum(V**2,axis=1)
p = np.cumsum(p / np.sum(p)) # item cumulative probabilities
i = int((np.random.rand() <= p).argmax()) # choose random item
sample[i] = True
j = (np.abs(V[i,:])>eps).argmax() # pick an eigenvector not orthogonal to e_i
Vj = V[:,j]
V = orth(V - (np.outer(Vj,(V[i,:]/Vj[i]))))
return sample
The output if I print s and tf.reshape(s) is
[False True True True True]
[5]
The output if I print VT and tf.reshape(VT) is
[[ 3. 1. 2. 4. 2.]
[ 4. 0. 1. 4. 1.]
[ 1. 1. 0. 4. 8.]
[ 1. 4. 2. 5. 2.]
[ 3. 2. 3. 2. 5.]]
[5 5]
Any help much appreciated.
Following example works for me.
import tensorflow as tf
import numpy as np
tensor = [[1, 2], [3, 4], [5, 6]]
mask = np.array([True, False, True])
t_m = tf.boolean_mask(tensor, mask)
sess = tf.Session()
print(sess.run(t_m))
Output:
[[1 2]
[5 6]]
Provide your runnable code snippet to reproduce the error. I think you might be doing something wrong in s.
Update:
s = tf.py_func(sample_dpp_bin,[e,v],tf.bool)
s_v = (s.eval())
X = tf.boolean_mask(VT,s_v)
print(X.eval())
mask should be a np array not TF tensor. You don't have to use tf.pyfunc.
The error message states that the shape of the mask is not defined. What do you get if you print tf.shape(s)? I'd bet the problem with your code is that the shape of s is completely unknown, and you could fix that with a simple call like s.set_shape((None)) (to simply specify that s is a 1-dimensional tensor). Consider this code snippet:
X = np.random.randint(0, 2, (100, 100, 3))
with tf.Session() as sess:
X_tf = tf.placeholder(tf.int8)
# X_tf.set_shape((None, None, None))
y = tf.greater(tf.reduce_max(X_tf, axis=(0, 1)), 0)
print(tf.shape(y))
z = tf.boolean_mask(X_tf, y, axis=2)
print(sess.run(z, feed_dict={X_tf: X}))
This prints a shape of Tensor("Shape_3:0", shape=(?,), dtype=int32) (i.e., even the dimensions of y are unknown) and returns the same error as you have. However, if you uncomment the set_shape line, then X_tf is known to be 3-dimensional and so s is 1-dimensional. The code then works. So, I think all you need to do is add a s.set_shape((None)) call after the py_func call.
I am trying to take a relatively simple limit using sympy:
from sympy import *
f,k,b = symbols('f k b')
test = f**b - k**b
limit(test,k,f)
I am expecting 0, but I am getting:
>>> limit(test,k,f)
f**b - exp(b*log(f))
Mathematically this is correct (and zero), but why doesn't it evaluate to zero?
Note if I define:
from sympy import *
f,k,b = symbols('f k b')
test = exp(b*log(f)) - exp(b*log(k))
limit(test,k,f)
then I do get zero.
It would be incorrect to assert the limit is zero in general. Consider the following computation in Python console:
>>> (-1)**(1/2)
(6.123233995736766e-17+1j)
>>> (-1 - 1e-15j)**(1/2)
(5.053215498074303e-16-1j)
Because of the branch cut of complex square root along the negative real axis, the two extremely close values of the base produce quite different results (the difference is about 2j).
The limit is indeed zero if we stick to positive base and real exponents
from sympy import *
k = symbols('k')
f = symbols('f', positive=True)
b = symbols('b', real=True)
test = f**b - k**b
limit(test,k,f) # returns 0
I have to calculate the standard deviation using loops for a list.
This is what I have:
elif user_option == 2:
stdev= 0
average = 0
for val in scores_list:
diffsquared= (val - average)**2
sum_diffsquared= diffsquared + val
stdev= ((sum_diffsquared)/num_scores)**(1/2)
print(stdev)
I keep on getting the wrong number.
You have a couple of problems in your code. First, "sum_diffsquared" is being calculated outside of the loop. Secondly, you declare the average as 0. You need to calculate the average first.
import numpy
import scipy
stddev = 0
average = 0
scores = [0,1,2,3,4]
average = numpy.mean(scores) #or calculate it yourself
diffsquared = 0
sum_diffsquared = 0
for val in scores:
diffsquared = (val-average)**2
sum_diffsquared = diffsquared + sum_diffsquared
stddev = ((sum_diffsquared)/len(scores))**(1/2) # corrected
print(stddev)
print(numpy.sqrt(numpy.var(scores)))
The output is:
1.41421356237
1.41421356237
Showing that the calculation is now correct.