How I could create a layer in Keras that ouputs matrix given dimensions (e.g. m, n) with cells having a value based on the row and column?
Here is the forumula:
A[i, 2j] = i / (10**(2*j))
A[i, 2j+1] = i / (10**(2*j))
I tried to look on the lamba function but it seems Keras passed only the cell value and not the indices! Any other options (not a loop)
You could do the following:
from keras.layers import Input
import keras.backend as K
import numpy as np
def CustomConstantInput(m, n):
x = np.arange(m)
y = 10 ** (2 * (np.arange(n) // 2))
matrix = x[:, None] / y[None, :]
print(matrix)
fixed_input = K.constant(matrix)
return Input(tensor=fixed_input)
t = CustomConstantInput(3, 4)
Related
try to build an image binary classification model using keras. Unfortunately, get a same output every time. The probability for each test sample was different, but they all favor one label.
The datasets are balanced. label L(n=250) vs. Label E(n=250): 300 for train, 100 for validate, 100 for test. There is no sample overlap among those groups.
After failing to predict the test dataset, I also used the training dataset for prediction which meant the model would make predictions for the samples that had just been trained. I know it does not make any sense. But it also got same output: Counter({0: 300}).
from keras.layers.core import Dense, Flatten, Dropout
from keras.layers.convolutional import Conv2D, MaxPooling2D, SeparableConv2D
import keras
from keras import layers
from skimage.transform import resize
import math
import os,random
import cv2
import numpy as np
import pandas as pd
from keras.models import Sequential
from sklearn.model_selection import KFold
from sklearn.metrics import confusion_matrix
from collections import Counter
import matplotlib.pyplot as plt
class DataGenerator(keras.utils.Sequence):
def __init__(self, datas, batch_size=32, shuffle=True):
self.batch_size = batch_size
self.datas = datas
self.indexes = np.arange(len(self.datas))
self.shuffle = shuffle
def __len__(self):
return math.ceil(len(self.datas) / float(self.batch_size))
def __getitem__(self, index):
batch_indexs = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
batch_datas = [self.datas[k] for k in batch_indexs]
X, y = self.data_generation(batch_datas)
return X, y
def on_epoch_end(self):
if self.shuffle == True:
np.random.shuffle(self.indexes)
def data_generation(self, batch_datas):
images = []
labels = []
for i, data in enumerate(batch_datas):
image = resize((cv2.imread(data)/255),(128, 128))
image = list(image)
images.append(image)
right = data.rfind("\\",0)
left = data.rfind("\\",0,right)+1
class_name = data[left:right]
if class_name=="e":
labels.append(0)
else:
labels.append(1)
return np.array(images), np.array(labels)
def create_model():
model = Sequential()
model.add(Conv2D(8, kernel_size=(3, 3),
input_shape=(128, 128, 3),
activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(16, kernel_size=(3, 3),
padding="same",
activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(32, kernel_size=(3, 3),
activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(units=64, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(units=1, activation='sigmoid'))
model.compile(optimizer='sgd',
loss='binary_crossentropy',
metrics=['accuracy'])
return model
e_train = []
e_test = []
l_test = []
l_train = []
for file in os.listdir('\e\train')
e_train.append(os.path.join('\e\train',file))
for file in os.listdir('\e\test')
e_test.append(os.path.join('\e\test',file))
for file in os.listdir('\l\train')
l_train.append(os.path.join('\l\train',file))
for file in os.listdir('\l\test')
e_test.append(os.path.join('\l\test',file))
data_tr = e_train + l_train
data_te = e_test + l_test
g_te = DataGenerator(data_te)
kf = KFold(n_splits=4, shuffle=True, random_state=seed)
fold = 1
for train, test in kf.split(data_tr):
model = create_model()
g_tr = DataGenerator(data_tr[train])
g_v = DataGenerator(data_tr[test])
H = model.fit_generator(generator = g_tr, epochs=10,
validation_data = g_v, shuffle=False,
max_queue_size=10,workers=1)
pred = model.predict(g_te, max_queue_size=10, workers=1, verbose=1)
print(pred)
# the probability was different, but the right column of probability were always bigger
# [[0.49817565 0.5018243 ]
# [0.4872172 0.5127828 ]
# [0.48092505 0.519075 ]
predicted_class_indices = [np.argmax(probas) for probas in pred]
print(Counter(predicted_class_indices))
# the output was the same
# Counter({0: 100})
fold = fold + 1
Any thoughts would be appreciated.
Solution:
Instead of solving a binary classification problem, convert it into a multi-class problem with two classes. So, the output of the last layer will have a softmax activation which will provide a probability distribution for the classes. Refer this tutorial wherein you'll understand the changes which need to be made.
Explanation
You should not use the sigmoid activation in the output layer of the model, while using relu activation in the intermediate layers. The vanilla ReLU ( Rectified Linear Unit ) activation is defined as,
Hence, the range of the activation function is [ 0 , infinity ). On the other hand, considering a sigmoid activation,
The range of the sigmoid function is ( 0 , 1 ). So, if a large signal ( > 0 ) is passed through the sigmoid function, the output will be very close to 1 i.e. a fully saturated firing. The output of the relu function can provide a large signal from the intermediate layers, hence making a fully saturated firing ( or 1s ) at the output layer where the sigmoid activation is performed.
If the logits are [ 5.6 , 1.2 , 3.2 , 4.8 ], the output of the sigmoid function is,
[0.9963157 , 0.76852477, 0.96083426, 0.99183744]
and that of softmax is,
[0.6441953 , 0.00790901, 0.05844009, 0.2894557 ]
I have a conditional statement that adds row of binary values from matrix A to matrix B. I want to put this in a loop so that it continues to add rows from matrix A until matrix B is full. Currently matrix B is initialized as 10 by 10 matrix of zeros. Do I need to initialize matrix B differently in order to create this condition or is there a way of doing it as is?
Below is roughly how my code looks so far
from random import sample
import numpy as np
matrixA = np.random.randint(2, size=(10,10))
matrixB = np.zeros((10,10))
x, y = sample(range(1, 10), k=2)
if someCondition:
matrixB = np.append(matrixB, [matrixA[x]], axis=0)
else:
matrixB = np.append(matrixB, [matrixA[y]], axis=0)
you don't need a loop for it. It is really easy to just do it using smart indexing. For example:
import numpy as np
A = np.random.randint(0, 10, size=(20,10))
B = np.empty((10, 10))
print(A)
# Copy till the row that satisfies your conditions. Here I assume it to be 10
B = A[:10, :]
print(B)
I want to be able to sample from a multinomial distribution very efficiently and apparently my TensorFlow code is very... very slow...
The idea is that, I have:
A vector: counts = [40, 50, 26, ..., 19] for example
A matrix of probabilities: probs = [[0.1, ..., 0.5], ... [0.3, ..., 0.02]] such that np.sum(probs, axis=1) = 1
Let's say len(counts) = N and len(probs) = (N, 50). What I want to do is (in our example):
sample 40 times from the first probability vector of the matrix probs
sample 50 times from the second probability vector of the matrix probs
...
sample 19 times from the Nth probability vector of the matrix probs
such that my final matrix looks like (for example):
A = [[22, ... 13], ..., [12, ..., 3]] where np.sum(A, axis=1) == counts
(i.e the sum over each row = the number in the corresponding row of counts vector)
Here is my TensorFlow code sample:
import numpy as np
import tensorflow as tf
import tensorflow.contrib.distributions as ds
import time
nb_distribution = 100 # number of probability distributions
counts = np.random.randint(2000, 3500, size=nb_distribution) # define number of counts (vector of size 100 with int in 2000, 3500)
# print(u[:40]) # should be the same as the output of print(np.sum(res, 1)[:40]) in the tf.Session()
# probsn is a matrix of probability:
# each row of probsn contains a vector of size 30 that sums to 1
probsn = np.random.uniform(size=(nb_distribution, 30))
probsn /= np.sum(probsn, axis=1)[:, None]
counts = tf.Variable(counts, dtype=tf.float32)
probs = tf.Variable(tf.convert_to_tensor(probsn.astype(np.float32)))
# sample from the multinomial
dist = ds.Multinomial(total_count=counts, probs=probs)
out = dist.sample()
start = time.time()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
res = sess.run(out)
# print(np.sum(res, 1)[:40])
print(time.time() - start)
elapsed time: 0.12 seconds
My equivalent code in Theano:
import numpy as np
import theano
from theano.tensor import _shared
nb_distribution = 100 # number of probability distributions
counts = np.random.randint(2000, 3500, size=nb_distribution)
#print(u[:40]) # should be the same as the output of print(np.sum(v_sample(), 1)[:40])
counts = _shared(counts) # define number of counts (vector of size 100 with int in 2000, 3500)
# probsn is a matrix of probability:
# each row of probsn contains a vector that sums to 1
probsn = np.random.uniform(size=(nb_distribution, 30))
probsn /= np.sum(probsn, axis=1)[:, None]
probsn = _shared(probsn)
from theano.tensor.shared_randomstreams import RandomStreams
np_rng = np.random.RandomState(12345)
theano_rng = RandomStreams(np_rng.randint(2 ** 30))
v_sample = theano.function(inputs=[], outputs=theano_rng.multinomial(n=counts, pvals=probsn))
start_t = time.time()
out = np.sum(v_sample(), 1)[:40]
# print(out)
print(time.time() - start_t)
elapsed time: 0.0025 seconds
Theano is like 100x faster... Is there something wrong with my TensorFlow code? How can I sample from a multinomial distribution efficiently in TensorFlow?
The problem is that the TensorFlow multinomial sample() method actually uses the method calls _sample_n(). This method is defined here. As we can see in the code to sample from the multinomial the code produces a matrix of one_hot for each row and then reduce the matrix into a vector by summing over the rows:
math_ops.reduce_sum(array_ops.one_hot(x, depth=k), axis=-2)
It is inefficient because it uses extra memory. To avoid this I have used the
tf.scatter_nd function. Here is a fully runnable example:
import tensorflow as tf
import numpy as np
import tensorflow.contrib.distributions as ds
import time
tf.reset_default_graph()
nb_distribution = 100 # number of probabilities distribution
u = np.random.randint(2000, 3500, size=nb_distribution) # define number of counts (vector of size 100 with int in 2000, 3500)
# probsn is a matrix of probability:
# each row of probsn contains a vector of size 30 that sums to 1
probsn = np.random.uniform(size=(nb_distribution, 30))
probsn /= np.sum(probsn, axis=1)[:, None]
counts = tf.Variable(u, dtype=tf.float32)
probs = tf.Variable(tf.convert_to_tensor(probsn.astype(np.float32)))
# sample from the multinomial
dist = ds.Multinomial(total_count=counts, probs=probs)
out = dist.sample()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
res = sess.run(out) # if remove this line the code is slower...
start = time.time()
res = sess.run(out)
print(time.time() - start)
print(np.all(u == np.sum(res, axis=1)))
This code took 0.05 seconds to compute
def vmultinomial_sampling(counts, pvals, seed=None):
k = tf.shape(pvals)[1]
logits = tf.expand_dims(tf.log(pvals), 1)
def sample_single(args):
logits_, n_draw_ = args[0], args[1]
x = tf.multinomial(logits_, n_draw_, seed)
indices = tf.cast(tf.reshape(x, [-1,1]), tf.int32)
updates = tf.ones(n_draw_) # tf.shape(indices)[0]
return tf.scatter_nd(indices, updates, [k])
x = tf.map_fn(sample_single, [logits, counts], dtype=tf.float32)
return x
xx = vmultinomial_sampling(u, probsn)
# check = tf.expand_dims(counts, 1) * probs
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
res = sess.run(xx) # if remove this line the code is slower...
start_t = time.time()
res = sess.run(xx)
print(time.time() -start_t)
#print(np.sum(res, axis=1))
print(np.all(u == np.sum(res, axis=1)))
This code took 0.016 seconds
The drawback is that my code doesn't actually parallelize the computation (even though parallel_iterations parameter is set to 10 by default in map_fn, putting it to 1 doesn't change anything...)
Maybe someone will find something better because it is still very slow as compare to Theano's implementation (due to the fact that it doesn't take advantage of the parallelization... and yet, here, parallelization makes sense because sampling one row is indenpendent from sampling another one...)
I wonder if anyone knows some python or java code to calculate 1D median filter.
I have a file comma delimited with two fields: Date and Signal.
Something like that:
2014-06-01 11:22:12, 23.8
2014-06-01 11:23:12, 25.9
2014-06-01 11:24:12, 45.7
I would like to read this file and apply the 1D Median Filter with size 23
for the field Signal and save it in another file to remove the noise.
Thanks in advance.
Alexandre.
In case someone stumbled on this later.
To extract the data you can use regex, while for the custom median filter you can have a look here.
I will leave a copy down here in case it is removed:
def medfilt (x, k):
"""Apply a length-k median filter to a 1D array x.
Boundaries are extended by repeating endpoints.
"""
assert k % 2 == 1, "Median filter length must be odd."
assert x.ndim == 1, "Input must be one-dimensional."
k2 = (k - 1) // 2
y = np.zeros ((len (x), k), dtype=x.dtype)
y[:,k2] = x
for i in range (k2):
j = k2 - i
y[j:,i] = x[:-j]
y[:j,i] = x[0]
y[:-j,-(i+1)] = x[j:]
y[-j:,-(i+1)] = x[-1]
return np.median (y, axis=1)
scipy.signal.medfilt accepts 1D kernels:
import pandas as pd
import scipy.signal
def median_filter(file_name, new_file_name, kernel_size):
with open(file_name, 'r') as f:
df = pd.read_csv(f, header=None)
signal = df.iloc[:, 1].values
median = scipy.signal.medfilt(signal, kernel_size)
df = df.drop(df.columns[1], 1)
df[1] = median
df.to_csv(new_file_name, sep=',', index=None, header=None)
if __name__=='__main__':
median_filter('old_signal.csv', 'new_signal.csv', 23)
I need to create a new (n*m) x 4 matrix (e.g. named b) from a n x m matrix (e.g. named a), but I don't want to use nested loops for speed reasons. Here how I would do it with a nested loop:
for j in xrange(1,m+1):
for i in xrange(1,n+1):
index = (j-1)*n+i
b[index,1] = a[i,j]
b[index,2] = index
b[index,3] = s1*i+s2*j+s3
b[index,4] = s4*i+s5*j+s6
The question is, therefore, how to create a new matrix with values derived from original matrix indexes? Thanks
If you can use numpy, you could try
import numpy as np
# Create an empty array
b = np.empty((np.multiply(*a.shape), 4), dtype=a.dtype)
# Get two (nxm) of indices
(irows, icols) = np.indices(a)
# Fill the b array
b[...,0] = a.flat
b[...,1] = np.arange(a.size)
b[...,2] = (s1*irows + s2*icols + s3).flat
b[...,3] = (s4*irows + s5*icols + s6).flat
some minor corrections (could not post as comment :/ ) for those who will have a similar question:
import numpy as np
# Create an empty array
b = np.empty((a.size, 4), dtype=a.dtype)
# Get two (nxm) of indices (starting from 1)
(irows, icols) = np.indices(a.shape) + 1
# Fill the b array
b[...,0] = a.flat
b[...,1] = np.arange(a.size) + 1
b[...,2] = (s1*irows + s2*icols + s3).flat
b[...,3] = (s4*irows + s5*icols + s6).flat