Related
how to convert Flux<List> into Flux<int[][]>.
I have a Flux<List> -> {1,2,3,.....100} I want to group them by 30 numbers -> [[1,2,3,.....30], [31,32....60],[61.....100]]
I have tried the below approach but was not successful. elements are getting grouped in batches of 5 [ [1,2,3,4,5], [6,7,8,9,10],.....]
Flux<int[][]> groupedData = fluxData.map(x -> {
int outerArraySize = (int) Math.ceil(x.size() / 30) +1;
System.out.println(outerArraySize);
int[][] boxedData = new int[30][outerArraySize];
AtomicInteger innerArray = new AtomicInteger(0);
AtomicInteger outerArray = new AtomicInteger(0);
x.forEach(ids -> {
boxedData[innerArray.get()][outerArray.get()] = ids;
innerArray.getAndIncrement();
if (innerArray.get() == 30) {
innerArray.set(0);
outerArray.getAndIncrement();
}
});
Flux has a useful operator called 'buffer', we can use it to batch the List of Integers.
I have created my 'fluxData' like this, just so I can test my code:
List<Integer> list1 = IntStream.rangeClosed(1, 100).boxed().collect(Collectors.toList());
List<Integer> list2 = IntStream.rangeClosed(1, 40).boxed().collect(Collectors.toList());
List<Integer> list3 = IntStream.rangeClosed(1, 70).boxed().collect(Collectors.toList());
Flux<List<Integer>> fluxData = Flux.just(list1, list2, list3);
Now, we can do the following:
fluxData.map(integersList -> {
List<List<Integer>> batchesList = Flux.fromStream(integersList.stream())
.buffer(30) // This the magic.
.collectList()
.block();
// List<List<Integer>> --> int[][]
int[][] batchesArray = new int[batchesList.size()][];
for(int i = 0;i < batchesArray.length;i++){
batchesArray[i] = new int[batchesList.get(i).size()];
for (int j = 0; j < batchesArray[i].length; j++) {
batchesArray[i][j] = batchesList.get(i).get(j);
}
}
return batchesArray;
})
.subscribe(batchesArray -> System.out.println(Arrays.deepToString(batchesArray)));
Output:
[[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], [31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60], [61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90], [91, 92, 93, 94, 95, 96, 97, 98, 99, 100]]
[[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], [31, 32, 33, 34, 35, 36, 37, 38, 39, 40]]
[[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30], [31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60], [61, 62, 63, 64, 65, 66, 67, 68, 69, 70]]
Running a simple Bayesian regression model, I am not able to replicate the results with multiple runs on GPU. I am wondering how I can set tfp.mcmc.sample_chain to generate reproducible results on GPU? Seeding the sample_chain didn't work for me.
The test code snippet:
import os
import random
from pprint import pprint
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
import pandas as pd
import tensorflow.compat.v2 as tf
tf.enable_v2_behavior()
import tensorflow_probability as tfp
sns.reset_defaults()
#sns.set_style('whitegrid')
#sns.set_context('talk')
sns.set_context(context='talk',font_scale=0.7)
%config InlineBackend.figure_format = 'retina'
%matplotlib inline
tfd = tfp.distributions
tfb = tfp.bijectors
dtype = tf.float64
dfhogg = pd.DataFrame(np.array([[1, 201, 592, 61, 9, -0.84],
[2, 244, 401, 25, 4, 0.31],
[3, 47, 583, 38, 11, 0.64],
[4, 287, 402, 15, 7, -0.27],
[5, 203, 495, 21, 5, -0.33],
[6, 58, 173, 15, 9, 0.67],
[7, 210, 479, 27, 4, -0.02],
[8, 202, 504, 14, 4, -0.05],
[9, 198, 510, 30, 11, -0.84],
[10, 158, 416, 16, 7, -0.69],
[11, 165, 393, 14, 5, 0.30],
[12, 201, 442, 25, 5, -0.46],
[13, 157, 317, 52, 5, -0.03],
[14, 131, 311, 16, 6, 0.50],
[15, 166, 400, 34, 6, 0.73],
[16, 160, 337, 31, 5, -0.52],
[17, 186, 423, 42, 9, 0.90],
[18, 125, 334, 26, 8, 0.40],
[19, 218, 533, 16, 6, -0.78],
[20, 146, 344, 22, 5, -0.56]]),
columns=['id','x','y','sigma_y','sigma_x','rho_xy'])
## for convenience zero-base the 'id' and use as index
dfhogg['id'] = dfhogg['id'] - 1
dfhogg.set_index('id', inplace=True)
## standardize (mean center and divide by 1 sd)
dfhoggs = (dfhogg[['x','y']] - dfhogg[['x','y']].mean(0)) / dfhogg[['x','y']].std(0)
dfhoggs['sigma_y'] = dfhogg['sigma_y'] / dfhogg['y'].std(0)
dfhoggs['sigma_x'] = dfhogg['sigma_x'] / dfhogg['x'].std(0)
X_np = dfhoggs['x'].values
sigma_y_np = dfhoggs['sigma_y'].values
Y_np = dfhoggs['y'].values
def sample(seed):
mdl_ols_batch = tfd.JointDistributionSequential([
# b0
tfd.Normal(loc=tf.cast(0, dtype), scale=1.),
# b1
tfd.Normal(loc=tf.cast(0, dtype), scale=1.),
# likelihood
# Using Independent to ensure the log_prob is not incorrectly broadcasted
lambda b1, b0: tfd.Independent(
tfd.Normal(
# Parameter transformation
loc=b0[..., tf.newaxis] + b1[..., tf.newaxis]*X_np[tf.newaxis, ...],
scale=sigma_y_np[tf.newaxis, ...]),
reinterpreted_batch_ndims=1
),
])
#tf.function(autograph=False, experimental_compile=True)
def run_chain(init_state,
step_size,
target_log_prob_fn,
unconstraining_bijectors,
num_steps=500,
burnin=50):
def trace_fn(_, pkr):
return (
pkr.inner_results.inner_results.target_log_prob,
pkr.inner_results.inner_results.leapfrogs_taken,
pkr.inner_results.inner_results.has_divergence,
pkr.inner_results.inner_results.energy,
pkr.inner_results.inner_results.log_accept_ratio
)
kernel = tfp.mcmc.TransformedTransitionKernel(
inner_kernel=tfp.mcmc.NoUTurnSampler(
target_log_prob_fn,
step_size=step_size),
bijector=unconstraining_bijectors)
hmc = tfp.mcmc.DualAveragingStepSizeAdaptation(
inner_kernel=kernel,
num_adaptation_steps=burnin,
step_size_setter_fn=lambda pkr, new_step_size: pkr._replace(
inner_results=pkr.inner_results._replace(step_size=new_step_size)),
step_size_getter_fn=lambda pkr: pkr.inner_results.step_size,
log_accept_prob_getter_fn=lambda pkr: pkr.inner_results.log_accept_ratio
)
# Sampling from the chain.
chain_state, sampler_stat = tfp.mcmc.sample_chain(
num_results=num_steps,
num_burnin_steps=burnin,
current_state=init_state,
kernel=hmc,
trace_fn=trace_fn,
seed=seed
)
return chain_state, sampler_stat
nchain = 4
b0, b1, _ = mdl_ols_batch.sample(nchain)
init_state = [b0, b1]
step_size = [tf.cast(i, dtype=dtype) for i in [.1, .1]]
target_log_prob_fn = lambda *x: mdl_ols_batch.log_prob(x + (Y_np, ))
# bijector to map contrained parameters to real
unconstraining_bijectors = [
tfb.Identity(),
tfb.Identity(),
]
samples, sampler_stat = run_chain(
init_state, step_size, target_log_prob_fn, unconstraining_bijectors)
print(tf.reduce_sum(samples))
seed = 24
os.environ['TF_DETERMINISTIC_OPS'] = 'true'
os.environ['PYTHONHASHSEED'] = f'{seed}'
np.random.seed(seed)
random.seed(seed)
tf.random.set_seed(seed)
sample(seed)
os.environ['TF_DETERMINISTIC_OPS'] = 'true'
os.environ['PYTHONHASHSEED'] = f'{seed}'
np.random.seed(seed)
random.seed(seed)
tf.random.set_seed(seed)
sample(seed)
Hello I am having trouble encrypting using an array as the key and the value with the ruby-mcrypt gem. The gem lets me use an array for the key fine, cipher = Mcrypt.new("rijndael-256", :ecb, secret) works. But it will give me an error when I try to encrypt. I've tried many things but no luck. Does anyone know if Mcrypt just doesn't like encrypting with an array?
require 'mcrypt'
def encrypt(plain, secret)
cipher = Mcrypt.new("rijndael-256", :ecb, secret)
cipher.padding = :zeros
encrypted = cipher.encrypt(plain)
p encrypted
encrypted.unpack("H*").first.to_s.upcase
end
array_to_encrypt = [16, 0, 0, 0, 50, 48, 49, 55, 47, 48, 50, 47, 48, 55, 32, 50, 50, 58, 52, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
key_array = [65, 66, 67, 68, 49, 50, 51, 52, 70, 71, 72, 73, 53, 54, 55, 56]
result = encrypt(array_to_encrypt, key_array)
p "RESULT IS #{result}"
The output is as follows:
Mcrypt::RuntimeError: Could not initialize mcrypt: Key length is not legal.
I traced this error to here in the ruby-mcrypt gem but don't understand it enough to figure out why I am getting the error message. Any help or insights would be amazing. Thanks!
The library doesn't support arrays. You'll need to use Strings instead:
def binary(byte_array)
byte_array.pack('C*')
end
array_to_encrypt = [16, 0, 0, 0, 50, 48, 49, 55, 47, 48, 50, 47, 48, 55, 32, 50, 50, 58, 52, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
key_array = [65, 66, 67, 68, 49, 50, 51, 52, 70, 71, 72, 73, 53, 54, 55, 56]
result = encrypt(binary(array_to_encrypt), binary(key_array))
p "RESULT IS #{result}"
What would be the idiomatic way of extracting a submatrix from a matrix in ruby.
I have a matrix like, this is an object of Matrix
[131, 673, 234, 103, 18]
[201, 96, 342, 965, 150]
[630, 803, 746, 422, 111]
[537, 699, 497, 121, 956]
[805, 732, 524, 37, 331]
I'm looking for a method with a signature like
matrix.submatrix(1,1) this should return
[96, 342, 965, 150]
[803, 746, 422, 111]
[699, 497, 121, 956]
[732, 524, 37, 331]
matrix.submatrix(2,2) would return
[746, 422, 111]
[497, 121, 956]
[524, 37, 331]
I browsed through the rubydoc but couldn't find any method that would give me what I wanted. How would I do this in ruby?
For a 2D array I have come up with
def submatrix(matrix)
submatrix = matrix.collect{|row| row.slice(1..-1)}
# Pop off the first row
submatrix[1..-1]
end
I am wondering if I should reinvent the wheel or could I use something from the Matrix class.
Take a look at Matrix#minor:
a = [[131, 673, 234, 103, 18],
[201, 96, 342, 965, 150],
[630, 803, 746, 422, 111],
[537, 699, 497, 121, 956],
[805, 732, 524, 37, 331]]
m = Matrix[*a]
m1 = m.minor(1..4, 1..4)
=> Matrix[[96, 342, 965, 150], [803, 746, 422, 111],
[699, 497, 121, 956], [732, 524, 37, 331]]
m2 = m1.minor(1..3, 1..3)
=> Matrix[[746, 422, 111], [497, 121, 956], [524, 37, 331]]
You can also do:
m1 = m.minor(1..-1, 1..-1)
m2 = m1.minor(1..-1, 1..-1)
Or:
class Matrix
def submatrix(x, y)
self.minor(x..-1, y..-1)
end
end
m.submatrix(2, 2)
=> Matrix[[746, 422, 111], [497, 121, 956], [524, 37, 331]]
If you are using Ruby 2.2.0 or later, you can use Matrix#first_minor, which removes a specified row and column. I'm not sure how efficient it is, but here is some code that solves your problem:
require 'matrix'
def my_submatrix(matrix, n)
matrix = matrix.first_minor(0, 0) while matrix.row_count > n
matrix
end
m = Matrix[[131, 673, 234, 103, 18],
[201, 96, 342, 965, 150],
[630, 803, 746, 422, 111],
[537, 699, 497, 121, 956],
[805, 732, 524, 37, 331]]
p my_submatrix(m, 3)
# => Matrix[[746, 422, 111], [497, 121, 956], [524, 37, 331]]
Here are a couple of ways, one using a Matrix object, the other just manipulating an array.
Manipulating a Matrix object
Code
require 'matrix'
def doit(matrix,i,j)
selection_matrix(matrix.row_count,i) * matrix *
selection_matrix(matrix.row_size,j)
end
def selection_matrix(n,m)
Matrix.diagonal(*(0...n).map { |i| (i<m) ? 0 : 1 })
end
Use doit(matrix,i,j).to_a to return an Array object.
Examples
a = [[131, 673, 234, 103, 18],
[201, 96, 342, 965, 150],
[630, 803, 746, 422, 111],
[537, 699, 497, 121, 956],
[805, 732, 524, 37, 331]]
matrix = Matrix[*a]
doit(matrix,2,2)
#=> Matrix[[0, 0, 0, 0, 0],
# [0, 0, 0, 0, 0],
# [0, 0, 746, 422, 111],
# [0, 0, 497, 121, 956],
# [0, 0, 524, 37, 331]]
doit(matrix,1,1)
#=> Matrix[[0, 0, 0, 0, 0],
# [0, 96, 342, 965, 150],
# [0, 803, 746, 422, 111],
# [0, 699, 497, 121, 956],
# [0, 732, 524, 37, 331]]
Explanation
selection_matrix(n,m) returns a diagonal matrix whose diagonal elements are ones and zeros, the ones stabing out the appropriate rows or columns of the matrix. The matrix is pre-multiplied (post-multiplied) by a diagonal matrix whose order equals the number of rows (columns) of the matrix.
selection_matrix(5,2)
#=> Matrix[[0, 0, 0, 0, 0],
# [0, 0, 0, 0, 0],
# [0, 0, 1, 0, 0],
# [0, 0, 0, 1, 0],
# [0, 0, 0, 0, 1]]
a = selection_matrix(5,2) * matrix
#=> Matrix[[ 0, 0, 0, 0, 0],
# [ 0, 0, 0, 0, 0],
# [630, 803, 746, 422, 111],
# [537, 699, 497, 121, 956],
# [805, 732, 524, 37, 331]]
b = a * selection_matrix(5,2)
#=> Matrix[[0, 0, 0, 0, 0],
# [0, 0, 0, 0, 0],
# [0, 0, 746, 422, 111],
# [0, 0, 497, 121, 956],
# [0, 0, 524, 37, 331]]
b.to_a
#=> [[0, 0, 0, 0, 0],
# [0, 0, 0, 0, 0],
# [0, 0, 746, 422, 111],
# [0, 0, 497, 121, 956],
# [0, 0, 524, 37, 331]]
Manipulating an Array object
Without creating a Matrix object, you could just do this:
Code
def doit(a,i,j)
a[i..-1].transpose[j..-1].transpose
end
Examples
a = [[131, 673, 234, 103, 18],
[201, 96, 342, 965, 150],
[630, 803, 746, 422, 111],
[537, 699, 497, 121, 956],
[805, 732, 524, 37, 331]]
doit(a,1,1)
#=> [[ 96, 342, 965, 150],
# [803, 746, 422, 111],
# [699, 497, 121, 956],
# [732, 524, 37, 331]]
doit(a,2,2)
#=> [[746, 422, 111],
# [497, 121, 956],
# [524, 37, 331]]
This is just a curiosity - I don't have a real question.
The output of AbsoluteTiming has a definite pattern; can anyone confirm/explain ?
xxx = RandomReal[NormalDistribution[0, 1], 10^6];
Sin[#] & /# xxx; // AbsoluteTiming
(* {0.0890089, Null} *)
Max[Exp[#] - 0.5, 0] & /# xxx; // AbsoluteTiming
(* {0.1560156, Null} *)
$Version
8.0 for Microsoft Windows (64-bit) (February 23, 2011)
According to the Documentation, "AbsoluteTiming is always accurate down to a granularity of $TimeUnit seconds, but on many systems is much more accurate." So evaluating $TimeUnit probably can elucidate this issue.
Yep. Let´s check if the time quantum is consistent:
Differences#
Round[10^5 Sort#
Union[AbsoluteTiming[
Sin[#] & /#
RandomReal[NormalDistribution[0, 1], #];][[1]] & /#
RandomInteger[10^6, 100]]]
(*
-> {1562, 1563, 1563, 1562, 1562, 1563, 1563, 1562, 1562, 1563, 1563, \
1562, 1562, 1563, 1563, 1562, 1562}
*)
Edit
Better code
Differences#
Sort#Union[
Round[10^5 AbsoluteTiming[
Sin[#] & /#
RandomReal[NormalDistribution[0, 1], #];][[1]] & /#
RandomInteger[10^6, 100]]]
Presumably your system's clock only has granularity to some fraction of a second that happens to produce a repeating decimal. I have never noticed this on my Macs.
It's cool, though.
EDIT
Now that I am home I can confirm this must be system-specific: here is my output from the code in belisarius's answer:
{56, 119, 28, 25, 33, 397, 35, 82, 185, 67, 41, 67, 218, 192, 115, \
28, 74, 16, 187, 222, 194, 8, 129, 399, 68, 75, 71, 34, 5, 37, 62, \
64, 137, 173, 24, 98, 135, 308, 63, 155, 208, 861, 22, 72, 72, 184, \
609, 564, 112, 1011, 118, 81, 158, 90, 351, 33, 35, 68, 10, 126, 39, \
194, 7, 108, 278, 75, 37, 214, 34, 166, 119, 10, 335, 141, 4, 988, \
90, 121, 71, 130, 117, 186, 33, 123, 111, 110, 57, 64, 213, 217, 210, \
204, 98, 247, 20, 1421, 28, 2003, 353}