I need to understand the following error: "CatBoostError: The left argument is not fitted, only fitted models could be compared." - performance

I am trying to run a RandomizedSearchCV on various classification models through "For" loop for Hyperparameter tuning. There is no issue with running any other models except CatBoost. Also the issue with Catboost arises when I used Pipeline in defining function.
My Code:
#Building the models:
lr = LogisticRegression()
knn = KNeighborsClassifier()
svm = SVC()
dt = DecisionTreeClassifier(random_state=1)
bag = BaggingClassifier(random_state=1)
adb = AdaBoostClassifier(random_state=1)
gb = GradientBoostingClassifier(random_state=1)
rf = RandomForestClassifier(random_state=1)
xgb = XGBClassifier()
cgb = CatBoostClassifier()
lgb = LGBMClassifier()
#Defining a function:
def fun_exp(model, name, x_tr, x_te, y_tr, y_te):
start = time.time()
pipe = Pipeline([('scale', StandardScaler()), ('pca', PCA(n_components = 62)), (name, model)])
rscv = RandomizedSearchCV(pipe, params, cv=10, random_state=1)
rscv.fit(x_tr, y_tr)
rscv_best_params = rscv.best_params_
rscv_best_score = rscv.best_score_
rscv_score_train = rscv.score(x_tr, y_tr)
rscv_score_test = rscv.score(x_te, y_te)
rscv_pred = rscv.predict(x_te)
end = time.time()
pickle.dump(rscv, open(name, 'wb'))
rscv_duration = end-start
return rscv_best_params, rscv_best_score, rscv_score_train, rscv_score_test, rscv_duration, rscv_pred
Running the above function in for loop & saving the result in a dictionary:
exp_result = {}
\#Fitting & Testing the model
for model, name in zip([lr, knn, svm, dt, bag, adb, gb, rf, lgb, cgb, xgb], ['Logistic Regression', 'KNeighbors', 'SVM', 'DecisionTree', 'Bagging', 'AdaBoost', 'GradientBoost', 'Random Forest', 'LightGBM', 'CatBoost', 'XGBoost']):
if model == lr:
params = {'Logistic Regression__solver': ['liblinear', 'lfbgs', 'sag', 'saga'], 'Logistic Regression__penalty':['elasticnet', 'l1', 'l2', 'none'], 'Logistic Regression__multi_class': ['auto', 'ovr', 'multinomial'], 'Logistic Regression__C':[0.1, 1, 10], 'Logistic Regression__tol': [0.00001, 0.0001, 0.001], 'Logistic Regression__class_weight': ['balanced', None]}
if model == knn:
params = {'KNeighbors__n_neighbors':np.arange(5,50,5), 'KNeighbors__weights': ['uniform', 'distance'], 'KNeighbors__algorithm':['auto', 'knn__ball_tree', 'kd_tree', 'brute'], 'KNeighbors__leaf_size': np.arange(10,51,10), 'KNeighbors__metric': ['minkowski', 'euclidean', 'manhattan']}
if model == svm:
params = {'SVM__gamma': [10, 1, 0.1, 0.01, 0.001, 0.0001], 'SVM__C': [1000, 100, 10, 1, 0.1, 0.01, 0.001], 'SVM__kernel': ['poly', 'rbf', 'sigmoid'], 'SVM__class_weight': ['balanced', None], 'SVM__decision_function_shape': ['ovo', 'ovr']}
if model == dt:
params = {'DecisionTree__criterion':['gini', 'entropy', 'log_loss'], 'DecisionTree__splitter':['best', 'random'], 'DecisionTree__max_depth':[None, np.arange(1,11)], 'DecisionTree__max_features': np.arange(8, 21, 2), 'DecisionTree__random_state':[1], 'DecisionTree__class_weight':['balanced', None]}
if model==bag:
params = {'Bagging__n_estimators': [10, 30, 50, 100, 500], 'Bagging__max_features': np.arange(8, 21, 2), 'Bagging__random_state':[1]}
if model == adb:
params = {'AdaBoost__n_estimators': [10, 30, 50, 100, 500], 'AdaBoost__learning_rate':[0.001, 0.01, 0.1, 1, 10], 'AdaBoost__algorithm':['SAMME.R', 'SAMME'], 'AdaBoost__random_state':[1]}
if model == gb:
params = {'GradientBoost__loss':['log_loss', 'exponential'], 'GradientBoost__learning_rate':[0.001, 0.01, 0.1, 1, 10], 'GradientBoost__n_estimators': [10, 30, 50, 100, 500], 'GradientBoost__max_depth':np.arange(1,11), 'GradientBoost__random_state':[1], 'GradientBoost__max_features': np.arange(8, 21, 2)}
if model == rf:
params = {'Random Forest__n_estimators': [10, 30, 50, 100, 500], 'Random Forest__criterion':['gini', 'entropy', 'log_loss'], 'Random Forest__max_depth':np.arange(1,11), 'Random Forest__max_features': np.arange(8, 21, 2), 'Random Forest__random_state':[1]}
if model == lgb:
params = {'LightGBM__boosting_type':['gbdt', 'rf'], 'LightGBM__num_leaves':np.arange(20, 40), 'LightGBM__max_depth':np.arange(1,11), 'LightGBM__learning_rate':[0.001, 0.01, 0.1, 1, 10], 'LightGBM__n_estimators': [10, 30, 50, 100, 500], 'LightGBM__class_weight': ['balanced', None], 'LightGBM__random_state':[1]}
if model == cgb:
params = {'CatBoost__learning_rate':[0.001, 0.01, 0.1, 1], 'CatBoost__n_estimators': [100, 500], 'CatBoost__max_depth':np.arange(1,11), 'CatBoost__random_state':[1], 'CatBoost__feature_border_type': ['Median', 'Uniform', 'UniformAndQuantiles', 'GreedyLogSum', 'MaxLogSum', 'MinEntropy']}
if model == xgb:
le = LabelEncoder()
y_tr = le.fit_transform(y_tr)
y_te = le.fit_transform(y_te)
params = {'XGBoost__n_estimators': [10, 30, 50, 100, 500], 'XGBoost__max_depth':np.arange(1,11), 'XGBoost__max_leaves': np.arange(0, 150), 'XGBoost__learning_rate':[0.001, 0.01, 0.1, 1, 10], 'XGBoost__random_state':[1]}
exp_result[name] = fun_exp(model, name, x_tr, x_te, y_tr, y_te)

Related

How to get reproducible samples from Tensorflow Probability: tfp.mcmc.sample_chain?

Running a simple Bayesian regression model, I am not able to replicate the results with multiple runs on GPU. I am wondering how I can set tfp.mcmc.sample_chain to generate reproducible results on GPU? Seeding the sample_chain didn't work for me.
The test code snippet:
import os
import random
from pprint import pprint
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
import pandas as pd
import tensorflow.compat.v2 as tf
tf.enable_v2_behavior()
import tensorflow_probability as tfp
sns.reset_defaults()
#sns.set_style('whitegrid')
#sns.set_context('talk')
sns.set_context(context='talk',font_scale=0.7)
%config InlineBackend.figure_format = 'retina'
%matplotlib inline
tfd = tfp.distributions
tfb = tfp.bijectors
dtype = tf.float64
dfhogg = pd.DataFrame(np.array([[1, 201, 592, 61, 9, -0.84],
[2, 244, 401, 25, 4, 0.31],
[3, 47, 583, 38, 11, 0.64],
[4, 287, 402, 15, 7, -0.27],
[5, 203, 495, 21, 5, -0.33],
[6, 58, 173, 15, 9, 0.67],
[7, 210, 479, 27, 4, -0.02],
[8, 202, 504, 14, 4, -0.05],
[9, 198, 510, 30, 11, -0.84],
[10, 158, 416, 16, 7, -0.69],
[11, 165, 393, 14, 5, 0.30],
[12, 201, 442, 25, 5, -0.46],
[13, 157, 317, 52, 5, -0.03],
[14, 131, 311, 16, 6, 0.50],
[15, 166, 400, 34, 6, 0.73],
[16, 160, 337, 31, 5, -0.52],
[17, 186, 423, 42, 9, 0.90],
[18, 125, 334, 26, 8, 0.40],
[19, 218, 533, 16, 6, -0.78],
[20, 146, 344, 22, 5, -0.56]]),
columns=['id','x','y','sigma_y','sigma_x','rho_xy'])
## for convenience zero-base the 'id' and use as index
dfhogg['id'] = dfhogg['id'] - 1
dfhogg.set_index('id', inplace=True)
## standardize (mean center and divide by 1 sd)
dfhoggs = (dfhogg[['x','y']] - dfhogg[['x','y']].mean(0)) / dfhogg[['x','y']].std(0)
dfhoggs['sigma_y'] = dfhogg['sigma_y'] / dfhogg['y'].std(0)
dfhoggs['sigma_x'] = dfhogg['sigma_x'] / dfhogg['x'].std(0)
X_np = dfhoggs['x'].values
sigma_y_np = dfhoggs['sigma_y'].values
Y_np = dfhoggs['y'].values
def sample(seed):
mdl_ols_batch = tfd.JointDistributionSequential([
# b0
tfd.Normal(loc=tf.cast(0, dtype), scale=1.),
# b1
tfd.Normal(loc=tf.cast(0, dtype), scale=1.),
# likelihood
# Using Independent to ensure the log_prob is not incorrectly broadcasted
lambda b1, b0: tfd.Independent(
tfd.Normal(
# Parameter transformation
loc=b0[..., tf.newaxis] + b1[..., tf.newaxis]*X_np[tf.newaxis, ...],
scale=sigma_y_np[tf.newaxis, ...]),
reinterpreted_batch_ndims=1
),
])
#tf.function(autograph=False, experimental_compile=True)
def run_chain(init_state,
step_size,
target_log_prob_fn,
unconstraining_bijectors,
num_steps=500,
burnin=50):
def trace_fn(_, pkr):
return (
pkr.inner_results.inner_results.target_log_prob,
pkr.inner_results.inner_results.leapfrogs_taken,
pkr.inner_results.inner_results.has_divergence,
pkr.inner_results.inner_results.energy,
pkr.inner_results.inner_results.log_accept_ratio
)
kernel = tfp.mcmc.TransformedTransitionKernel(
inner_kernel=tfp.mcmc.NoUTurnSampler(
target_log_prob_fn,
step_size=step_size),
bijector=unconstraining_bijectors)
hmc = tfp.mcmc.DualAveragingStepSizeAdaptation(
inner_kernel=kernel,
num_adaptation_steps=burnin,
step_size_setter_fn=lambda pkr, new_step_size: pkr._replace(
inner_results=pkr.inner_results._replace(step_size=new_step_size)),
step_size_getter_fn=lambda pkr: pkr.inner_results.step_size,
log_accept_prob_getter_fn=lambda pkr: pkr.inner_results.log_accept_ratio
)
# Sampling from the chain.
chain_state, sampler_stat = tfp.mcmc.sample_chain(
num_results=num_steps,
num_burnin_steps=burnin,
current_state=init_state,
kernel=hmc,
trace_fn=trace_fn,
seed=seed
)
return chain_state, sampler_stat
nchain = 4
b0, b1, _ = mdl_ols_batch.sample(nchain)
init_state = [b0, b1]
step_size = [tf.cast(i, dtype=dtype) for i in [.1, .1]]
target_log_prob_fn = lambda *x: mdl_ols_batch.log_prob(x + (Y_np, ))
# bijector to map contrained parameters to real
unconstraining_bijectors = [
tfb.Identity(),
tfb.Identity(),
]
samples, sampler_stat = run_chain(
init_state, step_size, target_log_prob_fn, unconstraining_bijectors)
print(tf.reduce_sum(samples))
seed = 24
os.environ['TF_DETERMINISTIC_OPS'] = 'true'
os.environ['PYTHONHASHSEED'] = f'{seed}'
np.random.seed(seed)
random.seed(seed)
tf.random.set_seed(seed)
sample(seed)
os.environ['TF_DETERMINISTIC_OPS'] = 'true'
os.environ['PYTHONHASHSEED'] = f'{seed}'
np.random.seed(seed)
random.seed(seed)
tf.random.set_seed(seed)
sample(seed)

D3 Chord Diagram Not Rendering Correctly

Using the excellent guide by Nadieh Bremer I'm making a stretched chord diagram.
However, with certain data inputs the rendering goes awry.
I've made a demo to demonstrate my issue here:
https://codepen.io/benmayocode/pen/MPEwdr
Specifically, in the .js file lines 269 to 281 file I have:
var respondents = 40,
emptyPerc = 0.4,
emptyStroke = Math.round(respondents*emptyPerc);
var Names = ['BEN', 'ROSE', '', '1', '2', '6', ''];
var matrix = [
[0, 0, 0, 10, 10, 0, 0] ,
[0, 0, 0, 0, 10, 10, 0] ,
[0, 0, 0, 0, 0, 0, 24] ,
[10, 0, 0, 0, 0, 0, 0] ,
[10, 10, 0, 0, 0, 0, 0] ,
[0, 10, 0, 0, 0, 0, 0] ,
[0, 0, 0, 24, 0, 0, 0] ,
];
This renders incorrectly - but if I change it to...
var respondents = 40,
emptyPerc = 0.4,
emptyStroke = Math.round(respondents*emptyPerc);
var Names = ['BEN', 'LIB', 'ROSE', '', '1', '2', '6', ''];
var matrix = [
[0, 0, 0, 0, 10, 10, 0, 0] ,
[0, 0, 0, 0, 0, 10, 0, 0] ,
[0, 0, 0, 0, 0, 10, 10, 0] ,
[0, 0, 0, 0, 0, 0, 0, 24] ,
[10, 0, 0, 0, 0, 0, 0, 0] ,
[10, 10, 10, 0, 0, 0, 0, 0] ,
[0, 0, 10, 0, 0, 0, 0, 0] ,
[0, 0, 0, 0, 24, 0, 0, 0] ,
];
Then it works great. I obviously see the difference between the two blocks of code, but why are they producing different results, and is it possible to modify my code to accommodate both examples?
If you examine the dodgy arc, you will see you can flip it into the right place by altering the sign on the transform from (50,0) to (-50,0). If you then look at the code that assigns the transform, it is
.attr("transform", function(d, i) {
d.pullOutSize = pullOutSize * ( d.startAngle + 0.01 > Math.PI ? -1 : 1);
return "translate(" + d.pullOutSize + ',' + 0 + ")";
});
with a note in the original text to say that "the 0.01 is for rounding errors". Given that the startAngle is already 3.13--i.e. very close to Pi--it looks like this is an edge case where the value fell just the wrong side of the cutoff. Changing the allowable rounding error value to 0.02 puts the arc in the correct place, or you could do something like
d.pullOutSize = pullOutSize * (
// is the start angle less than Pi?
d.startAngle + 0.01 < Math.PI ? 1 :
// if yes, is the end angle also less than Pi?
d.endAngle < Math.PI ? 1 : -1 );
to prevent edge cases like that in your dataset.

Dual pivot quick sort algorithm

I was analyzing the code for Arrays.sort() method in java . My question is for what values of integer array a[] will this code return true ?
if (less < e1 && e5 < great)
After Sorting left and right parts recursively, excluding known pivots for what value of array a[] will the center part become too large (comprises > 4/7 of the array) ?
Given QUICKSORT_THRESHOLD = 286 .
Array size cannot be more than 286
Any example of int array please .
It happens when all candidates for pivots are close to either the maximum or the minimum value of the array.
java.util.DualPivotQuicksort#sort() chooses the pivots from 5 positions in the array:
int seventh = (length >> 3) + (length >> 6) + 1;
int e3 = (left + right) >>> 1; // The midpoint
int e2 = e3 - seventh;
int e1 = e2 - seventh;
int e4 = e3 + seventh;
int e5 = e4 + seventh;
So, in order to construct an array that satisfies the condition, we need to fill those 5 positions with extreme values. For example:
int[] x = {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -2, /* e1 = 10 */
0, 0, 0, 0, 0, 0, -1, /* e2 = 17 */
0, 0, 0, 0, 0, 0, 0, /* e3 = 24 */
0, 0, 0, 0, 0, 0, 1, /* e4 = 31 */
0, 0, 0, 0, 0, 0, 2, /* e5 = 38 */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
};
Arrays.sort(x);
And a non-trivial case where the method changes the boundaries of the central part before sorting it:
int[] x = {
70, 66, 11, 24, 10, 28, 58, 13, 19, 90, 15,
79, 16, 69, 39, 14, 10, 16,
40, 59, 47, 77, 90, 50, 50,
50, 16, 76, 86, 70, 33, 90,
24, 35, 73, 93, 87, 19, 91,
73, 87, 22, 15, 24, 92, 34, 35, 98, 11, 40
};

Why can't I do an equality test of a synth parameter?

I'm mystified. In this code:
SynthDef(\acid,
{
|out, gate = 1, freq, myParam, amp, cutoff, resonance, filtEnvAmt|
var env, audio, filtEnv;
if (myParam == \something, { freq = 200; });
env = Linen.kr(gate, 0, 1, 0, doneAction: 2);
audio = LFSaw.ar(freq, mul: amp);
filtEnv = Line.kr(midicps(cutoff + filtEnvAmt), midicps(cutoff), 0.2);
audio = RLPFD.ar(audio, ffreq: filtEnv + MouseX.kr(0, 5000), res: MouseY.kr(0, 1), dist: 0);
Out.ar(out, audio * env);
}
).add;
b = Pbind(*[
out: 0,
instrument: \acid,
stepsPerOctave: 19,
scale: [0, 3, 5, 8, 11, 14, 17],
octave: 3,
degree: Pseq([0, \, 3, 3, 4, 4, 9, 4, 4]),
myParam: \something,
//prevFreq: Pseq([0, 0, 0, 0, 9, 0, 0, 0, 0]),
dur: Pseq([0.4, 0.4, 0.1, 0.1, 0.1, 0.1, 0.2, 0.1, 0.1]),
cutoff: Pseq([60, \, 50, 60, 80, 60, 90, 80, 50]),
filtEnvAmt: Pseq([20, \, 20, 20, 20, 20, -10, 20, 20]),
resonance: Pseq([0.5, \, 0.5, 0.5, 0.5, 0.5, 0.3, 0.5, 0.5])
]);
b.play;
..the equality test myParam == \something never returns true, despite the fact that the Pbind is clearly sending \something each time. No other value will work either: 0, nil etc.
The equality tests myParam == myParam and \something == \something do work however, so in these cases I get a monotone melody.
I can only guess that a Pbind sends each value in some kind of wrapper, but I've no idea how to then check them from inside the synth. Can anyone help?
First: you can't send symbols to a synth control. You can only send numbers.
Second: your example doesn't say what freq should be if the test is false. In fact, you should write it in more of a dataflow style such as:
freq = if(testcondition, 200, 300);
That's the kind of thing that will work in a synthdef.
Third is a slightly frustrating thing in sc language, which is that the == message is always evaluated at synthdef compile time - the equality is checked once, and then never again. In order to have "live" equality checking, you can use this slightly clunky expression:
BinaryOpUGen("==", thinga, thingb)
So in summary you might write
freq = if(BinaryOpUGen("==", myParam, 1), 200, 300);

Can I send values in a Pbind that are interpreted like midinote or degree?

I'm not sure whether SuperCollider can deliver moons on sticks, but I'd really like to be able to specify values in my Pbind that are interpreted in the same way as midinote or degree: i.e. converted automatically to a frequency.
So, an excerpt of such a Pbind, which produces a TB-303-style slide from one frequency to another:
b = Pbind(*[
out: 0,
instrument: \acid,
stepsPerOctave: 19,
scale: [0, 3, 5, 8, 11, 14, 17],
octave: 3,
degree: Pseq([0, \, 3, 3, 4, 4, 9, 4, 4]),
prevFreq: Pseq([\, \, 0, 3, 3, 4, 4, 9, 4]),
dur: Pseq([0.4, 0.4, 0.1, 0.1, 0.1, 0.1, 0.2, 0.1, 0.1]),
]);
...it would be super-duper if prevFreq were interpreted as containing degree values in the same way as degree.
In the absence of some kind of automatic conversion, I assume I need to do some kind of calculation within the synth itself in order to convert my values from a degree-type value to an actual frequency. I'm aware I can use foo.midicps to convert midinote-type values to a frequency, but is there a similar convenience function to convert degree-type values to a frequency (presumably also using the current scale and octave values)?
If you look at the helpfile for Event, you can see how it computes the frequency from the degree and scale:
note: #{    // note is the note in halftone steps from the root
    (~degree + ~mtranspose).degreeToKey(~scale, ~stepsPerOctave);
}
midinote: #{    // midinote is the midinote (continuous intermediate values)
    ((~note.value + ~gtranspose + ~root) / ~stepsPerOctave + ~octave) * 12.0;
}
freq: #{
    (~midinote.value + ~ctranspose).midicps * ~harmonic;
}
detunedFreq: #{    // finally sent as "freq" to the synth as a parameter, if given
    ~freq.value + ~detune
}
Event is an associative array and those ~variables can also be used as keys to the array (something which will hopefully become clear in a moment. It's also possible to get access to the events in a Pbind, by using a Pfunc. Let's say we want to calculate the current frequency for your Pbind:
b = Pbind(*[
out: 0,
instrument: \default,
stepsPerOctave: 19,
scale: [0, 3, 5, 8, 11, 14, 17],
octave: 3,
degree: Pseq([0, \, 3, 3, 4, 4, 9, 4, 4]),
dur: Pseq([0.4, 0.4, 0.1, 0.1, 0.1, 0.1, 0.2, 0.1, 0.1]),
foo: Pfunc({|evt|
var note, midinote, freq, detuned, result;
note = (evt[\degree] + evt[\mtranspose]).degreeToKey(evt[\scale], evt[\stepsPerOctave]);
midinote = ((note + evt[\gtranspose] + evt[\root]) / evt[\stepsPerOctave] + evt[\octave]) * 12.0;
freq = (midinote + evt[\ctranspose]).midicps * evt[\harmonic];
detuned = freq + evt[\detune];
detuned.postln;
})
]).play
Those calculations for note, midinote, freq and detuned freq are the same calculations we saw in the event helpfile. Therefore, this Pbind will now print out the frequency that you are currently playing.
What you actually want is the frequency you were previously playing, which we could figure out from your array of previous degrees. Or we could just keep track of the previous frequency in a variable. This will be a lot easier to keep track of!
(
var prev;
b = Pbind(*[
out: 0,
instrument: \default,
stepsPerOctave: 19,
scale: [0, 3, 5, 8, 11, 14, 17],
octave: 3,
degree: Pseq([0, \rest, 3, 3, 4, 4, 9, 4, 4]),
dur: Pseq([0.4, 0.4, 0.1, 0.1, 0.1, 0.1, 0.2, 0.1, 0.1]),
prevFreq: Pfunc({|evt|
var note, midinote, freq, detuned, result;
if (evt[\degree] == \rest, { detuned = \rest} , {
note = (evt[\degree] + evt[\mtranspose]).degreeToKey(evt[\scale], evt[\stepsPerOctave]);
midinote = ((note + evt[\gtranspose] + evt[\root]) / evt[\stepsPerOctave] + evt[\octave]) * 12.0;
freq = (midinote + evt[\ctranspose]).midicps * evt[\harmonic];
detuned = freq + evt[\detune];
});
//detuned.postln;
if (prev.isNil(), {
result = \rest;
} ,
{
result = prev;
});
prev = detuned
})
]).play
)

Resources