Reduce memory allocation and increase speed in Julia code - performance

Question

I am trying to run multiple regressions for all possible combinations of selected variables. My code is running but it takes a long time and the memory allocation seems very big. Is there a way to optimize this:
using DataFrames
using Statistics
using GLM
using Combinatorics
using Distributions
### read the data
reg_dat = DataFrame(
sample = wsample([1,2],[0.8,0.2], 1000000),y_var = rand(1000000),
ind = rand(1000000), brak = rand(1000000), times = rand(1000000),
tiny = rand(1000000), regr = rand(1000000), breaker = 1 .- rand(1000000),
x_var = 10*(1 .- rand(1000000)), kink = rand(1000000), h_var = rand(1000000),
ind_x = rand(1000000), brak_x = 1 .- rand(1000000), times_x = 1 .- rand(1000000),
tiny_x = 1 .- rand(1000000), regr_x = rand(1000000), breaker_x = rand(1000000),
t_var = rand(1000000), x_var_x = rand(1000000), kink_x = rand(1000000),
units = rand((1,2,3,4,6,7,8,9,15,78,76,10,23,45,54,87,98), 1000000),
trent = 1:1000000, rz = rand(string.(1:25), 1000000)
)
# find all possible combinations of the independent variables
function combinat(names::Array{String,1},indices::Array{Int64,1},N::Int64)::Vector{Array{String,1}}
cols = Vector{String}()
coms = Vector{Array{String}}()
for i in indices
push!(cols, names[i]::String)
end
for j in 1:N
append!(coms, collect(combinations(cols,j))::Array{Array{String,1},1})
end
return coms
end
combs = combinat(names(reg_dat),[3:20...], length([3:20...]))
### Find all unique units
units = unique(reg_dat.units)
## Convert combinations to regression formulas
function convert_to_formula(vals::Array{Array{String,1},1})::Array{FormulaTerm{Term,R} where R,1}
forms = Array{FormulaTerm{Term,R} where R,1}()
for i in vals
new_i = [i;["rz","trent"]]
symbs = Array{Symbol,1}()
for j in new_i
push!(symbs, Symbol(j))
end
push!(forms, Term(:y_var) ~ sum(term.(symbs)))
end
return forms
end
# Run regressions for each combination of units and combs
function logit_run(data::DataFrame, units::Array{Int64,1}, ind_vars::Array{Array{String,1},1})::Array{Tuple{Int64,String,Float64},1}
forms = convert_to_formula(ind_vars)
out = Array{Tuple{Int64,String,Float64},1}()
for i in units
data_train = data[(data.sample .== 1) .& (data.units .== i),:]
data_test = data[(data.sample .== 2) .& (data.units .== i),:]
for j in forms
try
logit = glm(j, data_train, Binomial(), LogitLink(), contrasts = Dict(:rz => DummyCoding()))
devs = data_test.y_var - predict(logit,data_test)
push!(out, (i, string(j), sqrt(mean(devs .* devs))))
catch
push!(out, (i, string(j), NaN))
end
end
end
return out
end
Now when I run:
#time logit_run(reg_dat, units, combs[1:10])
> 18.413586 seconds (26.24 M allocations: 10.644 GiB, 10.50% gc time)
170-element Array{Tuple{Int64,String,Float64},1}:
(1, "y_var ~ ind + rz + trent", 0.2880341157225821)
(1, "y_var ~ brak + rz + trent", 0.2880411138604235)
(1, "y_var ~ times + rz + trent", 0.2880380466963764)
(1, "y_var ~ tiny + rz + trent", 0.2880396984065766)
(1, "y_var ~ regr + rz + trent", 0.2880343939542883)
(1, "y_var ~ breaker + rz + trent", 0.2880393689495619)
(1, "y_var ~ x_var + rz + trent", 0.288038968246708)
(1, "y_var ~ kink + rz + trent", 0.288043185030096)
(1, "y_var ~ h_var + rz + trent", 0.28804607747341865)
(1, "y_var ~ ind_x + rz + trent", 0.2880387457490556)
(98, "y_var ~ ind + rz + trent", 0.29044425999770246)
(98, "y_var ~ brak + rz + trent", 0.2904354957341227)
(98, "y_var ~ times + rz + trent", 0.2904391507891304)
The memory allocation is high and it is not as fast as I would want it. I will be running millions of iterations with this and will want 170 elements to be faster than this. Any help is appreciated.
Update:
Using a low level API did not quite work for me so I found a python implementation of the logistic regression and wrote it in Julia
### Adding an intercept of ones
reg_dat[!, :Intercept] = ones(nrow(reg_dat))
### Creating dummies for the rz variable
for i in unique(reg_dat.rz)[2:end]
reg_dat[!, Symbol(i)] = ifelse.(reg_dat[!, :rz] .== i, 1, 0)
end
# find all possible combinations
function combinat(indices::Array{Int64,1},N::Int64)::Vector{Array{Int64,1}}
coms = Vector{Array{Int64}}()
for j in 1:N
append!(coms, collect(combinations(indices,j)))
end
return coms
end
combins = combinat([3:20...], length([3:20...]))
unit_s = unique(reg_dat.units)
### Now the logistic regression function from scratch
function log_reod(unit_s::Array{Int64,1}, combos::Array{Array{Int64,1},1}, cols::Array{String,1}, mydata::DataFrame, iterations::Int64)
out = Array{Tuple{Int64,String,Float64},1}()
Threads.#threads for j in combos::Array{Array{Int64,1},1}
Threads.#threads for i in unit_s::Array{Int64,1}
try
train = mydata[(mydata.sample .== 1) .& (mydata.units .== i),:]
test = mydata[(mydata.sample .== 2) .& (mydata.units .== i),:]
X = Matrix(train[!, [24; j;[22,25:48...]]])[:,:]
y = train[!, :y_var]
X_test = Matrix(test[!, [24; j;[22,25:48...]]])[:,:]
y_test = test[!, :y_var]
w = zeros(size(X)[2])
y_bar = mean(y)
w_init = log(y_bar/(1-y_bar))
converged = false
nll_sequence = Array{Float64,1}(undef,iterations)
ab = Array{Tuple{Float64, Bool}, 1}(undef,iterations)
Threads.#threads for i in 1:iterations
h = X*w
p = 1 ./ (1 .+ exp.(-h))
p_adj = p
p_adj[p_adj .== 1.0] .= 0.99999999
nll = -(1 - y'log.(1 .- p_adj)) + y'log.(p_adj)
nll_sequence[i] = nll
if i .> 1
if !converged & (abs(nll_sequence[end]-nll_sequence[end-1]) < 0.000001)
converged = true
else
converged = false
end
else
converged = false
end
s = p .* (1 .- p)
S = Diagonal(s)
arb_small = ones(length(s)) .* 0.000001
arb_small[s .!= 0] = ((y.-p)./s)[s .!= 0]
z = h + arb_small
w = ((inv(X'*S*X)*X')*S)*z
pred = exp.(X_test*w) ./ (1 .+ exp.(X_test*w))
rmse = sqrt(mean((y_test - pred) .* (y_test - pred)))
ab[i] = (rmse, converged)
end
global ab_fin = ab[findfirst(getfield.(ab,2))-1][1]
catch
global ab_fin = NaN
end
push!(out, (i, join(cols[j],","), ab_fin))
end
end
return out
end
This seems to work. However, memory allocation is worse and not as fast as the implementation with the GLM (high level) API. Why is this the case?
#time log_reod(unit_s, combins[1:10],names(reg_dat), reg_dat, 15)
171.060764 seconds (5.88 M allocations: 91.736 GiB, 12.45% gc time)
170-element Array{Tuple{Int64,String,Float64},1}:
(4, "ind", 0.29000435564005556)
(15, "ind", 0.28887795028663027)
(8, "ind", 0.286764063267709)
(98, "ind", 0.28941319680023303)
(76, "ind", 0.2898017575615879)
(3, "ind", 0.2910086150956134)
(45, "ind", 0.288856696122843)
(9, "ind", 0.287618950665158)
(1, "ind", 0.2884813713274216)
(2, "ind", 0.28729283992016885)
(7, "ind", 0.28700362633959264)
(6, "ind", 0.2873866890697135)
(10, "ind", 0.28889513223197577)
⋮
(3, "ind_x", 0.29102070956343595)
(45, "ind_x", 0.28884184457957934)
(9, "ind_x", 0.2876167225404162)
(1, "ind_x", 0.2884805471334929)
(2, "ind_x", 0.2873338122136366)
(7, "ind_x", 0.2870078816051725)
(6, "ind_x", 0.28739698518229273)
(10, "ind_x", 0.28891515579773225)
(23, "ind_x", 0.2898286130508298)
(78, "ind_x", 0.2890331733804241)
(54, "ind_x", 0.28855221319296853)
(87, "ind_x", 0.2875034656519413)

Answer 1

Actually I have misdirected you - sorry for this.
GLM.jl does not support views. Still you can reduce the time if you use matrices directly, but not much unfortunately.
Data preparation
df = DataFrame(rand(10^6, 10))
df.y = sum.(eachrow(df)) .+ 10 .* randn(10^6) .< 5
function some_models(df)
[glm(#formula(y~x1+x2+x3+x4+x5+x6+x7+x8+x9+x10), df, Binomial(), LogitLink()),
glm(#formula(y~x1+x2+x3+x4+x5+x6+x7+x8+x9), df, Binomial(), LogitLink()),
glm(#formula(y~x1+x2+x3+x4+x5+x6+x7+x8+x10), df, Binomial(), LogitLink()),
glm(#formula(y~x1+x2+x3+x4+x5+x6+x7+x9+x10), df, Binomial(), LogitLink()),
glm(#formula(y~x1+x2+x3+x4+x5+x6+x8+x9+x10), df, Binomial(), LogitLink()),
glm(#formula(y~x1+x2+x3+x4+x5+x7+x8+x9+x10), df, Binomial(), LogitLink()),
glm(#formula(y~x1+x2+x3+x4+x6+x7+x8+x9+x10), df, Binomial(), LogitLink()),
glm(#formula(y~x1+x2+x3+x5+x6+x7+x8+x9+x10), df, Binomial(), LogitLink()),
glm(#formula(y~x1+x2+x4+x5+x6+x7+x8+x9+x10), df, Binomial(), LogitLink()),
glm(#formula(y~x1+x3+x4+x5+x6+x7+x8+x9+x10), df, Binomial(), LogitLink()),
glm(#formula(y~x2+x3+x4+x5+x6+x7+x8+x9+x10), df, Binomial(), LogitLink())]
end
function some_models2(df)
X = [ones(nrow(df)) Matrix(df[!, 1:end-1])]
y = df.y
[fit(GeneralizedLinearModel, X, y, Binomial(), LogitLink()),
fit(GeneralizedLinearModel, X[:, Not(11)], y, Binomial(), LogitLink()),
fit(GeneralizedLinearModel, X[:, Not(10)], y, Binomial(), LogitLink()),
fit(GeneralizedLinearModel, X[:, Not(9)], y, Binomial(), LogitLink()),
fit(GeneralizedLinearModel, X[:, Not(8)], y, Binomial(), LogitLink()),
fit(GeneralizedLinearModel, X[:, Not(7)], y, Binomial(), LogitLink()),
fit(GeneralizedLinearModel, X[:, Not(6)], y, Binomial(), LogitLink()),
fit(GeneralizedLinearModel, X[:, Not(5)], y, Binomial(), LogitLink()),
fit(GeneralizedLinearModel, X[:, Not(4)], y, Binomial(), LogitLink()),
fit(GeneralizedLinearModel, X[:, Not(3)], y, Binomial(), LogitLink()),
fit(GeneralizedLinearModel, X[:, Not(2)], y, Binomial(), LogitLink()),
fit(GeneralizedLinearModel, X[:, Not(1)], y, Binomial(), LogitLink())
]
end
Benchmarking:
julia> some_models(df); # first run
julia> #time some_models(df); # second run
8.955404 seconds (10.72 k allocations: 4.713 GiB, 5.36% gc time)
julia> some_models2(df); # first run
julia> #time some_models2(df); # second run
8.598572 seconds (1.20 k allocations: 3.412 GiB, 8.65% gc time)