laravel 4 artisan command memory issue - memory-management

Question

I wrote an artisan command to export exercises from a database, to standalone packages that can be used in an e-learning system like moodle, ...
It is a huge amount of exercises, and after a while the memory gets exhausted.
I tried to unset variables, activate the garbage collector, disabled the query log, and did some profiling, but till now with no success
I attached my script below, with each exercise I process, the memory usage adds up with 300k
Any idea's what I can do?
use Illuminate\Console\Command;
use Illuminate\Filesystem\Filesystem;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Input\InputArgument;
set_time_limit(0);
class ExerciseExportCommand extends Command {
/**
* The console command name.
*
* #var string
*/
protected $name = 'exercises:export';
/**
* The console command description.
*
* #var string
*/
protected $description = 'Export all exercises of a method or a specific exercise.';
/**
* Create a new command instance.
*
* #return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* #return void
*/
public function fire()
{
try {
DB::disableQueryLog();
ini_set('memory_limit','1024M');
$this->info('Initial: ' . number_format(memory_get_usage(), 0, '.', ',') . " bytes\n");
$base = base_path() . "/export/";
$playerPath = base_path() . "/public/preview/dist/";
$uploadsPath = base_path() . "/public/uploads/";
$methodId = $this->option('method');
$categoryId = $this->option('category');
$exerciseId = $this->option('exercise');
$this->info("Swing baby...");
$this->info("Let's export some exercises, shall we ?");
//we make an array which holds all the exercises we have to export
$exercises = array();
if($methodId === NULL && $categoryId === NULL && $exerciseId === NULL){
//we are here now anyways, let's do all exercises at once...
$this->comment("Nothing specified, let's do all exercises ");
$exercises2 = Exercise::all();
foreach ($exercises2 as $exercise){
array_push($exercises, $exercise->id);
}
unset($exercises2);
}
//get all exercises for given methodId
if($methodId !== NULL){
$method = Method::with('categories.exercises')->find($methodId);
if($method == NULL) break;
$this->comment("We are ready to roll method " . $method->code);
foreach($method->categories as $category){
foreach($category->exercises as $exercise->id){
array_push($exercises, $exercise);
}
}
unset($method);
}
//get all exercises for given categoryId
if($categoryId !== NULL){
$category = Category::with('exercises')->find($categoryId);
if($category == NULL) break;
$this->comment("We are ready to roll category " . $category->name_prefix . " " . $category->name);
foreach($category->exercises as $exercise->id){
array_push($exercises, $exercise);
}
unset($category);
}
if($exerciseId != null){
$exercise = Exercise::find($exerciseId);
if($exercise != NULL) {
array_push($exercises, $exercise->id);
$this->comment("Exercise added for export: " . $exercise->name_prefix . " " . $exercise->name);
} else {
}
unset($exercise);
}
if(empty($exercises)){
$this->error("No exercises could be found for given method/exercise");
exit();
} else {
$this->comment("Currently counting " . count($exercises) . " exerises to export");
}
$fs = new Filesystem();
//loop the exercises and publish like a charm
foreach($exercises as $exerciseId){
$exercise = Exercise::find($exerciseId);
//determine destination
$path = $base . $exercise->getPath();
$this->comment("starting exercise " . $exercise->id);
//check if path exists, if it does, wipe it out
if($fs->exists($path)){
$fs->deleteDirectory($path, true);
$this->comment("wiped out " . $path);
}
//copy player files
//echo "copying " . $path . "<br />";
$fs->copyDirectory($playerPath, $path);
$fs->cleanDirectory($path."styles/skins");
//copy only necesary skin files to save disk space
$method = $exercise->method();
if($fs->exists($playerPath."styles/skins/".$method->code)){
$fs->copyDirectory($playerPath."styles/skins/".$method->code, $path."styles/skins/".$method->code);
} elseif($method->code == "kameleonspelling" || $method->code == "kameleontaalbeschouwing"){
$fs->copyDirectory($playerPath."styles/skins/kameleon", $path."styles/skins/kameleon");
}
if($fs->exists($playerPath."styles/skins/".$method->code.".css")){
$fs->copy($playerPath."styles/skins/".$method->code.".css", $path."styles/skins/".$method->code.".css");
}
$this->comment("copied player files to " . $path);
//copy resources
//echo "copying resources " . $path . "<br />";
$fs->copyDirectory($uploadsPath . $exercise->id . "/", $path);
$this->comment("copied resources to " . $path);
//copy slide resources
$slides = Slide::where('exerciseID',"=",$exercise->id)->get();
mkdir($path."slides/");
foreach ($slides as $slide) {
$image = $slide->image()->first();
if($image != NULL){
$this->info($uploadsPath."slides/".$image->resourceUri);
$this->info($path."slides/".$image->resourceUri);
$fs->copy($uploadsPath."slides/".$image->resourceUri, $path."slides/".$image->resourceUri);
}
unset($image);
}
$this->comment("copied slide resources to " . $path);
//save xml file
$content = Exercise::getXmlContent($exercise->id);
$fs->put($path . "exercise.xml", View::make('xml', $content));
$this->comment("saved xml to " . $path);
$this->info("finished exercise " . $exercise->id);
unset($method);
unset($content);
unset($slides);
gc_collect_cycles();
$this->info('Peak: ' . number_format(memory_get_peak_usage(), 0, '.', ',') . " bytes\n");
$this->info('End: ' . number_format(memory_get_usage(), 0, '.', ',') . " bytes\n");
}
$this->info("Awesome Possum => finished all exercises ");
$this->info('Peak: ' . number_format(memory_get_peak_usage(), 0, '.', ',') . " bytes\n");
$this->info('End: ' . number_format(memory_get_usage(), 0, '.', ',') . " bytes\n");
} catch(Exception $e){
$this->error($e->getMessage());
$this->comment($e->getTraceAsString());
}
}
/**
* Get the console command arguments.
*
* #return array
*/
protected function getArguments()
{
return array(
//array('example', InputArgument::REQUIRED, 'An example argument.'),
);
}
/**
* Get the console command options.
*
* #return array
*/
protected function getOptions()
{
return array(
array('method', null, InputOption::VALUE_OPTIONAL, 'The id of a method for which all the exercises to export.', null),
array('category', null, InputOption::VALUE_OPTIONAL, 'The id of a category for which all the exercises to export.', null),
array('exercise', null, InputOption::VALUE_OPTIONAL, 'The id of an exercise to export.', null),
);
}
}
This is a dump of my xdebug trace command, with the 20 most memory consuming statements:
Showing the 20 most costly calls sorted by 'memory-own'.
Inclusive Own
function #calls time memory time memory
-------------------------------------------------------------------------------------------------------
debug_backtrace 646 0.0420 20353496 0.0420 20353496
Composer\Autoload\ClassLoader->loadClass 259 0.1911 17556224 0.1139 13953824
PDOStatement->execute 743 0.1184 13729408 0.1184 13729408
array_merge 4051 0.1282 3894816 0.1282 3894816
Illuminate\Database\Eloquent\Model->newInstance 1534 0.4715 3806344 0.0791 3732712
PDOStatement->fetchAll 742 0.0323 2364264 0.0323 2364264
Illuminate\Database\Eloquent\Model->newBaseQueryBuilder 738 0.6625 2177352 0.0657 1688968
explode 3396 0.1026 1296960 0.1026 1296960
Illuminate\Database\Eloquent\Model->newFromBuilder 1534 0.6883 5139552 0.0944 1259576
str_replace 10254 0.3176 1228824 0.3176 1228824
compact 920 0.0339 1181384 0.0339 1181384
PDO->prepare 743 0.1403 816488 0.1403 816488
sprintf 2381 0.0741 802968 0.0741 802968
implode 5586 0.1722 536688 0.1722 536688
array_map 864 0.3164 588512 0.0386 477088
get_class_methods 15 0.0059 472296 0.0059 472296
Illuminate\Database\Eloquent\Model->newQuery 738 0.9783 3044352 0.0656 448488
include 263 6.7525 5732672 0.0468 410416
call_user_func_array 1585 0.5734 3937936 0.0659 357056
SplFileInfo->getPathname 2724 0.0847 344768 0.0847 344768

Answer 1

Turns out that DB::disableQueryLog();fixed it after all. !!
At first I thought it didn't help, because memory kept adding up, and I manually cancelled my script each time. Now while debugging with Memtrack I kept my command running and i noticed after a while the memory usage stagnates.
I'm guessing that the garbage collector doesn't clean up the memory until it decides its necesary?

Answer 2

What is your servers memory_limit?
One solution could be that you split up your command in smaller commands, and run them accordingly. Maybe automate that a new command is run after one is completed?
I would also suggest using some sort of queue provider. This way you could split up the workload to a longer period of time. Additionally the workload is not distributed on your server at all.
Laravel has built in support for: Pheanstalk, Amazon SQS and IronMQ.
Heres the doc link: Laravel queue docs