Migrating Parse files from Gridstore to S3 - parse-platform

Question

We migrated the data(without files) to mLab and Heroku. So the old files are still on Parse.
Since then, any new file added goes into Gridstore, which is the default file storage for mLab.
Now I migrated old parse files from Parse to an S3 Bucket using sashido
The files are migrated and are accessible using S3Adapter in Heroku.
But the files on Gridstore are not accessible now. How can I migrate them to the same S3 bucket and change references in mLab?

Answer 1

Maybe you're interested in the solution I've tried. It's not a simple operation, but I migrated successfully 3 databases with my parse server configuration.
It's based in a PHP script (with the Parse PHP SDK) that runs through every object, it gets the file from Parse.com and sets it (with any of your adapter configuration) in your own server.
The script looks like:
<?php
ini_set('display_errors', 1);
ini_set('display_startup_errors', 1);
error_reporting(E_ALL);
date_default_timezone_set('America/New_York');
$fileField = $argv[1];
$class = $argv[2];
require_once 'vendor/autoload.php';
use Parse\ParseObject;
use Parse\ParseQuery;
use Parse\ParseACL;
use Parse\ParsePush;
use Parse\ParseUser;
use Parse\ParseInstallation;
use Parse\ParseException;
use Parse\ParseAnalytics;
use Parse\ParseFile;
use Parse\ParseCloud;
use Parse\ParseClient;
$app_id = "******";
$rest_key = "******";
$master_key = "******";
ParseClient::initialize($app_id, $rest_key, $master_key);
ParseClient::setServerURL('http://localhost:1338/', 'parse');
$query = new ParseQuery($class);
$query->ascending("createdAt"); // it's just my preference
$query->exists($fileField);
$query->limit(1);
$count = $query->count();
for ($i = 0; $i < $count; $i = $i + 1) {
try {
$query->skip($i);
// get Entry
$entryWithFile = $query->first();
// get file
$parseFile = $entryWithFile->get($fileField);
// filename
$fileName = $parseFile->getName();
// if the file is hosted in Parse, do the job, otherwise continue with the next one
if (strpos($fileName, "tfss-") === false) {
echo "\nThis is already an internal file, skipping...";
continue;
}
$newFileName = str_replace("tfss-", "", $fileName);
$binaryFile = file_get_contents($parseFile->getURL());
$newFile = ParseFile::createFromData($binaryFile, $newFileName);
$entryWithFile->set($fileField, $newFile);
$entryWithFile->save(true);
echo "\nFile saved\n";
}
catch (Exception $e) {
// The conection with mongo or the server could be off for some second, let's retry it ;)
sleep(10);
continue;
}
}
echo "\n";
echo "END!";
?>
set your parse url correctly.
Imagine you want to migrate the file from class _User with field imageProfile, so be sure that you pass $fileField = "imageProfile"; $class = "_User".
Run that code for any field per class.
I did a dumb solution to work in parallel, which would be skipping steps in the for loop, for example:
<?php
ini_set('display_errors', 1);
ini_set('display_startup_errors', 1);
error_reporting(E_ALL);
date_default_timezone_set('America/New_York');
$index = $argv[1];
$of = $argv[2];
$fileField = $argv[3];
$class = $argv[4];
require_once 'vendor/autoload.php';
use Parse\ParseObject;
use Parse\ParseQuery;
use Parse\ParseACL;
use Parse\ParsePush;
use Parse\ParseUser;
use Parse\ParseInstallation;
use Parse\ParseException;
use Parse\ParseAnalytics;
use Parse\ParseFile;
use Parse\ParseCloud;
use Parse\ParseClient;
$app_id = "********";
$rest_key = "********";
$master_key = "********";
ParseClient::initialize($app_id, $rest_key, $master_key);
ParseClient::setServerURL('http://localhost:1338/', 'parse');
$query = new ParseQuery($class);
$query->ascending("createdAt");
$query->exists($fileField);
$query->limit(1);
$count = $query->count();
for ($i = $index; $i < $count; $i = $i + $of) {
try {
$query->skip($i);
// get Entry
$entryWithFile = $query->first();
// get file
$parseFile = $entryWithFile->get($fileField);
// filename
$fileName = $parseFile->getName();
// if the file is hosted in Parse, do the job, otherwise continue with the next one
if (strpos($fileName, "tfss-") === false) {
echo "\nThis is already an internal file, skipping...";
continue;
}
$newFileName = str_replace("tfss-", "", $fileName);
$binaryFile = file_get_contents($parseFile->getURL());
$newFile = ParseFile::createFromData($binaryFile, $newFileName);
$entryWithFile->set($fileField, $newFile);
$entryWithFile->save(true);
echo "\nFile saved\n";
}
catch (Exception $e) {
// The conection with mongo or the server could be off for some second, let's retry it ;)
sleep(10);
continue;
}
}
echo "\n";
echo "END!";
?>
so if you configure $fileField and $class as before, and you can open 3 threads and run:
php migrator.php 0 3 "imageProfile" "_User"
php migrator.php 1 3 "imageProfile" "_User"
php migrator.php 2 3 "imageProfile" "_User"
so you will have loops running like:
object 0, 3, 6
object 1, 4, 7
object 2, 5, 8
Good luck, and be quick! It's going to shut down in a few days.