I am using Laravel-5.8 and Maatwebsite/excel-3.1 package in Excel to import users profile in a web application.
namespace App\Imports;
use App\User;
use App\StudentInfo;
use Illuminate\Support\Facades\Hash;
use Maatwebsite\Excel\Row;
use Maatwebsite\Excel\Concerns\OnEachRow;
use Maatwebsite\Excel\Concerns\WithHeadingRow;
class FirstStudentSheetImport implements OnEachRow, WithHeadingRow
{
protected $class, $section;
public function onRow(Row $row)
{
$rowIndex = $row->getIndex();
if($rowIndex >= 200)
return; // Not more than 200 rows at a time
$row = $row->toArray();
$this->class = (string) $row[__('class')];
$this->section = (string) $row[__('section')];
$user = [
'name' => $row[__('name')],
'email' => $row[__('email')],
'password' => Hash::make($row[__('password')]),
'active' => 1,
'code' => auth()->user()->code,
'address' => $row[__('address')],
'about' => $row[__('about')],
'pic_path' => '',
'birthday' => $row[__('birthday')]?? date('Y-m-d'),
'phone_number' => $row[__('phone_number')],
'verified' => 1,
'section_id' => $this->getSectionId(),
'blood_group' => $row[__('blood_group')],
'nationality' => $row[__('nationality')],
'gender' => $row[__('gender')],
];
$tb = create(User::class, $user);
}
From the code above, how do I validate:
birthday to be less than current date
phone_number to be integer and 13 digits
email to be strictly email
How do I display list of fields with errors in it fails
How do I display imported files if successful
How do I remove the first row (header)
Related
I am importing Excel File using Laravel-8 endpoints and Maatwebsite-3.1 package.
The way I did it is that, the Uploaded Excel file will first get to the storage (storage/file_imports/student_imports). Then Laravel pickes it up from there, stores it into the DB table through StudentImport using Maatwebsites.
StudentImport:
<?php
namespace App\Imports;
use App\Models\User;
use App\Models\Student;
use Illuminate\Support\Facades\Hash;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Str;
use Illuminate\Support\Facades\Auth;
use Illuminate\Support\Facades\Password;
use Illuminate\Validation\Rule;
use Maatwebsite\Excel\Concerns\ToModel;
use Maatwebsite\Excel\Concerns\Importable;
use Maatwebsite\Excel\Concerns\WithBatchInserts;
use Maatwebsite\Excel\Concerns\WithValidation;
use Maatwebsite\Excel\Concerns\WithHeadingRow;
use Maatwebsite\Excel\Concerns\SkipsErrors;
use Maatwebsite\Excel\Concerns\SkipsOnError;
use Maatwebsite\Excel\Concerns\SkipsFailures;
use Maatwebsite\Excel\Concerns\SkipsOnFailure;
use Illuminate\Support\Facades\Validator;
use Maatwebsite\Excel\Concerns\SkipsEmptyRows;
use Maatwebsite\Excel\Validators\Failure;
use Throwable;
class StudentImport implements
ToModel,
WithValidation,
WithHeadingRow,
SkipsOnError,
SkipsOnFailure,
WithBatchInserts
{
protected $companyId;
public function __construct()
{
$this->companyId = Auth::user()->company_id;
}
use Importable, SkipsErrors, SkipsFailures;
public function model(array $row)
{
$student_data = [
'first_name' => $row[0],
'other_name' => $row[1] ?? '',
'last_name' => $row[2],
'email' => preg_replace('/\s+/', '', strtolower($row[3])),
'gender' => $row[4],
'nationality_id' => $this->getNationality() ?? '',
'school_id' => Auth::user()->school_id,
];
$student = Student::create($student_data);
if (User::where('email', '=', $student->email)->exists()) {
$user = User::update([
'first_name' => $student->first_name,
'other_name' => $student->other_name,
'last_name' => $student->last_name,
'complete_profile' => 1,
'active' => 1,
'user_type' => 'Driver',
'company_id' => Auth::user()->company_id,
'updated_at' => date("Y-m-d H:i:s"),
'updated_by' => Auth::user()->id,
]);
}else{
$user = User::create([
'email' => $student->email,
'username' => strtok($row[3], '#'),
'password' => bcrypt("123456"),
'first_name' => $student->first_name,
'other_name' => $student->other_name,
'last_name' => $student->last_name,
'activation_token' => str_random(10),
]);
}
}
public function headingRow(): int
{
return 1;
}
public function getRowCount(): int
{
return $this->rows;
}
public function customValidationAttributes()
{
return [
'0' => 'First Name',
'1' => 'Other Name',
'2' => 'Last Name',
'3' => 'Email',
'4' => 'Gender',
];
}
public function rules(): array
{
return [
'*.0' => [
'required',
'string',
'max:50'
],
'*.1' => [
'nullable',
'string',
'max:50'
],
'*.2' => [
'required',
'string',
'max:50'
],
'*.3' => [
'required',
'email',
'max:100',
Rule::unique('studentss')->where(function ($query) {
return $query->where('school_id', Auth::user()->school_id);
})
],
'*.4' => [
'required',
'string',
'max:20'
],
];
}
public function batchSize(): int
{
return 1000;
}
public function chunkSize(): int
{
return 1000;
}
public function onFailure(Failure ...$failures)
{
// Handle the failures how you'd like.
}
public function customValidationMessages()
{
return [
'1.in' => 'Custom message for :attribute.',
'nim.unique' => 'Custom message',
];
}
}
Controller:
public function importStudent(Request $request)
{
try {
$user = Auth::user()->id;
$validator = Validator::make($request->all(), [
'document' => 'file|mimes:xlsx|max:10000',
]);
if($validator->fails()) {
return $this->error($validator->errors(), 401);
} else {
$check = User::where('id', $user)->pluck('id');
if($check[0] !== null || $check[0] !== undefined) {
$file = $request->file('document');
$file->move(public_path('storage/file_imports/student_imports'), $file->getClientOriginalName());
Excel::import(new StudentImport, public_path('storage/file_imports/student_imports/' . $file->getClientOriginalName() ));
return $this->success('Students Successfully Imported.', [
'file' => $file
]);
} else {
return $this->error('Not allowed', 401);
}
}
} catch(\Exception $e) {
return $this->error($e->getMessage());
}
}
As I stated earlier it will first store the Excel file into:
storage/file_imports/student_imports
Then pick it from there and store in the DB.
This code is in the controller above.
The file is store in the
public/storage/file_imports/student_imports
as expected, but nothing is found in the DB. Yes it displays Success with no error.
What could be the problem and how do I resolve it?
Thanks
model method in your StudentImport class must return new instance of Eloquent model.
You shouldn't make create or update method call through your model in here.
Maatwebsite-Excel manage your insert process with own manager. Check source code.
https://github.com/Maatwebsite/Laravel-Excel/blob/3.1/src/Imports/ModelImporter.php#L74
https://github.com/Maatwebsite/Laravel-Excel/blob/3.1/src/Imports/ModelImporter.php#L92
https://github.com/Maatwebsite/Laravel-Excel/blob/3.1/src/Imports/ModelManager.php#L45
https://github.com/Maatwebsite/Laravel-Excel/blob/3.1/src/Imports/ModelManager.php#L64
TLDR: I have a 150 second loop to run triggered by a HTTP request, I need to async this as the request will die due to time before it's completed - Laravel Queues seem to handle a large single import, not thousands of small ones - so anyone know how to do this?
Longer version:
I'm trying to import a large spreadsheet into my MySQL database, but the execution time takes longer than 60 seconds so the request times out. My code loops through the data creating rows in the database:
This is a method within a controller - "ProjectController" the main point is there is a very long loop as there is so much data to process and store
public function bulkStore(Request $request)
{
$allProjects = $request->all();
$properties = Property::where('client_id', 1)->get();
$propertiesArray = [];
foreach($properties as $property)
{
$propertiesArray[$property->name] = $property->id;
}
foreach($allProjects as $project)
{
$project = json_decode(json_encode($project), true);
$project_id = Project::create([
'client_id' => 1,
'reference' => $project['Reference'],
'name' => $project['Project Name'],
'description' => null,
'created_at' => Carbon::now()->format('Y-m-d H:i:s'),
'updated_at' => Carbon::now()->format('Y-m-d H:i:s')
])->id;
foreach($project as $property => $value)
{
if(!in_array($property, array('Reference', 'Project Name')))
{
$property_id = $propertiesArray[$property];
$value = json_encode($value);
DB::table('property_values')->insert([
'project_id' => $project_id,
'property_id' => $property_id,
'data' => $value,
'created_at' => Carbon::now()->format('Y-m-d H:i:s'),
'updated_at' => Carbon::now()->format('Y-m-d H:i:s')
]);
}
}
}
}
I'm not sure where to put this code and how to handle the request so that it's completed in the background after a quick validation 200:ok response is given to the client.
Any help much appreciated - apologies if this loop is hideous - I'm quite new to programming.
Creating eloquent models is the heavy part, I guess. You could just store the given data in a temporary json in file and return a 200 OK.
Storage::disk('your_temp_disk')->put('import.json', json_encode($project));
Then queue a job:
ImportProjects::dispatch($path_to_the_import_json);
Then pick up the file in the ImportProjects job, create your models and finally delete the temporary file.
You can move the logic to a job.
In your controller
use App\Jobs\ProcessProperties;
public function bulkStore(Request $request)
{
$allProjects = $request->all();
$properties = Property::where('client_id', 1)->get();
ProcessProperties::dispatch($properties);
}
app/Jobs/ProcessProperties.php
<?php
namespace App\Jobs;
use Illuminate\Bus\Queueable;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
class ProcessProperties implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $properties;
public function __construct($properties)
{
$this->properties = $properties;
}
public function handle()
{
$propertiesArray = [];
foreach($this->properties as $property)
{
$propertiesArray[$property->name] = $property->id;
}
foreach($allProjects as $project)
{
$project = json_decode(json_encode($project), true);
$project_id = Project::create([
'client_id' => 1,
'reference' => $project['Reference'],
'name' => $project['Project Name'],
'description' => null,
'created_at' => Carbon::now()->format('Y-m-d H:i:s'),
'updated_at' => Carbon::now()->format('Y-m-d H:i:s')
])->id;
foreach($project as $property => $value)
{
if(!in_array($property, array('Reference', 'Project Name')))
{
$property_id = $propertiesArray[$property];
$value = json_encode($value);
DB::table('property_values')->insert([
'project_id' => $project_id,
'property_id' => $property_id,
'data' => $value,
'created_at' => Carbon::now()->format('Y-m-d H:i:s'),
'updated_at' => Carbon::now()->format('Y-m-d H:i:s')
]);
}
}
}
// You can emit and event here to be notified when the job completes
}
}
Then its all about running the queue worker with php artisan queue:work.
I have JSON file and have data in it. I want to import all data in the database through DB seeders. I am getting an error Trying to get property name of non-object. I have multiple data how I can insert in the database?
public function run()
{
$json = File::get("public/kmz/WASASubdivisions.geojson");
$data = json_decode($json);
// dd($data);
foreach ($data as $obj){
Regions::create(array(
'name' => $obj[0]->Name,
'description' => $obj[0]->description,
'altitudeMode' => $obj[0]->altitudeMode,
'Town' => $obj[0]->Town,
'AC' => $obj[0]->AC,
'No_of_TW' => $obj[0]->No_of_TW,
'No' => $obj[0]->No,
'DC'=> $obj[0]->DC,
'HH_2017' => $obj[0]->HH_2017,
'FID' => $obj[0]->FID,
'Area_ha' => $obj[0]->Area_ha,
'Field_1' => $obj[0]->Field_1,
'Pop_Dens' => $obj[0]->Pop_Dens,
'Id' => $obj[0]->Id,
'Pop_2017' => $obj[0]->Pop_2017,
'Area_Sq'=> $obj[0]->Area_Sq,
));
}
}
Sample Json Format
31 => {#837
+"type": "Feature"
+"properties": {#838
+"Name": "Gujjar Pura"
+"description": null
+"altitudeMode": "clampToGround"
+"Town": "Shalimar Town"
+"AC": "31"
+"No_of_TW": "11"
+"No": "13"
+"DC": "38"
+"HH_2017": "30478"
+"FID": "31"
+"Area_ha": "648.327"
+"Field_1": "Gujjar Pura"
+"Pop_Dens": "54063.141167"
+"Id": "0"
+"Pop_2017": "196619"
+"Area_Sq": "3.63684"
}
+"geometry": {#839
+"type": "MultiPolygon"
+"coordinates": array:1 [
0 => array:1 [
0 => array:169 [ …169]
]
]
}
}
Let's support I have a model Post and I want to save additional data in JSON format in the posts table. In that case, we can use property $casts in Laravel. Which will cast our field value to whatever we gave.
<?php
namespace App\Models;
use Illuminate\Database\Eloquent\Model;
class Post extends Model
{
protected $table='posts';
protected $fillable = ['user_id', 'title', 'short_description', 'description', 'status', 'json_data'];
/**
* The attributes that should be cast to native types.
*
* #var array
*/
protected $casts = [
'json_data' => 'array',
];
}
Now we want to save data something like this
$data = [
'user_id' => 1,
'title' => 'abc',
'short_description' => 'test',
'description' => 'test',
'status' => true,
'json_data' => [
'additional_info' => '',
'post_image' => '',
...
],
];
$item = new Post;
$item->fill($data)->save();
This will save your json_data array values to JSON in the database. But when you will get data from the database it will convert that to array automatically.
For reference read this
since i am not a big fan of processing the json as a object
So the json_decode will accept the second arg so
$json = File::get("public/kmz/WASASubdivisions.geojson");
$data = json_decode($json,true);
dd($data);
foreach ($data as $obj)
{
Regions::create(array(
'name' => $obj['Name'],
'description' => $obj['description'],
'altitudeMode' => $obj['altitudeMode'],
'Town' => $obj['Town'],
'AC' => $obj['AC'],
'No_of_TW' => $obj['No_of_TW'],
'No' => $obj['No'],
'DC'=> $obj['DC'],
'HH_2017' => $obj['HH_2017'],
'FID' => $obj['FID'],
'Area_ha' => $obj['Area_ha'],
'Field_1' => $obj['Field_1'],
'Pop_Dens' => $obj['Pop_Dens'],
'Id' => $obj['Id'],
'Pop_2017' => $obj['Pop_2017'],
'Area_Sq'=> $obj['Area_Sq'],
));
}
Can You Post the dd() result and fileds sets of the table
public function run()
{
$json = File::get("public/kmz/WASASubdivisions.geojson");
$data = json_decode($json);
dd($data);
foreach ($data as $obj){
Regions::create(array(
'name' => $obj->Name,
'description' => $obj->description,
'altitudeMode' => $obj->altitudeMode,
'Town' => $obj->Town,
'AC' => $obj->AC,
'No_of_TW' => $obj->No_of_TW,
'No' => $obj->No,
'DC'=> $obj->DC,
'HH_2017' => $obj->HH_2017,
'FID' => $obj->FID,
'Area_ha' => $obj->Area_ha,
'Field_1' => $obj->Field_1,
'Pop_Dens' => $obj->Pop_Dens,
'Id' => $obj->Id,
'Pop_2017' => $obj->Pop_2017,
'Area_Sq'=> $obj->Area_Sq,
));
}
}
The attributes are under the properties key, but you're referencing them from the root of the object. e.g. $obj[0]->Name should be $obj[0]->properties->Name, etc.
I had user migration:
$table->enum('type',['seller','buyer'])->default('seller');
I want when using ModelFactory how to get random value seller or buyer?
$factory->define(App\User::class, function (Faker\Generator $faker) {
static $password;
return [
'firstName' => $faker->name,
'lastName' => $faker->name,
'username' => $faker->unique()->username,
'email' => $faker->unique()->safeEmail,
'password' => md5('user123'),
'bio' => $faker->sentence(3, true),
'type' => ???,
];
});
Make use of randomElement method
'type' => $faker->randomElement(['seller', 'buyer']),
Laravel version >= 5.6
use Illuminate\Support\Arr;
$array = [1, 2, 3, 4, 5];
$random = Arr::random($array);
// 4 - (retrieved randomly)
"type" => Arr::random($array);
Just in case that anyone is looking for the answer of this question with newer version of Laravel and PHP, you can utilize the enum in PHP like so:
<?php
namespace App\Enums;
enum UserTypeEnum: string
{
case SELLER = 'seller';
case BUYER = 'buyer';
}
and then your factory will look like this:
<?php
namespace Database\Factories;
use App\Enums\UserTypeEnum;
use Illuminate\Database\Eloquent\Factories\Factory;
class TaskFactory extends Factory
{
public function definition()
{
return [
'firstName' => fake()->firstName,
'lastName' => fake()->lastName,
'username' => fake()->unique()->username,
'email' => fake()->unique()->safeEmail,
'password' => '$2y$10$92IXUNpkjO0rOQ5byMi.Ye4oKoEa3Ro9llC/.og/at2.uheWG/igi', // password,
'bio' => fake()->sentence(3, true),
'type' => fake()->randomElement(UserTypeEnum::cases()),
];
}
}
And also if your type column is nullable you can have your seeder type like fake()->randomElement([...UserTypeEnum::cases(), null]) as well.
Everytime I need to test method as authenticated user, I always insert role table because it has relationship with user table, then create new user.
Like this below code.
<?php
use Illuminate\Foundation\Testing\WithoutMiddleware;
use Illuminate\Foundation\Testing\DatabaseMigrations;
use Illuminate\Foundation\Testing\DatabaseTransactions;
use App\Models\User;
use App\Models\Role;
class CouponsTest extends TestCase
{
/**
* A basic test example.
*
* #return void
*/
use DatabaseMigrations;
//use WithoutMiddleware;
public function test_index_coupon(){
//factory(App\Models\Coupon::class, 5)->create()->toArray();
DB::table('roles')->insert([
['id' => 1, 'name' => 'customer'],
['id' => 2, 'name' => 'partner'],
['id' => 3, 'name' => 'admin'],
]);
$user = User::create([
'id' => 3,
'password' => bcrypt('123456789'),
'email' => 'admin#domain.co.id',
'role_id' => '3',
'status' => 'confirmed',
'balance' => 0,
]);
$this->actingAs($user)->visit('admin/coupons')->seePageIs('admin/coupons');
}
public function test_create_coupon()
{
DB::table('roles')->insert([
['id' => 1, 'name' => 'customer'],
['id' => 2, 'name' => 'partner'],
['id' => 3, 'name' => 'admin'],
]);
$user = User::create([
'id' => 3,
'full_name' => 'Admin Full Name',
'password' => bcrypt('123456789'),
'email' => 'admin#domain.co.id',
'role_id' => '3',
'status' => 'confirmed',
'balance' => 0,
]);
$this->actingAs($user)->visit('admin/coupons/create')->seePageIs('admin/coupons/create');
}
}
I know this is bad practice.
How should my code looks like to follow DRY principle?
It is common to use ModelFactory.php and define factories for your models.
You can also pass arguments to those models.
$factory->define(App\User::class, function (Faker\Generator $faker) {
static $password;
return [
'name' => $faker->name,
'email' => $faker->unique()->safeEmail,
'password' => $password ?: $password = bcrypt('secret'),
'remember_token' => str_random(10),
'role_id' => factory(App\Role::class)->create()->id,
];
});
$factory->define(App\Role::class, function(Faker\Generator $faker){
return [
'name' => $faker->name,
];
}
You can then do the following in your test:
$role = factory(App\Role::class)->create(['name' => 'admin']);
$adminUser = factory(App\User::class)->create(['role_id' => $role->id]);
Which if needed in many tests can be done once in your tests setup() method and define your admin user as protected variable of your test class .
Keep in mind you should probably also use DatabaseTransactions; trait in your test class in order to remove any entries created by the commands above at the end of your tests.