Google Client API setAccessToken() before isAccessTokenExpired() results in invalid credentials - laravel

Question

I am working with the Google Client API in Laravel to allow my users to sync their calendars with Google. Everything works, but the issue I am running into is when their tokens expire they are getting an "Invalid Credentials" error, in order to fix it they have to log out and log back in which I am trying to avoid.
I don't understand why setAccessToken() is to be called before isAccessTokenExpired().
I need to check if the access token is expired before I set it but if I do it this way then isAccessTokenExpired() always returns true.
Any ideas would be helpful. Thanks!
Here is my code:
GoogeServiceController.php
class GoogleServiceController extends Controller
{
protected $client;
protected $service;
public function __construct()
{
$client = new Google_Client();
$client->setAuthConfig(Config::get('google_config.web'));
$client->setAccessType('offline');
$client->addScope(Google_Service_Calendar::CALENDAR);
$service = new Google_Service_Calendar($client);
$this->client = $client;
$this->service = $service;
}
public function oauth(Request $request)
{
if (App::environment('local')) {
$this->client->setRedirectUri('http://esm.development.com/oauth');
} else {
$this->client->setRedirectUri('https://essentialstudiomanager.com/oauth');
}
if (is_null($request->user()->refresh_token)) {
$this->client->setApprovalPrompt("force");
}
if (!$request->has('code')) {
$auth_url = $this->client->createAuthUrl();
$filtered_url = filter_var($auth_url, FILTER_SANITIZE_URL);
return redirect($filtered_url);
} else {
$this->client->authenticate($request->code);
if (is_null($request->user()->refresh_token)) {
$refresh_token = $this->client->getRefreshToken();
$user = $request->user();
$user->refresh_token = $refresh_token;
$user->save();
}
$request->session()->put('access_token', $this->client->getAccessToken());
$notification = ['message' => 'Your calendar is now synced with your Google Calendar.', 'alert-type' => 'success'];
return redirect()->route('home')->with($notification);
}
}
}
GoogleEventController.php
public function updateGoogleEvent($request, $event, $title, $description, $start, $end)
{
if ($request->session()->has('access_token')) {
$this->client->setAccessToken(session('access_token'));
if ($this->client->isAccessTokenExpired()) {
$this->client->refreshToken($request->user()->refresh_token);
$request->session()->put('access_token', $this->client->getAccessToken());
$this->client->setAccessToken(session('access_token'));
}
} else {
return redirect()->route('oauthCallBack');
}
$users_calendar = $this->service->calendars->get('primary');
$get_event = $this->service->events->get('primary', $event->google_event_id);
$get_event->setSummary($title);
$get_event->setDescription($description);
$start_date = new Google_Service_Calendar_EventDateTime();
$start_date->setDateTime($start);
$start_date->setTimeZone($users_calendar->timeZone);
$get_event->setStart($start_date);
$end_date = new Google_Service_Calendar_EventDateTime();
$end_date->setDateTime($end);
$end_date->setTimeZone($users_calendar->timeZone);
$get_event->setEnd($end_date);
$updatedEvent = $this->service->events->update('primary', $get_event->getId(), $get_event);
}