IndexedDB "updates" every browser restart and erases data - firefox

Question

I wrote a Firefox WebExtension that downloads data files from a website and uses IndexedDB to store/update the data. The .SQLite file that is created is ~2GB in size. Whenever I restart Firefox, the extension executes the onupgradeneeded event, even though I always use version "1". I create the database object stores and indexes in that event, so all my data ends up getting deleted.
The only time this doesn't happen is when I close Firefox while the data is being downloaded or stored. The next time I start Firefox, it does not execute the event (as should be the case). It then continues to update the database as it was programmed to do.
I installed the SQLite Manager extension in hopes that I could identify something causing the issue to the database, but nothing was obvious to me.
Here is part of my background script:
init().then(fetchData).then(addData).catch(dberror);
function init() {
req = indexedDB.open("db", 1);
req.onupgradeneeded = e => {
var name;
var key;
console.log("Upgrading database...", e.oldVersion, e.newVersion);
db = e.currentTarget.result;
var store = db.createObjectStore("db", { keyPath: "KEY" });
db.createObjectStore("version", { keyPath: "version" });
for (name in indexes) {
key = ...
store.createIndex(name, key);
};
};
return new Promise( (resolve, reject) => {
req.onsuccess = e => {
db = e.currentTarget.result;
db.onerror = dberror;
var cursor = db.transaction("MECs").objectStore("MECs").index("STATUS_DATE").openCursor(null, 'prev');
cursor.onsuccess = e => {
if (e.target.result) {
lastMod = e.target.result.key;
fileYear = lastMod.getFullYear();
}
else lastMod = new Date(startingfileYear, 0);
resolve(lastMod);
}
cursor.onerror = reject;
};
req.onerror = e => {
dberror(e);
reject(e);
}
});
}
function fetchData(param) {
// Get data based on the param and return it
return fetchFile(filename);
}
function addData(data) {
var trans = db.transaction("db", "readwrite");
var store = trans.objectStore("db");
var req;
var n = 0;
var data2 = [];
var addPromise;
trans.onerror = event => console.log("Error! Error! ", event.target.error);
trans.onabort = event => console.log("Abort! Abort! ", event.target.error);
data.forEach((row, index) => {
//process data here
data2 = ...
});
(function storeRegData(n) {
var row = data[n];
if (!row) return;
req = store.put(row);
req.onsuccess = event => {
numUpdated++;
storeRegData(++n);
}
req.onabort = event => console.log("Abort! Abort! ", event.target.error);
req.onerror = event => console.log("Error! Error! ", event.target.error);
})(0); // I'm storing one row at a time because the transaction is failing when I queue too many rows.
addPromise = fetchData(data2).then(
response => {
var trans2 = db.transaction("db", "readwrite");
var store2 = trans2.objectStore("db");
var req2;
response.forEach(row => {
req2 = store2.put(row);
req2.onsuccess = event => numUpdated++;
req2.onerror = console.log;
});
return new Promise((resolve, reject) => trans2.oncomplete = e => resolve(response));
},
console.log)
);
return new Promise((resolve, reject) => trans.oncomplete = e => {
if (noMoreData)
resolve(addPromise);
else if (moreData)
resolve( addPromise.then(fetchData).then(addData) );
});
}
And here is my manifest
{
"author": "Name",
"manifest_version": 2,
"name": "Extension",
"description": "Extension",
"version": "3.0",
"applications": {
"gecko": {
"strict_min_version": "50.0",
"id": "myID",
"update_url": "https://update.me"
}
},
"background": {
"scripts": [
"js/background.js"
]
},
"content_scripts": [
{
"matches": [ "https://match.me/*" ],
"js": [
"script.js"
],
"css": [
"style.css"
]
}
],
"icons": {
"48": "icon.png"
},
"options_ui": {
"page": "options.html"
},
"page_action": {
"browser_style": true,
"default_icon": {
"19": "icon-19.png",
"38": "icon-38.png"
},
"default_title": "Extension",
"default_popup": "popup.html"
},
"permissions": [
"https://web.address/*",
"downloads",
"notifications",
"storage",
"tabs",
"webRequest",
"webNavigation"
],
"web_accessible_resources": [
"pictures.png"
]
}
Why does Firefox think the database is at version 0 when I restart the browser? I can use the stored data after I download it, so why does it overwrite it on every restart? I could possibly do a workaround where I only create the store and indexes on extension installation or update, but that's not a solution to the actual issue.
UPDATE: I tried the following to no avail -
Close the database and re-open after storing each data file
Create a new object store for each data file
UPDATE 2: It appears this is related to a storage issue. Apparently, 2GB is the storage limit for non-persistent storage. In Firefox you can by-pass this by making the storage persistent with the following command:
indexedDB.open("db", { version: 1, storage: "persistent" })
See the bugzilla report here.
Unfortunately, when run from a background page, the popup asking for confirmation is not handled, so you can never acknowledge it. Supposedly, when Firefox 56 comes out, you'll be able to use the "unlimitedStorage" permission, which will by-pass the confirmation popup, so it should work from the background page.
Update 3: So it looks like the limit is actually ~1.5 GB. I just spent over a week re-coding the extension to create and use a different database for each year of data, making each database no larger than 150 MB. And still onupgradeneeded executes when I restart the browser and wipes all my data. If, however, I limit the total amount of data in all the databases to the above limit, it works. Unfortunately, I'm still in the same boat.
Does no one have any ideas?

Answer 1

As I mentioned in the updates to my question, there appears to be a limit of ~1.5GB for the "default" storage of indexedDB. Changing the storage to "persistent" will remove that limit. Because persistent storage currently requires user input, however, the database has to be opened from a window that can handle a UI response.
This can be done from the background script by creating a new window with browser.window.create() and opening the database from there. There are security restrictions that prevent inline scripts from running in the new page, so I had to link to local javascript files for that (i.e. <script src="db.js"></script>. I think you can also change the content security policy with a manifest instruction, but I didn't do that.
Hopefully, the unlimitedStorage permission will be supported in Firefox 56, which will remove the popup, allowing a persistent database to be created/accessed directly from the background script.