After dec 21 update all sitemaps disappeared

Bug Reports
#21

Let me know if you want to expand somewhere specifically.

#22

Try this one:

var version = 5;
var openRequest = window.indexedDB.open("_pouch_scraper-sitemaps", version);
openRequest.onerror = function (event) {
	console.error("Database error: " + JSON.stringify(event));
};
openRequest.onupgradeneeded = function (event) {
	console.error("Database upgrade needed: " + JSON.stringify(event));
};
openRequest.onsuccess = function (event) {
	console.log("connected to database");
	let indexedDb = event.target.result;
	let transaction = indexedDb.transaction(["document-store", "by-sequence"], "readonly");
	let indexStore = transaction.objectStore("document-store");

	console.log("fetching sitemap index");
	let sitemapInfosRequest = indexStore.getAll();

	console.log("fetching sitemaps");
	let sitemapStore = transaction.objectStore("by-sequence");
	let sitemapsRequest = sitemapStore.getAll();
	sitemapsRequest.onsuccess = function(event) {

		console.log("fetchied sitemaps");
		let sitemaps = event.target.result;
		console.log(sitemaps);

		let sitemapRevs = {};

		for(let i in sitemaps) {
			let sitemap = sitemaps[i];
			let sitemapId = sitemap._id;
			let sitemapRev = sitemap._rev;
			if(sitemapRev === undefined) {
				continue;
			}

			let revId  = parseInt(sitemapRev.match(/^(\d+)\-/)[1]);

			if(sitemapRevs[sitemapId] === undefined) {
				sitemapRevs[sitemapId] = revId;
			}
			else if(revId > sitemapRevs[sitemapId]) {
				sitemapRevs[sitemapId] = revId;
			}
		}

		for(let i in sitemaps) {
			let sitemap = sitemaps[i];
			let sitemapRev = sitemap._rev;
			if(sitemapRev === undefined || sitemap._deleted === true) {
				continue;
			}
			let sitemapId = sitemap._id;
			let revId  = parseInt(sitemapRev.match(/^(\d+)\-/)[1]);

			if(sitemapRevs[sitemapId] === revId) {
				delete sitemap._rev;
				delete sitemap._doc_id_rev;
				let sitemapJSON = JSON.stringify(sitemap);
				console.log(sitemapJSON);
			}
		}

	};
	sitemapsRequest.onerror = function (event) {
		console.error("sitemaps error: " + JSON.stringify(event));
	};
};
1 Like
#23

Yay! I can see sitemaps printed! Will backup them as text files on pc and will try to import on fresh web scraper since doing it no yields no results (after I click import nothing happens).
Thank you so much

#24

That's good yo hear! It seems that you were the only one that experienced this after the update. Probably the internal database got somehow corrupted. I think this was caused by some other external factor.

#25

Strange since I'm using browser only for browsing ofc and scraping with your awesome scraper. Could other chrome extensions theoretically have something to do with this?

#26

Other extensions shouldn't have access web scrapers internal storage. They work in a separate sandboxes.

#27

I'm certain many more people other than just the OP have run into this problem. I've encountered it dozens of times. Often with simple enough scrapes that I just recreate them. I've far too many scrapes today that are too time consuming to replicate each time this happens.

I've hit the problem yet again today. I found this post which has indeed helped me find my sitemaps in the console. My pertinent followup question though is - what is the best way of getting the sitemaps out of the console view and into a backup? If it's a matter of manually copy/pasting each sitemap then my question is how do I do so reliably - as the console display is a tree hierarchy that is in a different order than how the sitemaps are constructed within the extension itself.

#28

@midwest81 Can you post your issue in a separate bug report. We will also need error messages from the background script.

At the moment I can tell that We have found another case when something like this might happen. Basically importing a large sitemap will cause chrome storage to fail. It seems that chrome might have recently added a limitation. We have already implemented a sitemap size limitation feature but it hasn't yet been published in chrome store.

#29