Web Scraper version: 0.3.3
Chrome version: 63.0.3239.84
OS: Windows 10 (Also tried on MacOS)
I have a Sitemap that I've been running for about a year that now fails after a couple of pages. I've extracted the portion that fails into it's own sitemap and tried to adjust a few things but still continues to fail. I also reduced the number of li elements per page to 50 and still fails after a few pages (Previously the sitemap was scraping 1000 li elements per page ( you can see the parameter on the URL). The sitemap has been failing since the switch to 0.3.2, don't recall any problems with it previous to that. Any assistance you could provide would greatly be appreciated.
Sitemap:
{"_id":"ck-buylist","startUrl":["https://www.cardkingdom.com/purchasing/mtg_singles?filter[ipp]=50&filter[sort]=edition&filter[search]=mtg_advanced&filter[category_id]=0&filter[foil]=1&filter[nonfoil]=1"],"selectors":[{"id":"ckTable","type":"SelectorElement","selector":"li.productItemWrapper","parentSelectors":["ckListing"],"multiple":true,"delay":""},{"id":"ckCardName","type":"SelectorText","selector":"span.productDetailTitle","parentSelectors":["ckTable"],"multiple":false,"regex":"","delay":""},{"id":"ckSetName","type":"SelectorText","selector":"div.productDetailSet","parentSelectors":["ckTable"],"multiple":false,"regex":".+(?= Foil \\()|.+(?=\\()","delay":""},{"id":"ckNumberWanted","type":"SelectorElementAttribute","selector":"input.maxQty","parentSelectors":["ckTable"],"multiple":false,"extractAttribute":"value","delay":""},{"id":"ckPrice_NM","type":"SelectorText","selector":"div.usdSellPrice","parentSelectors":["ckTable"],"multiple":false,"regex":"\\d+(.*)\\d\\d","delay":""},{"id":"ckChaseType","type":"SelectorText","selector":"div.foil","parentSelectors":["ckTable"],"multiple":false,"regex":"FOIL","delay":""},{"id":"ckPager","type":"SelectorLink","selector":"ul.pagination li a:contains('ยป')","parentSelectors":["ckListing"],"multiple":false,"delay":""},{"id":"ckListing","type":"SelectorElement","selector":"div.mainListing","parentSelectors":["_root","ckPager"],"multiple":false,"delay":"20000"}]}
Error Message:
18:42:37.063 background_script.js:1 {"url":"https://www.cardkingdom.com/purchasing/mtg_singles?filter[ipp]=50&filter[sort]=edition&filter[search]=mtg_advanced&filter[category_id]=0&filter[foil]=1&filter[nonfoil]=1","timestamp":1515033757,"level_name":"INFO","message":"Job execution started"}
18:42:38.007 background_script.js:24 GET http://192.168.0.175:5984/sitemap-data-ck-buylist/ 404 (Object Not Found)
(anonymous) @ background_script.js:24
Ct @ background_script.js:24
Dt @ background_script.js:24
At @ background_script.js:24
n @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
u @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
ue.execute @ background_script.js:24
ue.ready @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
n @ background_script.js:24
characterData (async)
r @ background_script.js:24
e.exports @ background_script.js:24
Pt @ background_script.js:24
se @ background_script.js:24
getSitemapDataDb @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
getSitemapData @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
getSitemapData @ background_script.js:24
chrome.runtime.onMessage.addListener @ background_script.js:24
EventImpl.dispatchToListener @ extensions::event_bindings:403
publicClassPrototype.(anonymous function) @ extensions::utils:138
EventImpl.dispatch_ @ extensions::event_bindings:387
EventImpl.dispatch @ extensions::event_bindings:409
publicClassPrototype.(anonymous function) @ extensions::utils:138
messageListener @ extensions::messaging:240
EventImpl.dispatchToListener @ extensions::event_bindings:403
publicClassPrototype.(anonymous function) @ extensions::utils:138
EventImpl.dispatch_ @ extensions::event_bindings:387
EventImpl.dispatch @ extensions::event_bindings:409
publicClassPrototype.(anonymous function) @ extensions::utils:138
dispatchOnMessage @ extensions::messaging:392
18:42:38.012 background_script.js:24 The above 404 is totally normal. PouchDB is just detecting if the remote exists.
18:43:20.934 background_script.js:1 {"timestamp":1515033801,"level_name":"PROFILE","message":"41053 DataExtractor getData"}
18:43:20.937 background_script.js:1 {"timestamp":1515033801,"level_name":"PROFILE","message":"43870 ms job executeion"}
18:43:21.008 background_script.js:1 {"url":"https://www.cardkingdom.com/purchasing/mtg_singles?filter[ipp]=50&filter[sort]=edition&filter[search]=mtg_advanced&filter[category_id]=0&filter[foil]=1&filter[nonfoil]=1&page=2","timestamp":1515033801,"level_name":"INFO","message":"Job execution started"}
18:44:04.667 background_script.js:1 {"timestamp":1515033845,"level_name":"PROFILE","message":"40951 DataExtractor getData"}
18:44:04.670 background_script.js:1 {"timestamp":1515033845,"level_name":"PROFILE","message":"43660 ms job executeion"}
18:44:04.745 background_script.js:1 {"url":"https://www.cardkingdom.com/purchasing/mtg_singles?filter[ipp]=50&filter[sort]=edition&filter[search]=mtg_advanced&filter[category_id]=0&filter[foil]=1&filter[nonfoil]=1&page=3","timestamp":1515033845,"level_name":"INFO","message":"Job execution started"}
18:44:48.396 background_script.js:1 {"timestamp":1515033888,"level_name":"PROFILE","message":"40977 DataExtractor getData"}
18:44:48.399 background_script.js:1 {"timestamp":1515033888,"level_name":"PROFILE","message":"43652 ms job executeion"}
18:44:48.461 background_script.js:1 {"url":"https://www.cardkingdom.com/purchasing/mtg_singles?filter[ipp]=50&filter[sort]=edition&filter[search]=mtg_advanced&filter[category_id]=0&filter[foil]=1&filter[nonfoil]=1&page=4","timestamp":1515033888,"level_name":"INFO","message":"Job execution started"}
18:45:32.042 background_script.js:1 {"timestamp":1515033932,"level_name":"PROFILE","message":"40960 DataExtractor getData"}
18:45:32.045 background_script.js:1 {"timestamp":1515033932,"level_name":"PROFILE","message":"43581 ms job executeion"}
18:45:32.104 background_script.js:1 {"url":"https://www.cardkingdom.com/purchasing/mtg_singles?filter[ipp]=50&filter[sort]=edition&filter[search]=mtg_advanced&filter[category_id]=0&filter[foil]=1&filter[nonfoil]=1&page=5","timestamp":1515033932,"level_name":"INFO","message":"Job execution started"}
18:46:15.748 background_script.js:1 {"timestamp":1515033976,"level_name":"PROFILE","message":"41155 DataExtractor getData"}
18:46:15.751 background_script.js:1 {"timestamp":1515033976,"level_name":"PROFILE","message":"43645 ms job executeion"}
18:46:15.817 background_script.js:1 {"url":"https://www.cardkingdom.com/purchasing/mtg_singles?filter[ipp]=50&filter[sort]=edition&filter[search]=mtg_advanced&filter[category_id]=0&filter[foil]=1&filter[nonfoil]=1&page=6","timestamp":1515033976,"level_name":"INFO","message":"Job execution started"}
18:47:09.163 background_script.js:1 {"timestamp":1515034029,"level_name":"PROFILE","message":"41088 DataExtractor getData"}
18:47:09.166 background_script.js:1 {"timestamp":1515034029,"level_name":"PROFILE","message":"53343 ms job executeion"}
18:47:09.231 background_script.js:1 {"url":"https://www.cardkingdom.com/purchasing/mtg_singles?filter[ipp]=50&filter[sort]=edition&filter[search]=mtg_advanced&filter[category_id]=0&filter[foil]=1&filter[nonfoil]=1&page=7","timestamp":1515034029,"level_name":"INFO","message":"Job execution started"}
18:47:54.990 background_script.js:1 {"timestamp":1515034075,"level_name":"PROFILE","message":"41132 DataExtractor getData"}
18:47:54.992 background_script.js:1 {"timestamp":1515034075,"level_name":"PROFILE","message":"45756 ms job executeion"}
18:47:55.056 background_script.js:1 {"url":"https://www.cardkingdom.com/purchasing/mtg_singles?filter[ipp]=50&filter[sort]=edition&filter[search]=mtg_advanced&filter[category_id]=0&filter[foil]=1&filter[nonfoil]=1&page=8","timestamp":1515034075,"level_name":"INFO","message":"Job execution started"}
18:48:38.866 background_script.js:1 {"timestamp":1515034119,"level_name":"PROFILE","message":"41001 DataExtractor getData"}
18:48:38.868 background_script.js:1 {"timestamp":1515034119,"level_name":"PROFILE","message":"43810 ms job executeion"}
18:48:38.943 background_script.js:1 {"url":"https://www.cardkingdom.com/purchasing/mtg_singles?filter[ipp]=50&filter[sort]=edition&filter[search]=mtg_advanced&filter[category_id]=0&filter[foil]=1&filter[nonfoil]=1&page=9","timestamp":1515034119,"level_name":"INFO","message":"Job execution started"}
18:49:32.480 background_script.js:1 {"timestamp":1515034172,"level_name":"PROFILE","message":"41058 DataExtractor getData"}
18:49:32.482 background_script.js:1 {"timestamp":1515034172,"level_name":"PROFILE","message":"53536 ms job executeion"}
18:49:32.555 background_script.js:1 {"url":"https://www.cardkingdom.com/purchasing/mtg_singles?filter[ipp]=50&filter[sort]=edition&filter[search]=mtg_advanced&filter[category_id]=0&filter[foil]=1&filter[nonfoil]=1&page=10","timestamp":1515034173,"level_name":"INFO","message":"Job execution started"}
18:50:16.340 background_script.js:1 {"timestamp":1515034216,"level_name":"PROFILE","message":"41167 DataExtractor getData"}
18:50:16.343 background_script.js:1 {"timestamp":1515034216,"level_name":"PROFILE","message":"43785 ms job executeion"}
18:50:16.408 background_script.js:1 {"url":"https://www.cardkingdom.com/purchasing/mtg_singles?filter[ipp]=50&filter[sort]=edition&filter[search]=mtg_advanced&filter[category_id]=0&filter[foil]=1&filter[nonfoil]=1&page=11","timestamp":1515034216,"level_name":"INFO","message":"Job execution started"}
18:50:31.476 background_script.js:1 {"timestamp":1515034231,"level_name":"WARNING","message":"request timed out. continuing"}
log @ background_script.js:1
warning @ background_script.js:1
(anonymous) @ background_script.js:24
setTimeout (async)
r @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
timeoutPromiseWithoutTimeoutError @ background_script.js:24
(anonymous) @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
openPage @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
openPage @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
handle @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
t @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
handle @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
t @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
handle @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
t @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
handle @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
t @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
handle @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
t @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
handle @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
t @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
handle @ background_script.js:24
(anonymous) @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
18:50:35.483 background_script.js:1 {"url":"https://www.cardkingdom.com/purchasing/mtg_singles?filter[ipp]=50&filter[sort]=edition&filter[search]=mtg_advanced&filter[category_id]=0&filter[foil]=1&filter[nonfoil]=1&page=11","error":"Error: timeout","timestamp":1515034235,"level_name":"WARNING","message":"connecting to content script timed out while waiting for page load"}
log @ background_script.js:1
warning @ background_script.js:1
(anonymous) @ background_script.js:24
s @ background_script.js:24
Promise rejected (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
openPage @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
openPage @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
handle @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
t @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
handle @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
t @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
handle @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
t @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
handle @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
t @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
handle @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
t @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
handle @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
t @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
handle @ background_script.js:24
(anonymous) @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
18:50:37.246 background_script.js:1 {"error":{"message":"Could not establish connection. Receiving end does not exist."},"timestamp":1515034237,"level_name":"ERROR","message":"Failed to send message to chrome tab"}
log @ background_script.js:1
error @ background_script.js:1
chrome.tabs.sendMessage.n @ background_script.js:24
sendResponseAndClearCallback @ extensions::messaging:417
disconnectListener @ extensions::messaging:435
EventImpl.dispatchToListener @ extensions::event_bindings:403
publicClassPrototype.(anonymous function) @ extensions::utils:138
EventImpl.dispatch_ @ extensions::event_bindings:387
EventImpl.dispatch @ extensions::event_bindings:409
publicClassPrototype.(anonymous function) @ extensions::utils:138
dispatchOnDisconnect @ extensions::messaging:378
18:50:37.398 background_script.js:1 {"url":"https://www.cardkingdom.com/purchasing/mtg_singles?filter[ipp]=50&filter[sort]=edition&filter[search]=mtg_advanced&filter[category_id]=0&filter[foil]=1&filter[nonfoil]=1&page=11","parentSelector":"ckPager","sitemapName":"ck-buylist","error":"connecting to content script timed out while waiting for page load","timestamp":1515034237,"level_name":"ERROR","message":"Job execution failed"}
log @ background_script.js:1
error @ background_script.js:1
(anonymous) @ background_script.js:24
s @ background_script.js:24
Promise rejected (async)
a @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
handle @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
t @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
handle @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
t @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
handle @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
t @ background_script.js:24
(anonymous) @ background_script.js:24
(anonymous) @ background_script.js:24
r @ background_script.js:24
handle @ background_script.js:24
(anonymous) @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
Promise resolved (async)
a @ background_script.js:24
u @ background_script.js:24
18:50:37.400 background_script.js:1 {"timestamp":1515034237,"level_name":"PROFILE","message":"20989 ms job executeion"}
18:50:37.401 background_script.js:1 {"timestamp":1515034237,"level_name":"INFO","message":"Syning storage because a job failed"}
18:50:37.403 background_script.js:1 {"timestamp":1515034237,"level_name":"INFO","message":"Scraper execution is finished"}