I have been using my sitemaps successfully for many months, but recently some have stopped working properly.
I think it may be something to do with the size of the return data set...?
On the page being scraped, there is a control to set the number of rows of a table of data to be displayed.
When set to '100', all data is scraped successfully.
When set to 'all' (=1800+), nothing is returned.
I've tried altering the 'request interval', and 'page load delay' without success.
'Data Preview' on the sitemap still appears to work, returning all the data, so for the time being, I am able to use this, and then cut and paste the returned data into an .xls, but this is rather tedious, and not as intended.
OS: Windows 10
Chrome: Version 63.0.3239.108 (Official Build) (64-bit)
webScraper 0.3.3
The sitemap needs a private login to access the page, so you would need to create a dummy account if you wanted to access the page to check the code.
{"_id":"fs-active-and-past",
"startUrl":"https://www.fundingsecure.com/myaccount/past-loans",
"selectors":[
{"id":"tbl","type":"SelectorElement","selector":"tbody","parentSelectors":["_root"],"multiple":false,"delay":""},
{"id":"trow","type":"SelectorElement","selector":"tr","parentSelectors":["tbl"],"multiple":true,"delay":""},
{"id":"ref","type":"SelectorText","selector":"td:nth-of-type(1)","parentSelectors":["trow"],"multiple":false,"regex":"","delay":""},
{"id":"invOverview","type":"SelectorElementAttribute","selector":"button.btn","parentSelectors":["trow"],"multiple":false,"extractAttribute":"data-content","delay":""},
{"id":"title","type":"SelectorText","selector":"td:nth-of-type(2)","parentSelectors":["trow"],"multiple":false,"regex":"","delay":""},
{"id":"amount","type":"SelectorText","selector":"td:nth-of-type(3)","parentSelectors":["trow"],"multiple":false,"regex":"","delay":""},
{"id":"rate","type":"SelectorText","selector":"td:nth-of-type(4)","parentSelectors":["trow"],"multiple":false,"regex":"","delay":""},
{"id":"dateActive","type":"SelectorText","selector":"td:nth-of-type(5)","parentSelectors":["trow"],"multiple":false,"regex":"","delay":""},
{"id":"defaultStatus","type":"SelectorText","selector":"td:nth-of-type(6)","parentSelectors":["trow"],"multiple":false,"regex":"","delay":""},
{"id":"loanStatus","type":"SelectorText","selector":"td:nth-of-type(7)","parentSelectors":["trow"],"multiple":false,"regex":"","delay":""},
{"id":"lastUpdated","type":"SelectorText","selector":"td:nth-of-type(8)","parentSelectors":["trow"],"multiple":false,"regex":"","delay":""}]}
background_script.js:24 initial configuration i
background_script.js:24 initializing Background Script message listener
background_script.js:1 {"url":"https://www.fundingsecure.com/myaccount/past-loans","timestamp":1514481772,"level_name":"INFO","message":"Job execution started"}
background_script.js:1 {"timestamp":1514481787,"level_name":"WARNING","message":"request timed out. continuing"}
log @ background_script.js:1
background_script.js:1 {"error":{"message":"Could not establish connection. Receiving end does not exist."},"timestamp":1514481787,"level_name":"ERROR","message":"Failed to send message to chrome tab"}
log @ background_script.js:1
background_script.js:1 {"url":"https://www.fundingsecure.com/myaccount/past-loans","error":"Failed to send message to chrome tab. [object Object]","timestamp":1514481787,"level_name":"WARNING","message":"connecting to content script timed out while waiting for page load"}
log @ background_script.js:1
background_script.js:1 {"url":"https://www.fundingsecure.com/myaccount/past-loans","parentSelector":"_root","sitemapName":"fs-active-and-past","error":"connecting to content script timed out while waiting for page load","timestamp":1514481787,"level_name":"ERROR","message":"Job execution failed"}
log @ background_script.js:1
background_script.js:1 {"timestamp":1514481787,"level_name":"PROFILE","message":"15484 ms job executeion"}
background_script.js:1 {"timestamp":1514481787,"level_name":"INFO","message":"Syning storage because a job failed"}
background_script.js:1 {"timestamp":1514481792,"level_name":"INFO","message":"Scraper execution is finished"}