I want to scrape all the results from the url below. It loads more data by clicking the load more button. The problem is how to know if all the results are scrapped and none is missed? because every time i run the scrapper, the number of returned results changes
Url: https://www.bloomberg.com/search?query=diesel
Sitemap:
{"_id":"bloomberg","startUrl":["https://www.bloomberg.com/search?query=diesel"],"selectors":[{"id":"wrappers","type":"SelectorElementClick","parentSelectors":["_root"],"selector":"div.text__1793994f","multiple":true,"delay":"10000","clickElementSelector":".loadMoreButtonContainer__e27276de button","clickType":"clickMore","discardInitialElements":"do-not-discard","clickElementUniquenessType":"uniqueText"},{"id":"Date","type":"SelectorText","parentSelectors":["wrappers"],"selector":"div.publishedAt__79f8aaad","multiple":false,"regex":"","delay":0},{"id":"Title","type":"SelectorText","parentSelectors":["wrappers"],"selector":"a.headline__55bd5397","multiple":false,"regex":"","delay":0},{"id":"Description","type":"SelectorText","parentSelectors":["wrappers"],"selector":"a.summary__bbda15b4","multiple":false,"regex":"","delay":0},{"id":"Category","type":"SelectorText","parentSelectors":["wrappers"],"selector":"div.eyebrow__4b7f0542","multiple":false,"regex":"","delay":0}]}