Hi All,
I am currently scraping this site and somehow, the scraping stops at 2000 data sets instead of 12,000. I have tried increasing the page reload delay and it does not help. Would really appreciate if someone can advise on this!
the sitemap is as follows below:
{"_id":"toko_aug_2","startUrl":["[https://www.tokopedia.com/search?ob=8&st=product&q=bosch&sc=4029&condition=1&goldmerchant=true&official=true","https://www.tokopedia.com/search?ob=8&st=product&q=bosch&sc=984&condition=1&goldmerchant=true&official=true"],"selectors":{"id":"element","type":"SelectorElementClick","parentSelectors":["_root"],"selector":"div._1a-JxuBv:nth-of-type(n+2)","multiple":true,"delay":"2000","clickElementSelector":"span:nth-of-type(8)a","clickType":"clickMore","discardInitialElements":"do-not-discard","clickElementUniquenessType":"uniqueText"},{"id":"name","type":"SelectorText","parentSelectors":["link"],"selector":"h1","multiple":false,"regex":"","delay":0},{"id":"price","type":"SelectorText","parentSelectors":["link"],"selector":"span[itemprop='price']","multiple":false,"regex":"","delay":0},{"id":"link","type":"SelectorLink","parentSelectors":["element"],"selector":"a","multiple":false,"delay":"0"},{"id":"product_id","type":"SelectorElementAttribute","parentSelectors":["link"],"selector":"input#product-id","multiple":false,"extractAttribute":"value","delay":0},{"id":"merchant","type":"SelectorText","parentSelectors":["link"],"selector":"span.shop-name","multiple":false,"regex":"","delay":0},{"id":"location","type":"SelectorText","parentSelectors":["link"],"selector":"span[itemprop='addressLocality']","multiple":false,"regex":"","delay":0},{"id":"cat1","type":"SelectorText","parentSelectors":["link"],"selector":"div.rvm-breadcrumb.mb-25 li:nth-of-type(3) a","multiple":false,"regex":"","delay":0},{"id":"cat2","type":"SelectorText","parentSelectors":["link"],"selector":"div.rvm-breadcrumb.mb-25 li:nth-of-type(5) a","multiple":false,"regex":"","delay":0},{"id":"cat3","type":"SelectorText","parentSelectors":["link"],"selector":"div.rvm-breadcrumb.mb-25 li:nth-of-type(7) a","multiple":false,"regex":"","delay":0},{"id":"no_transactions","type":"SelectorText","parentSelectors":["link"],"selector":"span.fw-600.all-transaction-count","multiple":false,"regex":"","delay":0},{"id":"success-rate","type":"SelectorText","parentSelectors":["link"],"selector":"span.fw-600.success-transaction-percent","multiple":false,"regex":"","delay":0},{"id":"items_sold","type":"SelectorText","parentSelectors":["link"],"selector":"div.rvm-product-info--item_value.item-sold-count","multiple":false,"regex":"","delay":0},{"id":"views","type":"SelectorText","parentSelectors":["link"],"selector":"div.rvm-product-info--item_value.view-count","multiple":false,"regex":"","delay":0},{"id":"reply_discussion","type":"SelectorText","parentSelectors":["link"],"selector":"div#shop-response-rate-talk.rvm-merchant-transaction div.description","multiple":false,"regex":"","delay":0},{"id":"reply_message","type":"SelectorText","parentSelectors":["link"],"selector":"span#footer-shop-response","multiple":false,"regex":"","delay":0},{"id":"price_confirmation","type":"SelectorText","parentSelectors":["link"],"selector":"div.rvm-price span:last-of-type","multiple":false,"regex":"","delay":0},{"id":"wishlist","type":"SelectorText","parentSelectors":["link"],"selector":"div.rvm-left-column--right div.rvm-button--wishlist.btn-wish-add span.rvm-wishlist-text","multiple":false,"regex":"","delay":0},{"id":"no_reviews","type":"SelectorText","parentSelectors":["link"],"selector":"span.review-count","multiple":false,"regex":"","delay":0},{"id":"5_stars","type":"SelectorText","parentSelectors":["link"],"selector":"tr.rate-quality:nth-of-type(1) div.ratingtotal","multiple":false,"regex":"","delay":0},{"id":"4_stars","type":"SelectorText","parentSelectors":["link"],"selector":"tr.rate-quality:nth-of-type(2) div.ratingtotal","multiple":false,"regex":"","delay":0},{"id":"3_stars","type":"SelectorText","parentSelectors":["link"],"selector":"tr.rate-quality:nth-of-type(3) div.ratingtotal","multiple":false,"regex":"","delay":0},{"id":"2_stars","type":"SelectorText","parentSelectors":["link"],"selector":"tr.rate-quality:nth-of-type(4) div.ratingtotal","multiple":false,"regex":"","delay":0},{"id":"1_star","type":"SelectorText","parentSelectors":["link"],"selector":"tr.rate-quality:nth-of-type(5) div.ratingtotal","multiple":false,"regex":"","delay":0}]}