I've configured webscraper to get data from a website where the pagination will need that the user click on a button. I've done a test extraction and I've noticed that not all the data are extracted, I will get only 2000 rows in the generated excel, but after a quick control, I've calculated that I will have 4400 products into the website. How I can set correctly the pagination click and the delay to get all the data I need?
Url: Bilans GES
Sitemap:
{"_id":"ademe","startUrl":["Bilans GES NAF","parentSelectors":["company_link"],"type":"SelectorText","selector":"div:nth-of-type(3) span.sc-identity-sheet-item__info-value","multiple":false,"regex":""},{"id":"Département","parentSelectors":["company_link"],"type":"SelectorText","selector":"div:nth-of-type(5) span.sc-identity-sheet-item__info-value","multiple":false,"regex":""},{"id":"Contact","parentSelectors":["company_link"],"type":"SelectorText","selector":"div:nth-of-type(9) span.sc-identity-sheet-item__info-value","multiple":false,"regex":""},{"id":"Salariés","parentSelectors":["company_link"],"type":"SelectorText","selector":"div:nth-of-type(4) span.sc-identity-sheet-item__info-value","multiple":false,"regex":""},{"id":"Bilan_carbone","parentSelectors":["Transition plan"],"type":"SelectorLink","selector":"[title='Tableau de déclaration'] a","multiple":false},{"id":"company_link","parentSelectors":["_root","Page"],"type":"SelectorLink","selector":"a.gouv-btn--flat","multiple":false},{"id":"Emissions","parentSelectors":["Bilan_carbone"],"type":"SelectorText","selector":"div.gouv-tabs__panel--selected","multiple":false,"regex":""},{"id":"Page","parentSelectors":["_root"],"type":"SelectorElementClick","clickElementSelector":".sc-home-inventories__paginator button.ri-arrow-right-s-line","clickElementUniquenessType":"uniqueCSSSelector","clickType":"clickMore","delay":2000,"discardInitialElements":"do-not-discard","multiple":true,"selector":"app-inventory-item"},{"id":"SIREN","parentSelectors":["company_link"],"type":"SelectorText","selector":"span:nth-of-type(6)","multiple":false,"regex":""},{"id":"Reporting","parentSelectors":["company_link"],"type":"SelectorText","selector":"div:nth-of-type(7) span.sc-identity-sheet-item__info-value","multiple":false,"regex":""},{"id":"Transition plan","parentSelectors":["company_link"],"type":"SelectorLink","selector":"[title='Plan de transition'] a","multiple":false},{"id":"Emissions directes","parentSelectors":["Transition plan"],"type":"SelectorText","selector":".sc-transition-plan-consultation__emissions div:nth-of-type(1)","multiple":false,"regex":""},{"id":"Emissions indirectes","parentSelectors":["Transition plan"],"type":"SelectorText","selector":"div.ng-star-inserted:nth-of-type(2)","multiple":false,"regex":""}]}