I finally got pagination to work correctly (I selected the ">" SVG rather than the LI) but due to the slow website, it is skipping to the next page before all of the products have loaded. How can I force the pagination to slow down or to wait for the final product elements to appear before moving to the next page? Putting a very long page load delay in the scrape parameters is making any scraping far too long.
Url: http://www,musgravemarket.ie
Sitemap:
{"_id":"musgrave","startUrl":["https://www.musgravemarketplace.ie/home"],"selectors":[{"clickElementSelector":"span.dropdown__item:nth(0) a","clickElementUniquenessType":"uniqueText","clickType":"clickOnce","delay":500,"discardInitialElements":"do-not-discard","id":"Click","multiple":false,"parentSelectors":["_root"],"selector":"body","type":"SelectorElementClick"},{"id":"Category","multiple":true,"parentSelectors":["_root"],"selector":"li.main-navigation-level1-item > span > a[href]:contains('Ambient')","type":"SelectorLink"},{"id":"Sub-category","multiple":true,"parentSelectors":["Category"],"selector":".filters-panel__container > ish-category-navigation > div > div > ul > li > ish-category-navigation > div > div > ul > li > a","type":"SelectorLink"},{"id":"Pagination","paginationType":"auto","parentSelectors":["Sub-category","Pagination"],"selector":".next svg","type":"SelectorPagination"},{"id":"Product-matrix","multiple":true,"parentSelectors":["Pagination"],"selector":"a.product-tile__title","type":"SelectorLink"},{"id":"Product-name","multiple":false,"parentSelectors":["Product-matrix"],"regex":"","selector":"span[itemprop='name']","type":"SelectorText"},{"id":"Product-SKU","multiple":false,"parentSelectors":["Product-matrix"],"regex":"","selector":".col-xs-12 span[itemprop='sku']","type":"SelectorText"},{"id":"Product-size","multiple":false,"parentSelectors":["Product-matrix"],"regex":"","selector":"span.mr-3","type":"SelectorText"},{"id":"Product-price","multiple":false,"parentSelectors":["Product-matrix"],"regex":"","selector":".price-container[itemprop='offers'] div.current-price","type":"SelectorText"},{"id":"Product-VAT","multiple":false,"parentSelectors":["Product-matrix"],"regex":"","selector":".product-id[ishvatable] span","type":"SelectorText"},{"id":"Product-image","multiple":false,"parentSelectors":["Product-matrix"],"selector":".product-main-image img","type":"SelectorImage"},{"id":"Product-description-html","multiple":false,"parentSelectors":["Product-matrix"],"regex":"","selector":"div.mat-expansion-panel-body","type":"SelectorHTML"},{"id":"Product-information","multiple":false,"parentSelectors":["Product-matrix"],"regex":"","selector":"div [class*=\"attribute__group ng-tns-c6\"]","type":"SelectorText"},{"id":"Product-stock","multiple":false,"parentSelectors":["Product-matrix"],"regex":"","selector":".stock-message span","type":"SelectorText"},{"id":"Product-sale-price","multiple":false,"parentSelectors":["Product-matrix"],"regex":"","selector":".price-container[itemprop='offers'] span.sale-price","type":"SelectorText"},{"id":"Sub-category-name","multiple":true,"parentSelectors":["Category"],"regex":"","selector":".filter-item .filter-item a","type":"SelectorText"},{"id":"Product-quantities","multiple":false,"parentSelectors":["Pagination"],"regex":"","selector":".pagination-total span","type":"SelectorText"}]}
