I am trying to collect data from all 162 pages of the site linked below. Right now, my pagination is grabbing pages 1-9 and page 161 only, and I can't seem to set a selector to cycle through all the pages. Any help is greatly appreciated as this is my first experience with data scraping.
Url: https://bcorporation.net/directory?sort_by=search_api_aggregation_1&sort_order=ASC
Sitemap:
{"_id":"blab_cert","startUrl":["https://bcorporation.net/directory?sort_by=search_api_aggregation_1&sort_order=ASC"],"selectors":[{"id":"company link","type":"SelectorLink","selector":"div.card__inner a","parentSelectors":["_root","company link","Pagination"],"multiple":true,"delay":0},{"id":"Company Name","type":"SelectorText","selector":"h1.serif","parentSelectors":["company link"],"multiple":false,"regex":"","delay":0},{"id":"certification date","type":"SelectorText","selector":"span.font-weight-bold","parentSelectors":["company link"],"multiple":false,"regex":"","delay":0},{"id":"location","type":"SelectorText","selector":"div.field.field-name-field-country div.field-item","parentSelectors":["company link"],"multiple":false,"regex":"","delay":0},{"id":"sector","type":"SelectorText","selector":"div.field.field-name-field-sector div.field-item","parentSelectors":["company link"],"multiple":false,"regex":"","delay":0},{"id":"website","type":"SelectorText","selector":"div.field-item a","parentSelectors":["company link"],"multiple":false,"regex":"","delay":0},{"id":"bio","type":"SelectorText","selector":"div.field-item p","parentSelectors":["company link"],"multiple":true,"regex":"","delay":0},{"id":"logo","type":"SelectorImage","selector":"div.field img","parentSelectors":["company link"],"multiple":false,"delay":0},{"id":"Governance Score","type":"SelectorText","selector":"div.card:nth-of-type(1) div.circle div","parentSelectors":["company link"],"multiple":false,"regex":"","delay":0},{"id":"community score","type":"SelectorText","selector":"div.card:nth-of-type(2) div.circle div","parentSelectors":["company link"],"multiple":false,"regex":"","delay":0},{"id":"Environment Score","type":"SelectorText","selector":"div.card:nth-of-type(3) div.circle div","parentSelectors":["company link"],"multiple":false,"regex":"","delay":0},{"id":"Customers Score","type":"SelectorText","selector":"div.card:nth-of-type(4) div.circle div","parentSelectors":["company link"],"multiple":false,"regex":"","delay":0},{"id":"Total Score","type":"SelectorText","selector":"span.text-grey-text","parentSelectors":["company link"],"multiple":false,"regex":"","delay":0},{"id":"Pagination","type":"SelectorLink","selector":"ul.pagination a","parentSelectors":["_root"],"multiple":true,"delay":0}]}