Opening multiple profiles, scraping from each profile

How to?
#1

Hello,

I am new to this, I am attempting to scrape a directory of each user profile and the details inside the profile page.

Ideally, I would like the script to open each profile, then scrape the details. Then go back to the root URL and select the next profile, scrape that profile, go back to root and so on.

{"_id":"Dulaa","startUrl":["https://doulamatch.net/list/birth/ca"],"selectors":[{"id":"Full Profile","linkType":"linkFromHref","multiple":true,"parentSelectors":["_root"],"selector":"div.col-lg-6:nth-of-type(1) .ml-auto a","type":"SelectorLink"},{"id":"full_name","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":".flex-grow-1 h3","type":"SelectorText"},{"id":"company_name","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"p.business","type":"SelectorText"},{"id":"location","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":".flex-grow-1 p.location","type":"SelectorText"},{"id":"website_url","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"p.profile-web:nth-of-type(3) a","type":"SelectorText"},{"id":"bio","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"p:nth-of-type(9)","type":"SelectorText"},{"id":"years_experiance","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"div.row:nth-of-type(3) .col-md p:nth-of-type(1)","type":"SelectorText"},{"id":"Location-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":".flex-grow-1 p.location","type":"SelectorText"},{"id":"Website-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"p.profile-web:nth-of-type(3) a","type":"SelectorText"},{"id":"Instagram-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"p.profile-web:nth-of-type(4) a","type":"SelectorText"},{"id":"Birth Fee-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":".col-md p.fee:nth-of-type(2)","type":"SelectorText"},{"id":"Postpartum Rate-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":".col-md p:nth-of-type(4)","type":"SelectorText"},{"id":"Availability Remarks-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":".flex-grow-1 p:nth-of-type(5)","type":"SelectorText"},{"id":"Medicaid and Third Party Insurers-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"li a.strongLink","type":"SelectorText"},{"id":"Postpartum Doula Experience-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"div.col-md:nth-of-type(1) p:nth-of-type(2)","type":"SelectorText"},{"id":"Birth Doula Certifications-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"ul.cert:nth-of-type(1) li","type":"SelectorText"},{"id":"Postpartum Doula Certifications-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"ul.cert:nth-of-type(2) li","type":"SelectorText"},{"id":"Doula Training-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":".tab-pane > ul.check li","type":"SelectorText"},{"id":"College Education-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"div:nth-of-type(5) .col-md p","type":"SelectorText"},{"id":"Special Services Offered-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"div:nth-of-type(6) ul","type":"SelectorText"},{"id":"Testimonials Count-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"#calendar-postpartum div:nth-of-type(15) tr:nth-of-type(5) td:nth-of-type(1) span","type":"SelectorText"},{"id":"Certifications Count-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"#calendar-postpartum div:nth-of-type(15) tr:nth-of-type(2) td:nth-of-type(1) span","type":"SelectorText"},{"id":"image-0","multipleType":"singleColumnWithSeparator","parentSelectors":["Full Profile"],"selector":"img.profile-img","type":"SelectorImage"},{"id":"image-1","multipleType":"singleColumnWithSeparator","parentSelectors":["Full Profile"],"selector":".more-thumb img","type":"SelectorImage"},{"id":"image-2","multipleType":"singleColumnWithSeparator","parentSelectors":["Full Profile"],"selector":".cert-thumb img","type":"SelectorImage"}]}
#2

Hi,

Sure! That can be set up. Please see the reference below on how to set up the scraper to open all links and perform the pagination:

{"_id":"Dulaa","startUrl":["https://doulamatch.net/list/birth/ca"],"selectors":[{"id":"Full Profile","linkType":"linkFromHref","multiple":true,"parentSelectors":["pagination"],"selector":".ml-auto a","type":"SelectorLink"},{"id":"full_name","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":".flex-grow-1 h3","type":"SelectorText"},{"id":"company_name","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"p.business","type":"SelectorText"},{"id":"location","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":".flex-grow-1 p.location","type":"SelectorText"},{"id":"website_url","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"p.profile-web:nth-of-type(3) a","type":"SelectorText"},{"id":"bio","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"p:nth-of-type(9)","type":"SelectorText"},{"id":"years_experiance","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"div.row:nth-of-type(3) .col-md p:nth-of-type(1)","type":"SelectorText"},{"id":"Location-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":".flex-grow-1 p.location","type":"SelectorText"},{"id":"Website-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"p.profile-web:nth-of-type(3) a","type":"SelectorText"},{"id":"Instagram-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"p.profile-web:nth-of-type(4) a","type":"SelectorText"},{"id":"Birth Fee-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":".col-md p.fee:nth-of-type(2)","type":"SelectorText"},{"id":"Postpartum Rate-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":".col-md p:nth-of-type(4)","type":"SelectorText"},{"id":"Availability Remarks-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":".flex-grow-1 p:nth-of-type(5)","type":"SelectorText"},{"id":"Medicaid and Third Party Insurers-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"li a.strongLink","type":"SelectorText"},{"id":"Postpartum Doula Experience-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"div.col-md:nth-of-type(1) p:nth-of-type(2)","type":"SelectorText"},{"id":"Birth Doula Certifications-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"ul.cert:nth-of-type(1) li","type":"SelectorText"},{"id":"Postpartum Doula Certifications-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"ul.cert:nth-of-type(2) li","type":"SelectorText"},{"id":"Doula Training-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":".tab-pane > ul.check li","type":"SelectorText"},{"id":"College Education-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"div:nth-of-type(5) .col-md p","type":"SelectorText"},{"id":"Special Services Offered-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"div:nth-of-type(6) ul","type":"SelectorText"},{"id":"Testimonials Count-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"#calendar-postpartum div:nth-of-type(15) tr:nth-of-type(5) td:nth-of-type(1) span","type":"SelectorText"},{"id":"Certifications Count-0","multiple":false,"multipleType":"singleColumn","parentSelectors":["Full Profile"],"regex":"","selector":"#calendar-postpartum div:nth-of-type(15) tr:nth-of-type(2) td:nth-of-type(1) span","type":"SelectorText"},{"id":"image-0","multipleType":"singleColumnWithSeparator","parentSelectors":["Full Profile"],"selector":"img.profile-img","type":"SelectorImage"},{"id":"image-1","multipleType":"singleColumnWithSeparator","parentSelectors":["Full Profile"],"selector":".more-thumb img","type":"SelectorImage"},{"id":"image-2","multipleType":"singleColumnWithSeparator","parentSelectors":["Full Profile"],"selector":".cert-thumb img","type":"SelectorImage"},{"id":"pagination","paginationType":"auto","parentSelectors":["_root","pagination"],"selector":"#top-pager a:contains('Next')","type":"SelectorPagination"}]}