Not scraping all links

cidreno · September 2, 2022, 9:46pm

Describe the problem.

My sitemap is scraping only the first 9 links on the page, although there are over 30 links. I had the request interval and page load delay set at 1000ms.

Any idea why this is happening and how I can solve it?

Url: PR Real Estate - Puerto Rico Homes For Sale | Zillow

Sitemap:
{"_id":"zillowHomesSoldPage1","startUrl":["https://www.zillow.com/pr/sold"],"selectors":[{"id":"Link","parentSelectors":["_root"],"type":"SelectorLink","selector":".photo-cards_extra-attribution a.list-card-img","multiple":true,"delay":0},{"id":"GenInfoBedrooms","parentSelectors":["Link"],"type":"SelectorText","selector":".hdp__sc-x83qu3-0 span > span.Text-c11n-8-65-2__sc-aiai24-0:nth-of-type(1)","multiple":false,"delay":0,"regex":""},{"id":"GenInfoBathrooms","parentSelectors":["Link"],"type":"SelectorText","selector":".TooltipPopper-c11n-8-65-2__sc-io290n-0 span.Text-c11n-8-65-2__sc-aiai24-0","multiple":false,"delay":0,"regex":""},{"id":"GenInfoSqft","parentSelectors":["Link"],"type":"SelectorText","selector":"span:nth-of-type(4)","multiple":false,"delay":0,"regex":""},{"id":"GenInfoAddress","parentSelectors":["Link"],"type":"SelectorText","selector":"h1","multiple":false,"delay":0,"regex":""},{"id":"GenInfoSaleType","parentSelectors":["Link"],"type":"SelectorText","selector":"span.dpf__sc-1yftt2a-1","multiple":false,"delay":0,"regex":""},{"id":"GenInfoSellPrice","parentSelectors":["Link"],"type":"SelectorText","selector":".hdp__sc-ym74hh-0 strong","multiple":false,"delay":0,"regex":""},{"id":"GenInfoSoldOn","parentSelectors":["Link"],"type":"SelectorText","selector":"span.hdp__sc-ym74hh-0","multiple":false,"delay":0,"regex":""},{"id":"Overview","parentSelectors":["Link"],"type":"SelectorText","selector":"div.sc-kfPuZi","multiple":false,"delay":0,"regex":""},{"id":"FactsType","parentSelectors":["Link"],"type":"SelectorText","selector":"li.dpf__sc-2arhs5-0:nth-of-type(1)","multiple":false,"delay":0,"regex":""},{"id":"FactsYearBuilt","parentSelectors":["Link"],"type":"SelectorText","selector":"li.dpf__sc-2arhs5-0:nth-of-type(2)","multiple":false,"delay":0,"regex":""},{"id":"FactsHeating","parentSelectors":["Link"],"type":"SelectorText","selector":"li.dpf__sc-2arhs5-0:nth-of-type(3)","multiple":false,"delay":0,"regex":""},{"id":"FactsCooling","parentSelectors":["Link"],"type":"SelectorText","selector":"li.dpf__sc-2arhs5-0:nth-of-type(4)","multiple":false,"delay":0,"regex":""},{"id":"FactsParking","parentSelectors":["Link"],"type":"SelectorText","selector":"li.dpf__sc-2arhs5-0:nth-of-type(5)","multiple":false,"delay":0,"regex":""},{"id":"FactsLotSize","parentSelectors":["Link"],"type":"SelectorText","selector":"li.dpf__sc-2arhs5-0:nth-of-type(6)","multiple":false,"delay":0,"regex":""},{"id":"InteriorBedrooms","parentSelectors":["Link"],"type":"SelectorText","selector":"div.kHzFzO:nth-of-type(1) div.dpf__sc-1j9xcg4-0:nth-of-type(1) li:nth-of-type(1) span","multiple":false,"delay":0,"regex":""},{"id":"InteriorBathrooms","parentSelectors":["Link"],"type":"SelectorText","selector":"div.kHzFzO:nth-of-type(1) div.dpf__sc-1j9xcg4-0:nth-of-type(1) li:nth-of-type(2) span","multiple":false,"delay":0,"regex":""},{"id":"InteriorFullBathrooms","parentSelectors":["Link"],"type":"SelectorText","selector":"div.kHzFzO:nth-of-type(1) li:nth-of-type(3) span","multiple":false,"delay":0,"regex":""},{"id":"InteriorFlooring","parentSelectors":["Link"],"type":"SelectorText","selector":"div.kHzFzO:nth-of-type(1) div:nth-of-type(2) span","multiple":false,"delay":0,"regex":""},{"id":"InteriorHeating","parentSelectors":["Link"],"type":"SelectorText","selector":"div.kHzFzO:nth-of-type(1) div:nth-of-type(3) span","multiple":false,"delay":0,"regex":""},{"id":"InteriorCooling","parentSelectors":["Link"],"type":"SelectorText","selector":"div.kHzFzO:nth-of-type(1) div:nth-of-type(4) span","multiple":false,"delay":0,"regex":""},{"id":"InteriorAppliances","parentSelectors":["Link"],"type":"SelectorText","selector":"div.dpf__sc-1j9xcg4-0:nth-of-type(5) span","multiple":false,"delay":0,"regex":""},{"id":"InteriorTotalArea","parentSelectors":["Link"],"type":"SelectorText","selector":"div.dpf__sc-1j9xcg4-0:nth-of-type(6) li:nth-of-type(1) span","multiple":false,"delay":0,"regex":""},{"id":"InteriorAttic","parentSelectors":["Link"],"type":"SelectorText","selector":"div.dpf__sc-1j9xcg4-0:nth-of-type(6) li:nth-of-type(2) span","multiple":false,"delay":0,"regex":""},{"id":"DetailsTotalParking","parentSelectors":["Link"],"type":"SelectorText","selector":"div.Spacer-c11n-8-65-2__sc-17suqs2-0:nth-of-type(2) div.dpf__sc-1j9xcg4-0:nth-of-type(1) li:nth-of-type(1) span","multiple":false,"delay":0,"regex":""},{"id":"DetailsParkingFeatures","parentSelectors":["Link"],"type":"SelectorText","selector":"div.Spacer-c11n-8-65-2__sc-17suqs2-0:nth-of-type(2) div.dpf__sc-1j9xcg4-0:nth-of-type(1) li:nth-of-type(2) span","multiple":false,"delay":0,"regex":""},{"id":"DetailsCoveredSpaces","parentSelectors":["Link"],"type":"SelectorText","selector":"div.Spacer-c11n-8-65-2__sc-17suqs2-0:nth-of-type(2) div.dpf__sc-1j9xcg4-0:nth-of-type(1) li:nth-of-type(3) span","multiple":false,"delay":0,"regex":""},{"id":"DetailsCarportSpaces","parentSelectors":["Link"],"type":"SelectorText","selector":"li.ListItem-c11n-8-65-2__sc-10e22w8-0:nth-of-type(4) span","multiple":false,"delay":0,"regex":""},{"id":"DetailsLotSize","parentSelectors":["Link"],"type":"SelectorText","selector":"div.Spacer-c11n-8-65-2__sc-17suqs2-0:nth-of-type(2) div:nth-of-type(3) span","multiple":false,"delay":0,"regex":""},{"id":"DetailsParcelNumber","parentSelectors":["Link"],"type":"SelectorText","selector":"div.Spacer-c11n-8-65-2__sc-17suqs2-0:nth-of-type(2) div:nth-of-type(4) li:nth-of-type(1) span","multiple":false,"delay":0,"regex":""},{"id":"DetailsZoning","parentSelectors":["Link"],"type":"SelectorText","selector":"div.dpf__sc-1j9xcg4-0:nth-of-type(4) li:nth-of-type(2) span","multiple":false,"delay":0,"regex":""},{"id":"DetailsSpecialConditions","parentSelectors":["Link"],"type":"SelectorText","selector":"div.dpf__sc-1j9xcg4-0:nth-of-type(4) li:nth-of-type(3) span","multiple":false,"delay":0,"regex":""},{"id":"DetailsLelvels","parentSelectors":["Link"],"type":"SelectorText","selector":"div.Spacer-c11n-8-65-2__sc-17suqs2-0:nth-of-type(2) div:nth-of-type(2) li:nth-of-type(1) span","multiple":false,"delay":0,"regex":""},{"id":"DetailsExteriorFeatures","parentSelectors":["Link"],"type":"SelectorText","selector":"div.Spacer-c11n-8-65-2__sc-17suqs2-0:nth-of-type(2) div:nth-of-type(2) li:nth-of-type(2) span.Text-c11n-8-65-2__sc-aiai24-0","multiple":false,"delay":0,"regex":""},{"id":"ConstructionHomeType","parentSelectors":["Link"],"type":"SelectorText","selector":"div.Spacer-c11n-8-65-2__sc-17suqs2-0:nth-of-type(3) div.dpf__sc-1j9xcg4-0:nth-of-type(1) li:nth-of-type(1) span","multiple":false,"delay":0,"regex":""},{"id":"ConstructionPropertySubType","parentSelectors":["Link"],"type":"SelectorText","selector":"div.Spacer-c11n-8-65-2__sc-17suqs2-0:nth-of-type(3) div.dpf__sc-1j9xcg4-0:nth-of-type(1) li:nth-of-type(2) span","multiple":false,"delay":0,"regex":""},{"id":"ConstructionMaterials","parentSelectors":["Link"],"type":"SelectorText","selector":"div.Spacer-c11n-8-65-2__sc-17suqs2-0:nth-of-type(3) div:nth-of-type(2) li:nth-of-type(1) span","multiple":false,"delay":0,"regex":""},{"id":"ConstructionFoundation","parentSelectors":["Link"],"type":"SelectorText","selector":"div.Spacer-c11n-8-65-2__sc-17suqs2-0:nth-of-type(3) div:nth-of-type(2) li:nth-of-type(2) span","multiple":false,"delay":0,"regex":""},{"id":"ConstructionRoof","parentSelectors":["Link"],"type":"SelectorText","selector":"div.dpf__sc-1j9xcg4-0:nth-of-type(2) li:nth-of-type(3) span","multiple":false,"delay":0,"regex":""},{"id":"ConstructionCondition","parentSelectors":["Link"],"type":"SelectorText","selector":"div.Spacer-c11n-8-65-2__sc-17suqs2-0:nth-of-type(3) div:nth-of-type(3) li:nth-of-type(1) span","multiple":false,"delay":0,"regex":""},{"id":"ConstructionNew","parentSelectors":["Link"],"type":"SelectorText","selector":"div.dpf__sc-1j9xcg4-0:nth-of-type(3) li:nth-of-type(2) span","multiple":false,"delay":0,"regex":""},{"id":"ConstructionYearBuilt","parentSelectors":["Link"],"type":"SelectorText","selector":"div.dpf__sc-1j9xcg4-0:nth-of-type(3) li:nth-of-type(3) span","multiple":false,"delay":0,"regex":""},{"id":"UtilitiesSewer","parentSelectors":["Link"],"type":"SelectorText","selector":"div:nth-of-type(4) .dpf__sc-1j9xcg4-0 li:nth-of-type(1) span","multiple":false,"delay":0,"regex":""},{"id":"UtilitiesWater","parentSelectors":["Link"],"type":"SelectorText","selector":"div:nth-of-type(4) .dpf__sc-1j9xcg4-0 li:nth-of-type(2) span","multiple":false,"delay":0,"regex":""},{"id":"UtilitiesForProperty","parentSelectors":["Link"],"type":"SelectorText","selector":"div:nth-of-type(4) .dpf__sc-1j9xcg4-0 li:nth-of-type(3)","multiple":false,"delay":0,"regex":""},{"id":"CommunityFeatures","parentSelectors":["Link"],"type":"SelectorText","selector":"div:nth-of-type(5) div.dpf__sc-1j9xcg4-0:nth-of-type(1) span","multiple":false,"delay":0,"regex":""},{"id":"CommunityRegion","parentSelectors":["Link"],"type":"SelectorText","selector":"div:nth-of-type(5) div:nth-of-type(2) li:nth-of-type(1) span.Text-c11n-8-65-2__sc-aiai24-0","multiple":false,"delay":0,"regex":""},{"id":"CommunitySubdivision","parentSelectors":["Link"],"type":"SelectorText","selector":"div:nth-of-type(5) li:nth-of-type(2) span.Text-c11n-8-65-2__sc-aiai24-0","multiple":false,"delay":0,"regex":""},{"id":"HOAHasHOA","parentSelectors":["Link"],"type":"SelectorText","selector":"div.Spacer-c11n-8-65-2__sc-17suqs2-0:nth-of-type(6) div.dpf__sc-1j9xcg4-0:nth-of-type(1) li:nth-of-type(1) span","multiple":false,"delay":0,"regex":""},{"id":"HOAServicesIncluded","parentSelectors":["Link"],"type":"SelectorText","selector":"div.Spacer-c11n-8-65-2__sc-17suqs2-0:nth-of-type(6) li:nth-of-type(2) span","multiple":false,"delay":0,"regex":""},{"id":"HOAFinancialInformation","parentSelectors":["Link"],"type":"SelectorText","selector":"div.Spacer-c11n-8-65-2__sc-17suqs2-0:nth-of-type(6) div:nth-of-type(2) span","multiple":false,"delay":0,"regex":""},{"id":"OtherOwnership","parentSelectors":["Link"],"type":"SelectorText","selector":"div:nth-of-type(7) li:nth-of-type(1) span","multiple":false,"delay":0,"regex":""},{"id":"OtherRoadSurfaceType","parentSelectors":["Link"],"type":"SelectorText","selector":"div:nth-of-type(7) li:nth-of-type(2) span","multiple":false,"delay":0,"regex":""},{"id":"RealtorLisingProvidedBy","parentSelectors":["Link"],"type":"SelectorText","selector":"div.juevXs","multiple":false,"delay":0,"regex":""},{"id":"PriceChange","parentSelectors":["Link"],"type":"SelectorTable","delay":0,"multiple":true,"selector":"table","tableDataRowSelector":"tr.hdp__sc-f00yqe-3","tableHeaderRowSelector":"thead tr","columns":[{"extract":true,"header":"Date","name":"Date"},{"extract":true,"header":"Event","name":"Event"},{"extract":true,"header":"Price","name":"Price"}]}]}

ViestursWS · September 8, 2022, 1:36pm

@cidreno Hi, it appears that more properties are loading after scrolling down the page, therefore you would have to add an additional 'Element scroll' selector in order to load all of the available results.

Example:

{"_id":"zillow_milwaukee_wi","startUrl":["https://www.zillow.com/milwaukee-county-wi/rent-houses/","https://www.zillow.com/waukesha-county-wi/rent-houses/","https://www.zillow.com/washington-county-wi/rent-houses/","https://www.zillow.com/ozaukee-county-wi/rent-houses/","https://www.zillow.com/dane-county-wi/rent-houses/"],"selectors":[{"id":"pagination","paginationType":"clickMore","parentSelectors":["_root","pagination"],"selector":"[title=\"Next page\"]","type":"SelectorPagination"},{"delay":1000,"elementLimit":0,"id":"scroll","multiple":true,"parentSelectors":["pagination"],"scrollElementSelector":"div.search-page-list-container","selector":"div#grid-search-results li","type":"SelectorElementScroll"},{"id":"listing_links","multiple":true,"parentSelectors":["scroll"],"selector":"a[data-test=\"property-card-link\"]:nth(0)","type":"SelectorLink"},{"id":"address","multiple":false,"parentSelectors":["listing-page"],"regex":"","selector":".ds-data-col h1","type":"SelectorText"},{"id":"rent","multiple":false,"parentSelectors":["listing-page"],"regex":"","selector":"[data-testid=\"price\"]","type":"SelectorText"},{"id":"sqft","multiple":false,"parentSelectors":["listing-page"],"regex":"","selector":"[data-testid=\"bed-bath-item\"]:contains('sqft') strong","type":"SelectorText"},{"id":"beds","multiple":false,"parentSelectors":["listing-page"],"regex":"","selector":"[data-testid=\"bed-bath-item\"]:contains('bd') strong","type":"SelectorText"},{"id":"baths","multiple":false,"parentSelectors":["listing-page"],"regex":"","selector":"[data-testid=\"bed-bath-item\"]:contains('ba') strong","type":"SelectorText"},{"id":"age","multiple":false,"parentSelectors":["listing-page"],"regex":"","selector":".ds-overview > div > div > div > div > div:contains('Days listed') + div, .ds-overview h4:contains(\"Overview\") + div > div:nth(0):contains('days listed') div:contains('days listed') div","type":"SelectorText"},{"id":"contacts","multiple":false,"parentSelectors":["listing-page"],"regex":"","selector":".ds-overview > div > div > div > div > div:contains('Contacts') + div, .ds-overview h4:contains(\"Overview\") + div > div:nth(0):contains('contacts') div:contains('contacts') div","type":"SelectorText"},{"id":"applications","multiple":false,"parentSelectors":["listing-page"],"regex":"","selector":".ds-overview > div > div > div > div > div:contains('Applications') + div, .ds-overview h4:contains(\"Overview\") + div > div:nth(0):contains('applications') div:contains('applications') div","type":"SelectorText"},{"id":"listing-page","multiple":true,"parentSelectors":["listing_links"],"selector":"body","type":"SelectorElement"}]}

cidreno · September 9, 2022, 1:00am

Oh. I see. I'll look into it tomorrow. Thanks, so much, @ViestursWS!

cidreno · September 16, 2022, 2:15pm

I added an element scroll selector, and it worked but took far too long, so I just zoomed out in the webscraper window, and that picked up everything and went fast.