Scraping breadcrumbs

VolkerSchroeder · February 23, 2020, 11:59am

I want to scrape the different breadcrumbs for the different levels.
bread1, bread2 etc.
With preview its all ok, but with scraping i often becomes only "null" as the result.
But i dont know why?

Url: https://www.aniforte.de/

Sitemap:
{"_id":"aniforte_2020","startUrl":["https://www.aniforte.de"],"selectors":[{"id":"titel","type":"SelectorText","parentSelectors":["produktuebersicht","variantenclick"],"selector":"p[itemprop='name']","multiple":false,"regex":"","delay":0},{"id":"preis_vk","type":"SelectorText","parentSelectors":["produktuebersicht","variantenclick"],"selector":"span.price--content","multiple":false,"regex":"","delay":0},{"id":"artikelnummer","type":"SelectorText","parentSelectors":["produktuebersicht","variantenclick"],"selector":"span.entry--content","multiple":false,"regex":"","delay":0},{"id":"beschreibung","type":"SelectorText","parentSelectors":["produktuebersicht","variantenclick"],"selector":"div.product--description","multiple":false,"regex":"","delay":0},{"id":"menu_oben","type":"SelectorLink","parentSelectors":["_root"],"selector":"div.navigation--list-wrapper a.navigation--link","multiple":true,"delay":"0"},{"id":"produktuebersicht","type":"SelectorLink","parentSelectors":["menu_oben","pagination","menu_bilder"],"selector":".product--info a.product--image","multiple":true,"delay":0},{"id":"bild1","type":"SelectorElementAttribute","parentSelectors":["produktuebersicht","variantenclick"],"selector":"span.image--element","multiple":false,"extractAttribute":"data-img-large","delay":0},{"id":"Inhalt","type":"SelectorText","parentSelectors":["produktuebersicht","variantenclick"],"selector":"div.product--data-container div.product--price-info div.product--price","multiple":false,"regex":"","delay":0},{"id":"Menu_link","type":"SelectorText","parentSelectors":["produktuebersicht","variantenclick"],"selector":"nav.content--breadcrumb div.container","multiple":false,"regex":"","delay":0},{"id":"pagination","type":"SelectorElementClick","parentSelectors":["menu_oben","pagination","menu_bilder"],"selector":"div.listing--container","multiple":true,"delay":0,"clickElementSelector":"a.is--primary.is--icon-right","clickType":"clickOnce","discardInitialElements":"do-not-discard","clickElementUniquenessType":"uniqueText"},{"id":"variantenclick","type":"SelectorElementClick","parentSelectors":["produktuebersicht"],"selector":".content","multiple":true,"delay":"600","clickElementSelector":".product--configurator option","clickType":"clickOnce","discardInitialElements":"discard","clickElementUniquenessType":"uniqueText"},{"id":"bilder","type":"SelectorGroup","parentSelectors":["produktuebersicht","variantenclick"],"selector":"span.image--element","delay":0,"extractAttribute":"data-img-large"},{"id":"variante","type":"SelectorText","parentSelectors":["produktuebersicht","variantenclick"],"selector":".product--configurator .js--fancy-select-text","multiple":false,"regex":"","delay":0},{"id":"breadcrumb","type":"SelectorText","parentSelectors":["produktuebersicht","variantenclick"],"selector":"nav.content--breadcrumb","multiple":false,"regex":"","delay":0},{"id":"menu_bilder","type":"SelectorLink","parentSelectors":["menu_oben"],"selector":"a.category-teaser--link","multiple":true,"delay":0},{"id":"bread1","type":"SelectorText","parentSelectors":["produktuebersicht","variantenclick"],"selector":"nav.content--breadcrumb div","multiple":false,"regex":"","delay":0},{"id":"bread2","type":"SelectorText","parentSelectors":["produktuebersicht","variantenclick"],"selector":"li:nth-of-type(3) span.breadcrumb--title","multiple":false,"regex":"","delay":0},{"id":"bread3","type":"SelectorText","parentSelectors":["produktuebersicht","variantenclick"],"selector":"li:nth-of-type(5) span.breadcrumb--title","multiple":false,"regex":"","delay":0},{"id":"bread4","type":"SelectorText","parentSelectors":["produktuebersicht","variantenclick"],"selector":"li:nth-of-type(7) span.breadcrumb--title","multiple":false,"regex":"","delay":0},{"id":"bread5","type":"SelectorText","parentSelectors":["produktuebersicht","variantenclick"],"selector":"li:nth-of-type(9) span.breadcrumb--title","multiple":false,"regex":"","delay":0},{"id":"bestellnr","type":"SelectorText","parentSelectors":["produktuebersicht","variantenclick"],"selector":".base-info--entry span:nth-of-type(2)","multiple":false,"regex":"","delay":0},{"id":"inhalttext","type":"SelectorText","parentSelectors":["produktuebersicht","variantenclick"],"selector":"div.product--price.price--unit","multiple":false,"regex":"","delay":0},{"id":"inhaltmenge","type":"SelectorText","parentSelectors":["produktuebersicht","variantenclick"],"selector":"div.product--configurator option","multiple":false,"regex":"","delay":0}]}

leemeng · February 24, 2020, 2:14pm

For bread1 you should use: li:nth-of-type(1) span.breadcrumb--title

If you just want the last (active) breadcrumb, you can use: ul > li[class='breadcrumb--entry is--active']

VolkerSchroeder · February 24, 2020, 2:39pm

@leemeng
the first and last breadcrumb ist ok. I know.
In the Preview the breadcrumbs are all shown correctly.

But please test a Scrape.
You will see, that the breadcrumbs are only in a few rows are visible.
Often ist the result "null".

leemeng · February 25, 2020, 1:20am

Can you confirm that the scrapers are working for individual product pages? Also, your sitemap structure looks a bit off. Below I have cut down your sitemap to the produktuebersicht section so it only scrapes individual product pages (navigation, pagination has been removed). I have added a couple of StartUrls. You should add 6-10 more to properly test it. If this produktuebersicht section is working, then the problem is likely with your navigation/pagination section.