Hi all,
I am very newbie but I successfully scaped my first pages, but this one gives me a hard time. The pagination works, but it stopps after 30 pages. I dont know how to solve it.
Next but minor issue: At the top there is a drop down where you can select items (in this case powerplants) from the list. Can I somehow make the choice before I crawl? Because I only need about 16000 out of the about 1600000 data sets.
Url: https://www.marktstammdatenregister.de/MaStR/Einheit/Einheiten/OeffentlicheEinheitenuebersicht
Sitemap:
{"_id":"marktstammdaten","startUrl":["https://www.marktstammdatenregister.de/MaStR/Einheit/Einheiten/OeffentlicheEinheitenuebersicht"],"selectors":[{"id":"click-selector","type":"SelectorElementClick","parentSelectors":["_root"],"selector":"tr:nth-of-type(1) a.action, tr:nth-of-type(n+2) a.action","multiple":true,"delay":"500","clickElementSelector":"li:nth-of-type(n+2) a.k-link, div.tab-pane.active span.k-state-selected, div.tab-pane.active a.k-link.k-pager-nav:nth-of-type(n+3) span.k-icon","clickType":"clickOnce","discardInitialElements":true,"clickElementUniquenessType":"uniqueText"},{"id":"stamm","type":"SelectorLink","parentSelectors":["click-selector"],"selector":"parent","multiple":true,"delay":0},{"id":"name","type":"SelectorText","parentSelectors":["stamm"],"selector":"div.tab-pane.active div.panel:nth-of-type(1) table:nth-of-type(1) td.display-template-value","multiple":false,"regex":"","delay":0},{"id":"mastr-nr","type":"SelectorText","parentSelectors":["stamm"],"selector":"div.panel:nth-of-type(1) table:nth-of-type(2) tr:nth-of-type(1) td.display-template-value","multiple":false,"regex":"","delay":0},{"id":"betreiber","type":"SelectorText","parentSelectors":["stamm"],"selector":"div.tab-pane.active div.panel:nth-of-type(1) tr:nth-of-type(2) td.display-template-value","multiple":false,"regex":"","delay":0},{"id":"strasse","type":"SelectorText","parentSelectors":["stamm"],"selector":"div.col-xs-11 div:nth-of-type(1)","multiple":false,"regex":"","delay":0},{"id":"ort","type":"SelectorText","parentSelectors":["stamm"],"selector":"div.col-xs-11 div:nth-of-type(2)","multiple":false,"regex":"","delay":0},{"id":"länge","type":"SelectorText","parentSelectors":["stamm"],"selector":"table.WGS84 tr:nth-of-type(2) td.display-template-value","multiple":false,"regex":"","delay":0},{"id":"breite","type":"SelectorText","parentSelectors":["stamm"],"selector":"table.WGS84 tr:nth-of-type(1) td.display-template-value","multiple":false,"regex":"","delay":0},{"id":"date-tech","type":"SelectorLink","parentSelectors":["stamm"],"selector":"ul.nav li:nth-of-type(2) a","multiple":false,"delay":0},{"id":"buttoleistung","type":"SelectorText","parentSelectors":["date-tech"],"selector":"div.tab-pane.active div.panel:nth-of-type(1) tr:nth-of-type(1) td.display-template-value","multiple":false,"regex":"","delay":0},{"id":"netto-leistung","type":"SelectorText","parentSelectors":["date-tech"],"selector":"div.tab-pane.active div.panel:nth-of-type(1) tr:nth-of-type(2) td.display-template-value","multiple":false,"regex":"","delay":0},{"id":"ibn","type":"SelectorText","parentSelectors":["stamm"],"selector":"div.tab-pane.active tr:nth-of-type(7) td.display-template-value","multiple":false,"regex":"","delay":0},{"id":"bula","type":"SelectorText","parentSelectors":["stamm"],"selector":"div.tab-pane.active div.panel:nth-of-type(2) tr:nth-of-type(1) td.display-template-value","multiple":false,"regex":"","delay":0},{"id":"technologie","type":"SelectorText","parentSelectors":["date-tech"],"selector":"div.tab-pane.active div.panel:nth-of-type(2) tr:nth-of-type(1) td.display-template-value","multiple":false,"regex":"","delay":0},{"id":"brennstoff","type":"SelectorText","parentSelectors":["date-tech"],"selector":"div.tab-pane.active div.panel:nth-of-type(2) tr:nth-of-type(2) td.display-template-value","multiple":false,"regex":"","delay":0},{"id":"biomasseart","type":"SelectorText","parentSelectors":["date-tech"],"selector":"div.tab-pane.active div.panel:nth-of-type(2) tr:nth-of-type(3) td.display-template-value","multiple":false,"regex":"","delay":0},{"id":"volleinspeisung","type":"SelectorText","parentSelectors":["date-tech"],"selector":"div.tab-pane.active div.panel:nth-of-type(3) table:nth-of-type(1) tr:nth-of-type(1) td.display-template-value","multiple":false,"regex":"","delay":0},{"id":"fernsteuerbar-netzbetreiber","type":"SelectorText","parentSelectors":["date-tech"],"selector":"div.tab-pane.active div.panel:nth-of-type(3) tr:nth-of-type(2) td.display-template-value","multiple":false,"regex":"","delay":0},{"id":"fernsteuerbar-dv","type":"SelectorText","parentSelectors":["date-tech"],"selector":"div.panel:nth-of-type(3) tr:nth-of-type(3) td.display-template-value","multiple":false,"regex":"","delay":0},{"id":"fernsteuerbar-dritter","type":"SelectorText","parentSelectors":["date-tech"],"selector":"div.tab-pane.active tr:nth-of-type(4) td.display-template-value","multiple":false,"regex":"","delay":0},{"id":"hochspannungsnetz","type":"SelectorText","parentSelectors":["date-tech"],"selector":"div.tab-pane.active tr:nth-of-type(6) td.display-template-value","multiple":false,"regex":"","delay":0},{"id":"eeg-link","type":"SelectorLink","parentSelectors":["date-tech"],"selector":"ul.nav li:nth-of-type(6) a","multiple":false,"delay":0},{"id":"eeg-anlagenschlüssel","type":"SelectorText","parentSelectors":["eeg-link"],"selector":"div.tab-pane.active tr:nth-of-type(5) td.display-template-value","multiple":false,"regex":"","delay":0},{"id":"einheit","type":"SelectorText","parentSelectors":["eeg-link"],"selector":"div.tab-pane.active td.columns-small:nth-of-type(2)","multiple":false,"regex":"","delay":0},{"id":"einheit-name","type":"SelectorText","parentSelectors":["eeg-link"],"selector":"div.tab-pane.active td.columns-small:nth-of-type(3)","multiple":false,"regex":"","delay":0},{"id":"einheit-standort","type":"SelectorText","parentSelectors":["eeg-link"],"selector":"div.tab-pane.active td.columns-small:nth-of-type(4)","multiple":false,"regex":"","delay":0}]}