Hi there,
it took me forever to figure out the problem of having a massive data loss. It seems that I found it now, but can't solve it.
I am scraping a whole bunch of kickstarter projects and it seems like as soon as the user name is previously scraped, the next project does not get saved. The scraper (in the window) visits the project, however, does not extract any data (project is simply not listed).
Please help! I am really getting crazy because this is for my master thesis and I need this data.
Thank you so much!
This is a great example page. Two projects created by the same users (Senic and Koma Elektronik GmbH) and only one project of each user gets saved.
Sitemap:
{"_id":"kickstarter_technology_germany_2019_06_27_version_8","startUrl":["https://www.kickstarter.com/discover/advanced?category_id=16&woe_id=23424829&sort=popularity&seed=2596372&page=3"],"selectors":[{"id":"select_project","type":"SelectorLink","parentSelectors":["_root"],"selector":"a.mb3","multiple":true,"delay":5000},{"id":"finanzierungszeitraum","type":"SelectorText","parentSelectors":["select_project"],"selector":"p.f5","multiple":false,"regex":"","delay":0},{"id":"success_financed","type":"SelectorText","parentSelectors":["select_project"],"selector":".mb0 span","multiple":false,"regex":"","delay":0},{"id":"success_fiance_goal","type":"SelectorText","parentSelectors":["select_project"],"selector":".type-12 span.money","multiple":false,"regex":"","delay":0},{"id":"anzahl_unterstuetzer_gesamt","type":"SelectorText","parentSelectors":["select_project"],"selector":".mb0 h3","multiple":false,"regex":"","delay":0},{"id":"ueber_uns_tag_1","type":"SelectorText","parentSelectors":["select_project"],"selector":"a.grey-dark.flex","multiple":false,"regex":"","delay":0},{"id":"ueber_uns_tag_2","type":"SelectorText","parentSelectors":["select_project"],"selector":"a.grey-dark:nth-of-type(2)","multiple":false,"regex":"","delay":0},{"id":"ueber_uns_tag_3","type":"SelectorText","parentSelectors":["select_project"],"selector":"a.grey-dark:nth-of-type(3)","multiple":false,"regex":"","delay":0},{"id":"ueber_uns_tag_4","type":"SelectorText","parentSelectors":["select_project"],"selector":"a.mr3:nth-of-type(1)","multiple":false,"regex":"","delay":0},{"id":"ueber_uns_tag_5","type":"SelectorText","parentSelectors":["select_project"],"selector":".hide a:nth-of-type(2) span","multiple":false,"regex":"","delay":0},{"id":"number_FAQ","type":"SelectorText","parentSelectors":["select_project"],"selector":".js-load-project-faqs span","multiple":false,"regex":"","delay":0},{"id":"number_updates","type":"SelectorText","parentSelectors":["select_project"],"selector":".js-load-project-updates span","multiple":false,"regex":"","delay":0},{"id":"number_comments","type":"SelectorText","parentSelectors":["select_project"],"selector":"data","multiple":false,"regex":"","delay":0},{"id":"failed_financed","type":"SelectorText","parentSelectors":["select_project"],"selector":".grid-col-4-md span.soft-black","multiple":false,"regex":"","delay":0},{"id":"failed_finance_goal","type":"SelectorText","parentSelectors":["select_project"],"selector":".grid-col-4-md .inline-block-sm span","multiple":false,"regex":"","delay":0},{"id":"failed_notification","type":"SelectorText","parentSelectors":["select_project"],"selector":".grid-col-4-md div.border","multiple":false,"regex":"","delay":0},{"id":"failed_anzahl_unterstutzer_geasmt","type":"SelectorText","parentSelectors":["select_project"],"selector":".grid-col-4-md .block.type-16 span","multiple":false,"regex":"","delay":0},{"id":"video_top","type":"SelectorText","parentSelectors":["select_project"],"selector":"button.m-auto","multiple":false,"regex":"","delay":0},{"id":"video_middle_abspielen","type":"SelectorText","parentSelectors":["select_project"],"selector":"button.play_button_big","multiple":false,"regex":"","delay":0},{"id":"Erstellt_von_successful ","type":"SelectorText","parentSelectors":["select_project"],"selector":"a.hero__link.remote_modal_dialog","multiple":false,"regex":"","delay":0},{"id":"failed_select_erstellt_von","type":"SelectorElementClick","parentSelectors":["select_project"],"selector":"div.pb3, div.py10-sm","multiple":false,"delay":"2000","clickElementSelector":"div.text-left.type-16","clickType":"clickOnce","discardInitialElements":"do-not-discard","clickElementUniquenessType":"uniqueCSSSelector"},{"id":"failed_erstellt_von ","type":"SelectorText","parentSelectors":["select_project"],"selector":"div.text-left.type-16","multiple":false,"regex":"","delay":0},{"id":"failed_facebook","type":"SelectorText","parentSelectors":["select_project"],"selector":"div.flex:nth-of-type(3) span","multiple":false,"regex":"","delay":0},{"id":"failed_erstelle_UND_untersütze_projekte","type":"SelectorText","parentSelectors":["select_project"],"selector":"div:nth-of-type(4) > span","multiple":false,"regex":"","delay":0},{"id":"failed_name_hiner_account","type":"SelectorText","parentSelectors":["select_project"],"selector":".flex.soft-black span","multiple":false,"regex":"","delay":0},{"id":"failed_select_name_for_profile_picture","type":"SelectorLink","parentSelectors":["select_project"],"selector":".mb0 a","multiple":false,"delay":0},{"id":"failed_profile_picture","type":"SelectorImage","parentSelectors":["failed_select_name_for_profile_picture"],"selector":"img.avatar-image","multiple":false,"delay":0},{"id":"select_community","type":"SelectorElementClick","parentSelectors":["select_project"],"selector":".js-project-nav-scroll div.row","multiple":false,"delay":"2000","clickElementSelector":"a.js-load-project-community","clickType":"clickOnce","discardInitialElements":"do-not-discard","clickElementUniquenessType":"uniqueCSSSelector"},{"id":"unterstuetzer_land_1","type":"SelectorText","parentSelectors":["select_project"],"selector":".js-locations-countries div:nth-of-type(1) .left a","multiple":false,"regex":"","delay":0},{"id":"unterstuetzer_anzahl_1","type":"SelectorText","parentSelectors":["select_project"],"selector":".js-locations-countries div:nth-of-type(1) div.tertiary-text","multiple":false,"regex":"","delay":0},{"id":"unterstuetzer_land_2","type":"SelectorText","parentSelectors":["select_project"],"selector":".js-locations-countries div:nth-of-type(2) a","multiple":false,"regex":"","delay":0},{"id":"unterstuetzer_anzahl_2","type":"SelectorText","parentSelectors":["select_project"],"selector":".js-locations-countries div:nth-of-type(2) .right div","multiple":false,"regex":"","delay":0},{"id":"unterstuetzer_land_3","type":"SelectorText","parentSelectors":["select_project"],"selector":".js-locations-countries div:nth-of-type(3) a","multiple":false,"regex":"","delay":0},{"id":"unterstuetzer_anzahl_3","type":"SelectorText","parentSelectors":["select_project"],"selector":".js-locations-countries div:nth-of-type(3) div.tertiary-text","multiple":false,"regex":"","delay":0},{"id":"unterstuetzer_land_4","type":"SelectorText","parentSelectors":["select_project"],"selector":".js-locations-countries div:nth-of-type(4) a","multiple":false,"regex":"","delay":0},{"id":"unterstuetzer_anzahl_4","type":"SelectorText","parentSelectors":["select_project"],"selector":".js-locations-countries div:nth-of-type(4) div.tertiary-text","multiple":false,"regex":"","delay":0},{"id":"select_erstellt_von_success","type":"SelectorElementClick","parentSelectors":["select_project"],"selector":"div.pb3, div.py10-sm","multiple":false,"delay":"2000","clickElementSelector":"a.hero__link.remote_modal_dialog","clickType":"clickOnce","discardInitialElements":"do-not-discard","clickElementUniquenessType":"uniqueCSSSelector"},{"id":"erstelle_UND_untersütze_projekte","type":"SelectorText","parentSelectors":["select_project"],"selector":"div.created-projects","multiple":false,"regex":"","delay":0},{"id":"name_hiner_account","type":"SelectorText","parentSelectors":["select_project"],"selector":"span.identity_name","multiple":false,"regex":"","delay":0},{"id":"mitarbeiter_1","type":"SelectorText","parentSelectors":["select_project"],"selector":"div.flag:nth-of-type(1) .flag-body a","multiple":false,"regex":"","delay":0},{"id":"rolle_mitarbeiter_1","type":"SelectorText","parentSelectors":["select_project"],"selector":"div.flag:nth-of-type(1) div.type-sm--sans","multiple":false,"regex":"","delay":0},{"id":"mitarbeiter_2","type":"SelectorText","parentSelectors":["select_project"],"selector":"div.col:nth-of-type(2) .flag-body a","multiple":false,"regex":"","delay":0},{"id":"rolle_mitarbeiter_2","type":"SelectorText","parentSelectors":["select_project"],"selector":"div.flag:nth-of-type(2) div.type-sm--sans","multiple":false,"regex":"","delay":0},{"id":"mitarbeiter_3","type":"SelectorText","parentSelectors":["select_project"],"selector":"div.flag:nth-of-type(3) a","multiple":false,"regex":"","delay":0},{"id":"rolle_mitarbeiter_3","type":"SelectorText","parentSelectors":["select_project"],"selector":"div.flag:nth-of-type(3) div.type-sm--sans","multiple":false,"regex":"","delay":0},{"id":"mitarbeiter_4","type":"SelectorText","parentSelectors":["select_project"],"selector":"div:nth-of-type(4) .flag-body a","multiple":false,"regex":"","delay":0},{"id":"rolle_mitarbeiter_4","type":"SelectorText","parentSelectors":["select_project"],"selector":"div:nth-of-type(4) div.type-sm--sans","multiple":false,"regex":"","delay":0},{"id":"facebook_account","type":"SelectorText","parentSelectors":["select_project"],"selector":"div.facebook.py2","multiple":false,"regex":"","delay":0},{"id":"success_select_name_for_profile_picture","type":"SelectorLink","parentSelectors":["select_project"],"selector":".f2 a","multiple":false,"delay":0},{"id":"success_profile_picture","type":"SelectorImage","parentSelectors":["success_select_name_for_profile_picture"],"selector":"img.avatar-image","multiple":false,"delay":0}]}