Describe the problem.
I found a similar post here, the problem was about incorrect link selection. Mine was different, so I posted a new thread here.
Please point out what I have done incorrectly as I am trying to learn using Web Scraper, thanks in advance.
I have two issues:
- Missed data extraction.
Please refer to the image attachments below.
It is a sample scrapped dataset and it contains many null values (for some columns not all).
However, if I only scrape the top 100 universities in 2019, it worked without a problem (refer to image 3).
- Incorrect data showed up
On the landing page, it shows 2019, and the ranking goes back to 2010.
I tried to add it as a link on top of university_link, but it messed up record, as 2019 records are showing up the year 2018, but year_link, which I defined under university_link shows an actual year. (refer to the second image.
Url: http://roundranking.com/ranking/world-university-rankings.html#world-2019
Sitemap:
{"_id":"rur_2019","startUrl":["http://roundranking.com/ranking/world-university-rankings.html#world-2019"],"selectors":[{"id":"university_link","type":"SelectorLink","parentSelectors":["year"],"selector":".td2 a","multiple":true,"delay":0},{"id":"university_name","type":"SelectorText","parentSelectors":["university_link"],"selector":".uni-info-title span","multiple":false,"regex":"","delay":0},{"id":"foundation_year","type":"SelectorText","parentSelectors":["university_link"],"selector":"dt[data-text='Foundation']","multiple":false,"regex":"","delay":0},{"id":"type","type":"SelectorText","parentSelectors":["university_link"],"selector":"dt[data-text='Type']","multiple":false,"regex":"","delay":0},{"id":"students","type":"SelectorText","parentSelectors":["university_link"],"selector":"dt#studts","multiple":false,"regex":"","delay":0},{"id":"faculty","type":"SelectorText","parentSelectors":["university_link"],"selector":"dt[data-text='Faculty']","multiple":false,"regex":"","delay":0},{"id":"region","type":"SelectorText","parentSelectors":["university_link"],"selector":"dt[data-text='Region']","multiple":false,"regex":"","delay":0},{"id":"location","type":"SelectorText","parentSelectors":["university_link"],"selector":"dt[data-text='Location']","multiple":false,"regex":"","delay":0},{"id":"website","type":"SelectorText","parentSelectors":["university_link"],"selector":"#WS dt","multiple":false,"regex":"","delay":0},{"id":"overall_world_university_ranking_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":"td[data-text='O_WR']","multiple":false,"regex":"","delay":0},{"id":"overall_world_university_ranking_score","type":"SelectorText","parentSelectors":["university_link"],"selector":"td[data-text='O_WS']","multiple":false,"regex":"","delay":0},{"id":"overall_teaching_ranking_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":"td[data-text='O_TR']","multiple":false,"regex":"","delay":0},{"id":"overall_teaching_ranking_score","type":"SelectorText","parentSelectors":["university_link"],"selector":"#tblrnk_O_TR td[data-text='O_TS']","multiple":false,"regex":"","delay":0},{"id":"overall_research_ranking_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":"td[data-text='O_RR']","multiple":false,"regex":"","delay":0},{"id":"overall_research_ranking_score","type":"SelectorText","parentSelectors":["university_link"],"selector":"td[data-text='O_RS']","multiple":false,"regex":"","delay":0},{"id":"overall_international_diversity_ranking_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":"td[data-text='O_IR']","multiple":false,"regex":"","delay":0},{"id":"overall_international_diversity_ranking_score","type":"SelectorText","parentSelectors":["university_link"],"selector":"td[data-text='O_IS']","multiple":false,"regex":"","delay":0},{"id":"overall_financial_sustainability_ranking_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":"td[data-text='O_FR']","multiple":false,"regex":"","delay":0},{"id":"overall_financial_sustainability_ranking_score","type":"SelectorText","parentSelectors":["university_link"],"selector":"td[data-text='O_FS']","multiple":false,"regex":"","delay":0},{"id":"overall_reputation_ranking_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":"td#reprnk","multiple":false,"regex":"","delay":0},{"id":"overall_reputation_ranking_score","type":"SelectorText","parentSelectors":["university_link"],"selector":"td#repscr","multiple":false,"regex":"","delay":0},{"id":"overall_academic_ranking_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":"td#resrnk","multiple":false,"regex":"","delay":0},{"id":"overall_academic_ranking_score","type":"SelectorText","parentSelectors":["university_link"],"selector":"td#resscr","multiple":false,"regex":"","delay":0},{"id":"overall_country_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":".uni-rank div","multiple":false,"regex":"[0-9]+","delay":0},{"id":"university_overview","type":"SelectorText","parentSelectors":["university_link"],"selector":"p.overview","multiple":false,"regex":"","delay":0},{"id":"university_mission_statement","type":"SelectorText","parentSelectors":["university_link"],"selector":"p.mission","multiple":false,"regex":"","delay":0},{"id":"t_academic_staff_per_student_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":".block1 tr:nth-of-type(1) .td3 span","multiple":false,"regex":"","delay":0},{"id":"t_academic_staff_per_student_score","type":"SelectorText","parentSelectors":["university_link"],"selector":".block1 tr:nth-of-type(1) .td4 span","multiple":false,"regex":"","delay":0},{"id":"t_academic_staff_per_bachelor_degree_awarded_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":".block1 tr:nth-of-type(2) .td3 span","multiple":false,"regex":"","delay":0},{"id":"t_academic_staff_per_bachelor_degree_awarded_score","type":"SelectorText","parentSelectors":["university_link"],"selector":".block1 tr:nth-of-type(2) .td4 span","multiple":false,"regex":"","delay":0},{"id":"t_doctoral_degrees_warded_per_academic_staff_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":".block1 tr:nth-of-type(3) .td3 span","multiple":false,"regex":"","delay":0},{"id":"t_doctoral_degrees_awarded_per_academic_staff_score","type":"SelectorText","parentSelectors":["university_link"],"selector":".block1 tr:nth-of-type(3) .td4 span","multiple":false,"regex":"","delay":0},{"id":"t_doctoral_degrees_awarded_per_bachelor_degrees_awarded_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":".block1 tr:nth-of-type(4) .td3 span","multiple":false,"regex":"","delay":0},{"id":"t_doctoral_degrees_awarded_per_bachelor_degrees_awarded_score","type":"SelectorText","parentSelectors":["university_link"],"selector":".block1 tr:nth-of-type(4) .td4 span","multiple":false,"regex":"","delay":0},{"id":"t_world_teaching_reputation_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":".block1 tr:nth-of-type(5) .td3 span","multiple":false,"regex":"","delay":0},{"id":"t_world_teaching_reputation_score","type":"SelectorText","parentSelectors":["university_link"],"selector":".block1 tr:nth-of-type(5) .td4 span","multiple":false,"regex":"","delay":0},{"id":"r_citation_per_academic_and_research_staff_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":".block2 tr:nth-of-type(1) .td3 span","multiple":false,"regex":"","delay":0},{"id":"r_citation_per_academic_and_research_staff_score","type":"SelectorText","parentSelectors":["university_link"],"selector":".block2 tr:nth-of-type(1) .td4 span","multiple":false,"regex":"","delay":0},{"id":"r_doctoral_degrees_awarded_per_admitted_phd_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":".block2 tr:nth-of-type(2) .td3 span","multiple":false,"regex":"","delay":0},{"id":"r_doctoral_degrees_awarded_per_admitted_phd_score","type":"SelectorText","parentSelectors":["university_link"],"selector":".block2 tr:nth-of-type(2) .td4 span","multiple":false,"regex":"","delay":0},{"id":"r_normalized_citation_impact_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":".block2 tr:nth-of-type(3) .td3 span","multiple":false,"regex":"","delay":0},{"id":"r_normalized_citation_impact_score","type":"SelectorText","parentSelectors":["university_link"],"selector":".block2 tr:nth-of-type(3) .td4 span","multiple":false,"regex":"","delay":0},{"id":"r_papers_per_academic_and_research_staff_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":".block2 tr:nth-of-type(4) .td3 span","multiple":false,"regex":"","delay":0},{"id":"r_papers_per_academic_and_research_staff_score","type":"SelectorText","parentSelectors":["university_link"],"selector":".block2 tr:nth-of-type(4) .td4 span","multiple":false,"regex":"","delay":0},{"id":"r_world_research_reputation_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":".block2 tr:nth-of-type(5) .td3 span","multiple":false,"regex":"","delay":0},{"id":"r_world_research_reputation_score","type":"SelectorText","parentSelectors":["university_link"],"selector":".block2 tr:nth-of-type(5) .td4 span","multiple":false,"regex":"","delay":0},{"id":"i_share_of_international_academic_staff_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":".block3 tr:nth-of-type(1) .td3 span","multiple":false,"regex":"","delay":0},{"id":"i_share_of_international_academic_staff_score","type":"SelectorText","parentSelectors":["university_link"],"selector":".block3 tr:nth-of-type(1) .td4 span","multiple":false,"regex":"","delay":0},{"id":"i_share_of_international_students_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":".block3 tr:nth-of-type(2) .td3 span","multiple":false,"regex":"","delay":0},{"id":"i_share_of_international_students_score","type":"SelectorText","parentSelectors":["university_link"],"selector":".block3 tr:nth-of-type(2) .td4 span","multiple":false,"regex":"","delay":0},{"id":"i_share_of_international_co-authored_papers_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":".block3 tr:nth-of-type(3) .td3 span","multiple":false,"regex":"","delay":0},{"id":"i_share_of_international_co-authored_papers_score","type":"SelectorText","parentSelectors":["university_link"],"selector":".block3 tr:nth-of-type(3) .td4 span","multiple":false,"regex":"","delay":0},{"id":"i_reputation_outside_region_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":".block3 tr:nth-of-type(4) .td3 span","multiple":false,"regex":"","delay":0},{"id":"i_reputation_outside_region_score","type":"SelectorText","parentSelectors":["university_link"],"selector":".block3 tr:nth-of-type(4) .td4 span","multiple":false,"regex":"","delay":0},{"id":"i_international_level_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":".block3 tr:nth-of-type(5) .td3 span","multiple":false,"regex":"","delay":0},{"id":"i_international_level_score","type":"SelectorText","parentSelectors":["university_link"],"selector":".block3 tr:nth-of-type(5) .td4 span","multiple":false,"regex":"","delay":0},{"id":"f_institutional_income_per_academic_staff_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":".block4 tr:nth-of-type(1) .td3 span","multiple":false,"regex":"","delay":0},{"id":"f_institutional_income_per_academic_staff_score","type":"SelectorText","parentSelectors":["university_link"],"selector":".block4 tr:nth-of-type(1) .td4 span","multiple":false,"regex":"","delay":0},{"id":"f_institutional_income_per_students_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":".block4 tr:nth-of-type(2) .td3 span","multiple":false,"regex":"","delay":0},{"id":"f_institutional_income_per_students_score","type":"SelectorText","parentSelectors":["university_link"],"selector":".block4 tr:nth-of-type(2) .td4 span","multiple":false,"regex":"","delay":0},{"id":"f_papers_per_research_income_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":".block4 tr:nth-of-type(3) .td3 span","multiple":false,"regex":"","delay":0},{"id":"f_papers_per_research_income_score","type":"SelectorText","parentSelectors":["university_link"],"selector":".block4 tr:nth-of-type(3) .td4 span","multiple":false,"regex":"","delay":0},{"id":"f_research_income_per_academic_and_research_staff_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":".block4 tr:nth-of-type(4) td.td3","multiple":false,"regex":"","delay":0},{"id":"f_research_income_per_academic_and_research_staff_score","type":"SelectorText","parentSelectors":["university_link"],"selector":".block4 tr:nth-of-type(4) .td4 span","multiple":false,"regex":"","delay":0},{"id":"f_research_income_per_institutional_income_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":".block4 tr:nth-of-type(5) .td3 span","multiple":false,"regex":"","delay":0},{"id":"f_research_income_per_institutional_income_score","type":"SelectorText","parentSelectors":["university_link"],"selector":".block4 tr:nth-of-type(5) .td4 span","multiple":false,"regex":"","delay":0},{"id":"t_country_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":".block1 tr:contains('Country rank:') td[data-text]","multiple":false,"regex":"","delay":0},{"id":"r_country_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":".block2 tr:contains('Country rank:') td[data-text]","multiple":false,"regex":"","delay":0},{"id":"i_country_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":".block3 tr:contains('Country rank:') td[data-text]","multiple":false,"regex":"","delay":0},{"id":"f_country_rank","type":"SelectorText","parentSelectors":["university_link"],"selector":".block4 tr:contains('Country rank:') td[data-text]","multiple":false,"regex":"","delay":0},{"id":"country","type":"SelectorText","parentSelectors":["university_link"],"selector":"p.uni-economy","multiple":false,"regex":"","delay":0},{"id":"address","type":"SelectorText","parentSelectors":["university_link"],"selector":"p.uni-address-pp","multiple":false,"regex":"","delay":0},{"id":"year","type":"SelectorLink","parentSelectors":["_root"],"selector":"a.point","multiple":true,"delay":0},{"id":"link_year","type":"SelectorText","parentSelectors":["university_link"],"selector":".active span.point-text","multiple":false,"regex":"","delay":0}]}