How to do pagination?

How to?
#1

how can scrape all the article info on this page the goto next issue again collect all article.

Url: https://asmepublications.onlinelibrary.wiley.com/toc/1743498x/2004/1/1

Sitemap:
{"_id":"Wiley-2000-2025(Test)","startUrl":["https://asmepublications.onlinelibrary.wiley.com/loi/1743498x/year/2004"],"selectors":[{"id":"Issues","linkType":"linkFromHref","multiple":false,"parentSelectors":["_root"],"selector":"a.visitable","type":"SelectorLink"},{"id":"Article","linkType":"linkFromHref","multiple":true,"parentSelectors":["Issues","Page"],"selector":"a.issue-item__title","type":"SelectorLink"},{"id":"Title","multiple":false,"parentSelectors":["Article"],"regex":"","selector":"h1","type":"SelectorText"},{"clickActionType":"real","clickElementSelector":"a.read-more","clickElementUniquenessType":"uniqueText","clickType":"clickOnce","delay":0,"discardInitialElements":"discard-when-click-element-exists","id":"Click","multiple":true,"parentSelectors":["Article"],"selector":"body","type":"SelectorElementClick"},{"elementLimit":0,"id":"Element","multiple":true,"parentSelectors":["Click"],"scroll":false,"selector":".accordion .comma__list span.accordion-tabbed__tab-mobile","type":"SelectorElement"},{"id":"Name","multiple":false,"parentSelectors":["Element"],"regex":"","selector":"p.author-name","type":"SelectorText"},{"id":"Email","multiple":false,"parentSelectors":["Element"],"regex":"","selector":"[href^=\"mailto\"]","type":"SelectorText"},{"id":"Orcid","multiple":false,"parentSelectors":["Element"],"regex":"","selector":".sm-account__link[href*=\"orcid.org\"]","type":"SelectorText"},{"id":"Affaliation 1","multiple":false,"parentSelectors":["Element"],"regex":"","selector":".author-info p:nth-of-type(3)","type":"SelectorText"},{"id":"Affaliation 2","multiple":false,"parentSelectors":["Element"],"regex":"","selector":".author-info p:nth-of-type(2)","type":"SelectorText"},{"elementLimit":0,"id":"Element 2","multiple":true,"parentSelectors":["Click"],"scroll":false,"selector":"div.article-header__correspondence-to","type":"SelectorElement"},{"id":"Correspondence Email","multiple":false,"parentSelectors":["Element 2"],"regex":"","selector":"[href^=\"mailto\"]","type":"SelectorText"},{"id":"Correspondence Address","multiple":false,"parentSelectors":["Element 2"],"regex":"","selector":"_parent_","type":"SelectorText"},{"id":"Page","paginationType":"clickOnce","parentSelectors":["Issues","Page"],"selector":".content-navigation__btn--next span","type":"SelectorPagination"}]}

image
image1136×767 117 KB

#2

Hi,

This should work. I changed the start URL and removed the span from the pagination selector since it was not the correct element containing the page URL.

{"_id":"Wiley-2000-2025(Test)","startUrl":["https://asmepublications.onlinelibrary.wiley.com/toc/1743498x/2004/1/1"],"selectors":[{"id":"Article","linkType":"linkFromHref","multiple":true,"parentSelectors":["Page"],"selector":"a.issue-item__title","type":"SelectorLink"},{"id":"Title","multiple":false,"parentSelectors":["Article"],"regex":"","selector":"h1","type":"SelectorText"},{"clickActionType":"real","clickElementSelector":"a.read-more","clickElementUniquenessType":"uniqueText","clickType":"clickOnce","delay":0,"discardInitialElements":"discard-when-click-element-exists","id":"Click","multiple":true,"parentSelectors":["Article"],"selector":"body","type":"SelectorElementClick"},{"elementLimit":0,"id":"Element","multiple":true,"parentSelectors":["Click"],"scroll":false,"selector":".accordion .comma__list span.accordion-tabbed__tab-mobile","type":"SelectorElement"},{"id":"Name","multiple":false,"parentSelectors":["Element"],"regex":"","selector":"p.author-name","type":"SelectorText"},{"id":"Email","multiple":false,"parentSelectors":["Element"],"regex":"","selector":"[href^=\"mailto\"]","type":"SelectorText"},{"id":"Orcid","multiple":false,"parentSelectors":["Element"],"regex":"","selector":".sm-account__link[href*=\"orcid.org\"]","type":"SelectorText"},{"id":"Affaliation 1","multiple":false,"parentSelectors":["Element"],"regex":"","selector":".author-info p:nth-of-type(3)","type":"SelectorText"},{"id":"Affaliation 2","multiple":false,"parentSelectors":["Element"],"regex":"","selector":".author-info p:nth-of-type(2)","type":"SelectorText"},{"elementLimit":0,"id":"Element 2","multiple":true,"parentSelectors":["Click"],"scroll":false,"selector":"div.article-header__correspondence-to","type":"SelectorElement"},{"id":"Correspondence Email","multiple":false,"parentSelectors":["Element 2"],"regex":"","selector":"[href^=\"mailto\"]","type":"SelectorText"},{"id":"Correspondence Address","multiple":false,"parentSelectors":["Element 2"],"regex":"","selector":"_parent_","type":"SelectorText"},{"id":"Page","paginationType":"auto","parentSelectors":["_root","Page"],"selector":"a.content-navigation__btn--next","type":"SelectorPagination"}]}
1 Like