Pagination issue New

How to?
#1

Hi,

I used to use the below scraper just fine and it used to provide all the needed info without any problems.

But suddenly and without any changes from my side it started scraping the first page only then it finishes before going to the second page as it used to (pagination issue). I tried several times but I just can't see what is wrong or what is preventing it from working as it used to!!!

Please help.

Url: https://wuzzuf.net/search/jobs/?start=0&filters[country][0]=Egypt&filters[country][1]=Saudi%20Arabia&filters[country][2]=United%20Arab%20Emirates&filters[country][3]=Bahrain&filters[country][4]=Kuwait&filters[country][5]=United%20States&filters[country][6]=China&filters[country][7]=Oman

Sitemap:
{"_id":"wuzzuf_new_jobs","startUrl":["https://wuzzuf.net/search/jobs/?start=0&filters[country][0]=Egypt&filters[country][1]=Saudi%20Arabia&filters[country][2]=United%20Arab%20Emirates&filters[country][3]=Bahrain&filters[country][4]=Kuwait&filters[country][5]=United%20States&filters[country][6]=China&filters[country][7]=Oman"],"selectors":[{"id":"Pegination","type":"SelectorLink","parentSelectors":["_root","Pegination"],"selector":"ul.pagination.desktop-pagination li.pag-next a","multiple":false,"delay":0},{"id":"Link Selector","type":"SelectorLink","parentSelectors":["_root","Pegination"],"selector":"h2.job-title a","multiple":true,"delay":0},{"id":"Job Title","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"h1.job-title","multiple":false,"regex":"","delay":0},{"id":"Company Name","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"a.job-company-name","multiple":false,"regex":"","delay":0},{"id":"Location","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"span.job-company-location span","multiple":false,"regex":"","delay":0},{"id":"Experience Needed","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"tr:nth-of-type(1) td:nth-of-type(1) dd","multiple":false,"regex":"","delay":0},{"id":"Career Level","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"tr:nth-of-type(1) td:nth-of-type(2) dd","multiple":false,"regex":"","delay":0},{"id":"About the Job","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"div.about-job","multiple":false,"regex":"","delay":0},{"id":"Job Requirements","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"div.job-requirements","multiple":false,"regex":"","delay":0},{"id":"Post Date","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"p.job-post-date","multiple":false,"regex":"","delay":0},{"id":"Job Type","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"span.job-type","multiple":false,"regex":"","delay":0},{"id":"Salary","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"dl.salary-info dd","multiple":false,"regex":"","delay":0},{"id":"About the Company","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"div.col-sm-8 p.company-brief","multiple":false,"regex":"","delay":0}]}

#2

I've fixed this for you. I changed your pagination to use element click selector instead. I will cycle through all available pages before clicking on any of the records. This until you see job postings (not the search results) you won't see anything in preview.

{"_id":"wuzzuf_new_jobs","startUrl":["https://wuzzuf.net/search/jobs/?start=0&filters[country][0]=Egypt&filters[country][1]=Saudi%20Arabia&filters[country][2]=United%20Arab%20Emirates&filters[country][3]=Bahrain&filters[country][4]=Kuwait&filters[country][5]=United%20States&filters[country][6]=China&filters[country][7]=Oman"],"selectors":[{"id":"Pegination","type":"SelectorElementClick","parentSelectors":["_root"],"selector":"div.result-wrp","multiple":true,"delay":0,"clickElementSelector":".desktop-pagination .pag-next a","clickType":"clickMore","discardInitialElements":"do-not-discard","clickElementUniquenessType":"uniqueCSSSelector"},{"id":"Link Selector","type":"SelectorLink","parentSelectors":["Pegination"],"selector":".job-title a","multiple":false,"delay":0},{"id":"Job Title","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"h1.job-title","multiple":false,"regex":"","delay":0},{"id":"Company Name","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"span.job-company-name","multiple":false,"regex":"","delay":0},{"id":"Location","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"span.job-company-location span","multiple":false,"regex":"","delay":0},{"id":"Experience Needed","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"tr:nth-of-type(1) td:nth-of-type(1) dd","multiple":false,"regex":"","delay":0},{"id":"Career Level","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"tr:nth-of-type(1) td:nth-of-type(2) dd","multiple":false,"regex":"","delay":0},{"id":"About the Job","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"[itemprop='description'] p:nth-of-type(1)","multiple":false,"regex":"","delay":0},{"id":"Job Requirements","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"div.job-requirements","multiple":false,"regex":"","delay":0},{"id":"Post Date","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"p.job-post-date","multiple":false,"regex":"","delay":0},{"id":"Job Type","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"span.job-type","multiple":false,"regex":"","delay":0},{"id":"Salary","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"dl.salary-info dd","multiple":false,"regex":"","delay":0},{"id":"About the Company","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"div.col-sm-8 p.company-brief","multiple":false,"regex":"","delay":0}]}
1 Like
#3

Hi,

Thank you for your help, but this is not how I want it to function and I want to highlight that not the full data is being scraped as it used to (all job titles and job details)... The problem is, 3 days ago the above code was just working fine and functioning well without any problems. Now it is not functioning and I don't know why...

Can you help identifying the issue so it would work again as it used to?

also, one last question, how to delete an old scraper (sitemap)?

Thank you in advance :blush:

#4

Can you please be more specific about what it's not capturing that you'd like it to?

Where is it missing data?

From where I stand, it's getting pretty much everything you asked for. I noticed an issue with Company Name and have changed that ..

image
image.png1910×598 74.3 KB

New map with change in company name selector

{"_id":"wuzzuf_new_jobs","startUrl":["https://wuzzuf.net/search/jobs/?start=0&filters[country][0]=Egypt&filters[country][1]=Saudi%20Arabia&filters[country][2]=United%20Arab%20Emirates&filters[country][3]=Bahrain&filters[country][4]=Kuwait&filters[country][5]=United%20States&filters[country][6]=China&filters[country][7]=Oman"],"selectors":[{"id":"Pegination","type":"SelectorElementClick","parentSelectors":["_root"],"selector":"div.result-wrp","multiple":true,"delay":0,"clickElementSelector":".desktop-pagination .pag-next a","clickType":"clickMore","discardInitialElements":"do-not-discard","clickElementUniquenessType":"uniqueCSSSelector"},{"id":"Link Selector","type":"SelectorLink","parentSelectors":["Pegination"],"selector":".job-title a","multiple":false,"delay":0},{"id":"Job Title","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"h1.job-title","multiple":false,"regex":"","delay":0},{"id":"Company Name","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"a.job-company-name","multiple":false,"regex":"","delay":0},{"id":"Location","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"span.job-company-location span","multiple":false,"regex":"","delay":0},{"id":"Experience Needed","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"tr:nth-of-type(1) td:nth-of-type(1) dd","multiple":false,"regex":"","delay":0},{"id":"Career Level","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"tr:nth-of-type(1) td:nth-of-type(2) dd","multiple":false,"regex":"","delay":0},{"id":"About the Job","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"[itemprop='description'] p:nth-of-type(1)","multiple":false,"regex":"","delay":0},{"id":"Job Requirements","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"div.job-requirements","multiple":false,"regex":"","delay":0},{"id":"Post Date","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"p.job-post-date","multiple":false,"regex":"","delay":0},{"id":"Job Type","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"span.job-type","multiple":false,"regex":"","delay":0},{"id":"Salary","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"dl.salary-info dd","multiple":false,"regex":"","delay":0},{"id":"About the Company","type":"SelectorText","parentSelectors":["Link Selector"],"selector":"div.col-sm-8 p.company-brief","multiple":false,"regex":"","delay":0}]}