Hello everyone,
I want to scrape data from the memewebsite 9gag.com
The data I want to collect is the following : Title of meme, section of meme, URL of meme, number of Upvotes, number of Downvotes, number of comments ; approx 120 memes in one go (one scrape session).
The problem is that the content loads in an infinite scroll and only some is "available to see", so or I try to scrape too much and my excel file is blank (or mostly "null"), or i can only scrape 12 memes and everything is fine (but i need ~120).
Down here is what I have used upto now. Some help would be much appreciated !!!
{"_id":"scrap9gagmemeV2","startUrl":["https://9gag.com/"],"selectors":[{"id":"Selects article","parentSelectors":["scrolldown a bit"],"type":"SelectorElement","selector":"article","multiple":true,"delay":0},{"id":"Select section and time","parentSelectors":["Selects article"],"type":"SelectorText","selector":"p","multiple":false,"delay":0,"regex":""},{"id":"saveURLmeme","parentSelectors":["Selects article"],"type":"SelectorElementAttribute","selector":"header a.badge-evt","multiple":false,"delay":0,"extractAttribute":"href"},{"id":"Upvotes","parentSelectors":["Selects article"],"type":"SelectorText","selector":".up span","multiple":false,"delay":0,"regex":""},{"id":"down votes","parentSelectors":["Selects article"],"type":"SelectorText","selector":".down span","multiple":false,"delay":0,"regex":""},{"id":"comments","parentSelectors":["Selects article"],"type":"SelectorText","selector":".comment span","multiple":false,"delay":0,"regex":""},{"id":"retrieves URL","parentSelectors":["saveURLmeme"],"type":"SelectorText","selector":"#jsid-post-a11rPOv a.share","multiple":false,"delay":0,"regex":""},{"id":"title","parentSelectors":["Selects article"],"type":"SelectorText","selector":"h1","multiple":false,"delay":0,"regex":""},{"id":"scrolldown a bit","parentSelectors":["_root"],"type":"SelectorElementScroll","selector":"div#page","multiple":false,"delay":2000}]}