Hey everyone - my web scraper sitemap for a Twitter advanced search crashes without collecting any data. I'm trying to scrape data from the following search results page:https://twitter.com/search?q="rule%20of%20law"%20min_retweets%3A250%20lang%3Aen%20until%3A2018-12-31%20since%3A2018-10-01%20-filter%3Areplies&src=typed_query. I'm trying to follow the instructions from this youtube video: https://www.youtube.com/watch?v=AmMpJmYiAkQ.
The sitemap JSON I copied from GitHub is below. Thanks for the help!
Web Scraper version: 0.4.2
Chrome version: 79
OS: Windows 10 Pro
Sitemap:
{id:"my sitemap"}
{
"_id":"twitter_feed",
"startUrl":[
"https://twitter.com/search?l=&q=web%20scraping%20since%3A2018-10-01%20until%3A2018-10-05&src=typd&lang=en"
],
"selectors":[
{
"id":"tweet",
"type":"SelectorElementScroll",
"parentSelectors":[
"_root"
],
"selector":"div.tweet",
"multiple":true,
"delay":"2000"
},
{
"id":"handle",
"type":"SelectorText",
"parentSelectors":[
"tweet"
],
"selector":"span.username",
"multiple":false,
"regex":"",
"delay":0
},
{
"id":"name",
"type":"SelectorText",
"parentSelectors":[
"tweet"
],
"selector":"strong.fullname",
"multiple":false,
"regex":"",
"delay":0
},
{
"id":"content",
"type":"SelectorText",
"parentSelectors":[
"tweet"
],
"selector":".tweet-text",
"multiple":false,
"regex":"",
"delay":0
},
{
"id":"replies",
"type":"SelectorText",
"parentSelectors":[
"tweet"
],
"selector":"div.ProfileTweet-action.ProfileTweet-action--reply span.ProfileTweet-actionCountForPresentation",
"multiple":false,
"regex":"",
"delay":0
},
{
"id":"retweets",
"type":"SelectorText",
"parentSelectors":[
"tweet"
],
"selector":"div.ProfileTweet-action.ProfileTweet-action--retweet button.ProfileTweet-actionButton span.ProfileTweet-actionCountForPresentation",
"multiple":false,
"regex":"",
"delay":0
},
{
"id":"favorites",
"type":"SelectorText",
"parentSelectors":[
"tweet"
],
"selector":"div.ProfileTweet-action.ProfileTweet-action--favorite button.ProfileTweet-actionButton span.ProfileTweet-actionCountForPresentation",
"multiple":false,
"regex":"",
"delay":0
},
{
"id":"unix_timestamp",
"type":"SelectorElementAttribute",
"parentSelectors":[
"tweet"
],
"selector":"span._timestamp",
"multiple":false,
"extractAttribute":"data-time-ms",
"delay":0
},
{
"id":"published_date",
"type":"SelectorElementAttribute",
"parentSelectors":[
"tweet"
],
"selector":".time a.tweet-timestamp",
"multiple":false,
"extractAttribute":"title",
"delay":0
},
{
"id":"url",
"type":"SelectorElementAttribute",
"parentSelectors":[
"tweet"
],
"selector":"a.tweet-timestamp",
"multiple":false,
"extractAttribute":"href",
"delay":0
}
]
}