I'm having trouble setting up a start URL that will guide my scrap through all article pages for a partner site.
The goal is to scrape data from all the article pages available on their site, the challenge is, there is no numeric parameters in the URL and the root page isn't an actual page so it's a 404 error and will not continue the scrape. All the selectors function properly but the start url just isn't working. Any ideas would be appreciated.
Url: https://www.digikey.com/en/articles/
Sitemap:
{"_id":"digikeyArticles","startUrl":["https://www.digikey.com/en/articles/[A-Z:5000]"],"selectors":[{"id":"digikeyArticleRelatedProductTrainingTitle","multiple":true,"parentSelectors":["_root"],"selector":"div.sidebar__section:nth-of-type(2) a","type":"SelectorLink"},{"extractAttribute":"content","id":"digikeyArticleKeywords","multiple":false,"parentSelectors":["_root"],"selector":"head > meta:nth-child(4)","type":"SelectorElementAttribute"},{"id":"digikeyArticleRelatedProductHighlightTitle","multiple":true,"parentSelectors":["_root"],"selector":"div:nth-of-type(3) .sidebar__content a","type":"SelectorLink"},{"id":"digikeyArticleTitle","multiple":false,"parentSelectors":["_root"],"regex":"","selector":"h1","type":"SelectorText"},{"id":"digikeyArticleAuthor","multiple":false,"parentSelectors":["_root"],"regex":"","selector":"p.authorName","type":"SelectorText"},{"id":"digikeyArticlePubDate","multiple":false,"parentSelectors":["_root"],"regex":"","selector":"p.articleDate","type":"SelectorText"},{"extractAttribute":"content","id":"digikeyArticleFunction","multiple":false,"parentSelectors":["_root"],"selector":"head > meta:nth-child(12)","type":"SelectorElementAttribute"},{"extractAttribute":"content","id":"digikeyArticleURL","multiple":false,"parentSelectors":["_root"],"selector":"head > meta:nth-child(20)","type":"SelectorElementAttribute"}]}