Hello everyone I need help to scrape a website. How do I go about it? This website doesn't show the body hence can't dig deeper to search for elements. I think its when loading then JS takes over to load the contents of the page.
The site is https://www.camplify.com.au/
I want to grab the name and various attributes. But first if I can be shown how to navigate and the name of motor homes then I can continue with the rest. I have already begun just need guidance and suggestions on how to go about it.
import requests
from bs4 import BeautifulSoup as Bsoup
from configured_selenium_driver import initialize_webdriver
# url = 'https://www.camplify.com.au/'
url = 'https://www.camplify.com.au/s?seed=14431&page=1'
# url = 'https://www.google.com'
def get_cookies(url):
try:
resp = requests.get(url)
cookies = resp.cookies
print(f'cookies retireved: {cookies}')
cookes_retrieved = len(cookies)
print(f'{cookes_retrieved} cookies retrieved')
for number,cookie in enumerate(cookies, start=1):
print(f'{number}: {cookie}')
except Exception as error:
print(f'error occured {error}')
return None
# get_cookies(url) # funtion to get cookies
resp = requests.get(url)
soup = Bsoup(resp.text, 'lxml')
# print(soup)
div_wrapper = soup.find_all('div', attrs={'class': 'wrapper'})
# divs = soup.find_all('div', attrs={'data-react-class':"index"})
# divs = soup.find_all('div', attrs={'id': 'navbar-container'})
# print(divs)
div1 = div_wrapper.f
for div in divs:
print(div.text)