Defining a method for collecting the urls we want to scrape
Tue May 21 2024 15:30:17 GMT+0000 (Coordinated Universal Time)
Saved by
@Uncoverit
#python
# Defining a method for collecting the urls we want to scrape
def scrape_perfume_links(url):
headers = {
"User-Agent": "Insert user agent",
"Referer": "https://www.parfumo.com/",
"Accept-Language": "en-US,en;q=0.9"
}
session = requests.Session()
page = session.get(url, headers=headers)
page.raise_for_status()
soup3 = BeautifulSoup(page.content, 'html.parser')
perfumes = soup3.find_all('div', class_='name')
perfume_links = []
for perfume in perfumes:
link = perfume.find('a')['href']
perfume_links.append(link)
return perfume_links
content_copyCOPY
Comments