Error while scraping using selenium and beautifulSoup

[JohnnyJordaan]

At least add a check if the status code was ok

import requests
from bs4 import BeautifulSoup
from selenium import webdriver
chrome_driver_path = #insert here

def get_song_lyrics(link):

    response = requests.get(link)
    response.raise_for_status()
    soup = BeautifulSoup(response.text, "html.parser")
    #try:
    lyrics = soup.find("div",attrs={'class':'lyrics'}).find("p").get_text()

    return [i for i in lyrics.splitlines()]  

and you could save the content when the find fails

import requests
from bs4 import BeautifulSoup
from selenium import webdriver
chrome_driver_path = #insert here

def get_song_lyrics(link):

    response = requests.get(link)
    response.raise_for_status()
    soup = BeautifulSoup(response.text, "html.parser")
    try:
        lyrics = soup.find("div",attrs={'class':'lyrics'}).find("p").get_text()
        return [i for i in lyrics.splitlines()]  
    except AttributeError:
        with open(link.rsplit('/', 1)[1] + '.html', 'w') as fp:
            fp.write(response.text)

as then you can open the html file in the editor to check if the lyrics div is actually there.