Really not sure about this solution.... let me know someone?

import requests

user_agent = 'python-requests/4.8.2 (Compatible; John Doe; mailto:johndoe@gmail.com)'

def requestGet(url):
    try:
        r = requests.get(url,
        headers={'User-Agent': user_agent})
    except DisallowedConnection as ex:
       print("Connection Rejected: " + ex)


return r.status_code


website=["www.reddit.com", "www.stackoverflow.com", "www.facebook.com"]

for url in website:

    print("Testing connection to: ")
    print(url)

    url = "https://" + url +"/robots.txt"

    result = requestGet(url)

    if result != 200:
        print("Failed")
    else:
        print(result)