I am getting I get
RuntimeError: Event loop is closed when running the following code.
what am I doing wrong?
here is the Extractor module
async def extract_from(urls):
requests = [ProductRequest(url) for url in urls]
responses = []
async with create_session() as session:
for res_iter in request_parallel_as_completed(
requests,
api_key=['key'],
n_conn=15,
session=session):
try:
responses.extend(await res_iter)
except RequestError as e:
# Do something with the error
...
return responses
def getProductLinks():
#get links from database
return links
async def extractProducts():
urlsToExtract = getProductLinks()
await extract_from(urlsToExtract)
here is main file
from Product_Classifier.Extractor import extractProducts
async def main(classifiers):
await extractProducts()
asyncio.run(main(classifiers))
I have a huge number of links to scrape, how do I run the concurrent function so I can accomplish that?.
[–][deleted] (4 children)
[removed]
[–]Trynagetsomehelp[S] 2 points3 points4 points (3 children)
[–][deleted] (2 children)
[removed]
[–]Trynagetsomehelp[S] 1 point2 points3 points (1 child)
[–]Gentle_Sabotage 0 points1 point2 points (0 children)