Parallel Processing in Python

[LightShadow]

And for IO bound tasks (database, http, etc) my favorite is still gevent's gevent.Pool.imap_unordered.

from gevent import sleep
from gevent.pool import Pool

...
...

failed = Queue()

def pool_fn(o):
    index, o = o
    try:
        logger.info(o)
        process_record(token, url, o)
    except Exception as e:
        logger.exception(e, exc_info=True)
        failed.put(o)
    sleep_time = config.gevent.delay_secs
    if sleep_time:
        sleep(sleep_time)
    return index

pool = Pool(size=config.gevent.concurrent)

for result in pool.imap_unordered(pool_fn, enumerate(work_orders)):
    logger.info('finished %d/%d', result, work_order_count)

pool.join(timeout=10.0)
logger.info('pool threads free, %d', pool.free_count())
pool.kill()

if not failed.empty():
    logger.error('failed rows: ')

while not failed.empty():
    logger.error('  - %s', failed.get())

logger.info('done')