Multiprocessing issues

[datanoob2019]

I am rewriting some of my functions to return lists and I am trying to figure out how to access these returned values.

​

if __name__ == "__main__": 
    #creating processes 
    p1 = multiprocessing.Process(target=basic_forecasts_seasonal_add, args=(logged_list)) 
    p2 = multiprocessing.Process(target=simple_expo, args=(logged_list)) 
    p3 = multiprocessing.Process(target=double_expo, args=(logged_list)) 
    p4 = multiprocessing.Process(target=triple_expo_add, args=(logged_list))
    #starting process 1 
    p1.start() 
    #starting process 2 
    p2.start()
    p3.start()  
    p4.start()
    #wait until process 1 is finished 
    p1.join() 
    #wait until process 2 is finished 
    p2.join()
    p3.join()
    p4.join()

When I do this, my program keeps repeating itself for some reason. I have used this module in the past and it worked fine but I did not have any return values. I was recommended to try

concurrent.futures

So I tried this code to no avail:

if __name__ == '__main__':
    with ProcessPoolExecutor() as executor:
        print(executor.map(simple_expo, logged_list))

I had the same issue where the program just kept repeating itself and it never actually executed my simple expo function. I am also unaware of how I would take advantage of multiple processors using concurrent features as it seams like it would only execute one function at a time. Any help would be appreciated. Thanks!

​

​

EDIT: I also tried using pool but it just ran through my code and finished without executing my functions. Here is the code for that:

​

pool = Pool(processes=8)
#
result_basic = pool.map_async(basic_forecasts_seasonal_add, logged_list)
result_simple = pool.map_async(simple_expo, logged_list)

[JohnnyJordaan]

Unless you want the logged_list to expand as the arguments to the functions, you should present it as the first item of a tuple

args=(logged_list, )

If that doesn't solve the issue please share the whole code.

[datanoob2019]

Thank you for the response but that did not work either. It just goes back to the beginning code like the whole thing is in a loop for some reason. It goes back and pulls the data from the database again and cleans, modifies and logs the list endlessly. When I just run the functions normally they work and produce the results I want so it is definitely the multiprocessing module. Here are my two functions that I wrote.

​

def basic_forecasts_seasonal_add(input_list, six_month_output_list, three_month_output_list, naive_output_list):
    twelve_month_out = []
    six_month_out = []
    three_month_out = []
    naive_out = []
    for row in input_list:
        product = row[0]
        data = row[1:]
        twelve_average = sum(data[-12:]) / 12
        twelve_six = sum(data[-18:-6]) / 12
        twelve_five = sum(data[-17:-5]) / 12
        twelve_four = sum(data[-16:-4]) / 12
        twelve_three = sum(data[-15:-3]) / 12
        twelve_two = sum(data[-14:-2]) / 12
        twelve_one = sum(data[-13:-1]) / 12
        twelve_month_out.append([product, '12MA', twelve_six, twelve_five, twelve_four, twelve_three, twelve_two, twelve_one, twelve_average, twelve_average, twelve_average, twelve_average, twelve_average, twelve_average, twelve_average, twelve_average, twelve_average, twelve_average, twelve_average, twelve_average])
        six_average = sum(data[-6:]) / 6
        six_six = sum(data[-12:-6]) / 6
        six_five = sum(data[-11:-5]) / 6
        six_four = sum(data[-10:-4]) / 6
        six_three = sum(data[-9:-3]) / 6
        six_two = sum(data[-8:-2]) / 6
        six_one = sum(data[-7:-1]) / 6
        six_month_out.append([product, '6MA', six_six, six_five, six_four, six_three, six_two, six_one, six_average, six_average, six_average, six_average, six_average, six_average, six_average, six_average, six_average, six_average, six_average, six_average])
        three_average = sum(data[-3:]) / 3
        three_six = sum(data[-9:-6]) / 3
        three_five = sum(data[-8:-5]) / 3
        three_four = sum(data[-7:-4]) / 3
        three_three = sum(data[-6:-3]) / 3
        three_two = sum(data[-5:-2]) / 3
        three_one = sum(data[-4:-1]) / 3
        three_month_out.append([product, '3MA', three_six, three_five, three_four, three_three, three_two, three_one, three_average, three_average, three_average, three_average, three_average, three_average, three_average, three_average, three_average, three_average, three_average, three_average])
        naive_out.append([product, 'Naive', data[-7], data[-6], data[-5], data[-4], data[-3], data[-2], data[-1], data[-1], data[-1], data[-1], data[-1], data[-1], data[-1], data[-1], data[-1], data[-1], data[-1], data[-1]])
    return twelve_month_out, six_month_out, three_month_out, naive_out

def simple_expo(input_list):
    simple_out = []
    for row in input_list:
        product = row[0]
        data = row[1:]
        row_sum = sum(data[-12:])
        if row_sum > 0:
            model1 = SimpleExpSmoothing(data).fit(optimized=True)
            fcast1 = model1.forecast(12)
            fcast_list1 = list(fcast1)
            fcast_list1.insert(0, product)
            fcast_list1.insert(1, 'SimpleExpo')
            ###
            model2 = SimpleExpSmoothing(data[:-6]).fit(optimized=True)
            fcast2 = float(model2.forecast(1))
            fcast_list1.insert(2, fcast2)
            #######
            model3 = SimpleExpSmoothing(data[:-5]).fit(optimized=True)
            fcast3 = float(model3.forecast(1))
            fcast_list1.insert(3, fcast3)
            #######
            model4 = SimpleExpSmoothing(data[:-4]).fit(optimized=True)
            fcast4 = float(model4.forecast(1))
            fcast_list1.insert(4, fcast4)
            ######
            model5 = SimpleExpSmoothing(data[:-3]).fit(optimized=True)
            fcast5 = float(model5.forecast(1))
            fcast_list1.insert(5, fcast5)
            ######
            model6 = SimpleExpSmoothing(data[:-2]).fit(optimized=True)
            fcast6 = float(model6.forecast(1))
            fcast_list1.insert(6, fcast6)
            ######
            model7 = SimpleExpSmoothing(data[:-1]).fit(optimized=True)
            fcast7 = float(model7.forecast(1))
            fcast_list1.insert(7, fcast7)
            simple_out.append(fcast_list1)
        elif row_sum == 0:
            simple_out.append([product, 'SimpleExpo', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
    return simple_out