python - Delete Objects in a List as Passed to Multiprocessing -
i need pass each object in large list function. after function completes no longer need object passed function , delete object save memory. if working single process following:
result = [] while len(mylist) > 0: result.append(myfunc(mylist.pop()) as loop on mylist pop off each object in list such object no longer stored in mylist after it's passed function. how achieve same effect in parallel using multiprocessing?
a simple consumer example (credits go here) :
import multiprocessing import time import random class consumer(multiprocessing.process): def __init__(self, task_queue, result_queue): multiprocessing.process.__init__(self) self.task_queue = task_queue self.result_queue = result_queue def run(self): while true: task = self.task_queue.get() if task none: # poison pill means shutdown self.task_queue.task_done() break answer = task.process() self.task_queue.task_done() self.result_queue.put(answer) return class task(object): def process(self): time.sleep(0.1) # pretend take time work return random.randint(0, 100) if __name__ == '__main__': # establish communication queues tasks = multiprocessing.joinablequeue() results = multiprocessing.queue() # start consumers num_consumers = multiprocessing.cpu_count() * 2 consumers = [consumer(tasks, results) in xrange(num_consumers)] consumer in consumers: consumer.start() # enqueue jobs num_jobs = 10 _ in xrange(num_jobs): tasks.put(task()) # add poison pill each consumer _ in xrange(num_consumers): tasks.put(none) # wait tasks finish tasks.join() # start printing results while num_jobs: result = results.get() print 'result:', result num_jobs -= 1
Comments
Post a Comment