Python - Set the first element of a generator - Applied to itertools -
what want achieve generate combinations in different processes them faster.
in this post there'a method kth combination of set of n numbers in groups of p numbers.
once have kth combination, can manage generate rest of combinations in different processes (dividing total number of combinations between processes know combination each process has begin with).
the way i've done has been same function gets kth combination, inside loop between first , last combination process has generate.
this works, it's quite slow, it's running inside loop (so slow it's faster generate combinations in 1 process itertools.combinations() instead of previous method running in 8 different processes).
after that, wonder if there's way set first element generator has begin with, beneficts of both methods (get first combination each process has begin kth combination method, , speed of itertools.combinations()).
edit: code added
import time import operator op math import factorial itertools import combinations multiprocessing import pool def ncr(n, r): # https://stackoverflow.com/a/4941932/1167783 r = min(r, n-r) if r == 0: return 1 numer = reduce(op.mul, xrange(n, n-r, -1)) denom = reduce(op.mul, xrange(1, r+1)) return numer // denom def kthcombination(k, l, r): # https://stackoverflow.com/a/1776884/1167783 if r == 0: return [] elif len(l) == r: return l else: = ncr(len(l)-1, r-1) if k < i: return l[0:1] + kthcombination(k, l[1:], r-1) else: return kthcombination(k-i, l[1:], r) def create_dividing_list(num_total, num_div): div = num_total / num_div remaining = num_total - div * num_div dividing_list = [] in xrange(0, remaining*(div+1), div+1): dividing_list.append([i, i+div+1]) in xrange(remaining*(div+1), num_total, div): dividing_list.append([i, i+div]) return dividing_list def comb_thread(comb_initial, comb_final, numbers_list, p): comb in xrange(comb_initial, comb_final): # something, example, store combinations # timing i'm going simple x = kthcombination(comb, numbers_list, p) def comb_main(n, p): cpus = 8 numbers_list = [i in range(n)] total_comb_number = factorial(n)/(factorial(p)*factorial(n-p)) if cpus < len(numbers_list): # list style >> [['comb_initial_1', 'comb_final_1'], ['comb_initial_2', 'comb_final_2'], ... ] dividing_list = create_dividing_list(total_comb_number, cpus) else: # list style >> [['comb_initial_1', 'comb_final_1'], ['comb_initial_2', 'comb_final_2'], ... ] dividing_list = create_dividing_list(total_comb_number, len(numbers_list)) if cpus >= len(numbers_list): pool = pool(processes=len(numbers_list)) in range(len(dividing_list)): pool.apply_async(comb_thread, [dividing_list[i][0], dividing_list[i][1], numbers_list, p]) else: pool = pool(processes=cpus) in range(cpus): pool.apply_async(comb_thread, [dividing_list[i][0], dividing_list[i][1], numbers_list, p]) pool.close() pool.join() print '\n' def iter(n, p): in combinations([i in range(n)], p): # something, example, store combinations # timing i'm going simple x = if __name__ == "__main__": n = 20 p = 10 print '%s combinations' % (factorial(n)/(factorial(p)*factorial(n-p))) t0_mult = time.time() comb_main(n, p) t1_mult = time.time() total_mult = t1_mult - t0_mult t0_iter = time.time() iter(n, p) t1_iter = time.time() total_iter = t1_iter - t0_iter print 'multiprocessing: %s' %total_mult print 'itertools: %s' %total_iter
Comments
Post a Comment