Source code for tkp.distribute
"""
`tkp.distribute` implement various computation distribution methods. All
sub modules should communicate with the other parts of TKP through the
`tkp.steps` submodule.
"""
import importlib
import logging
logger = logging.getLogger(__name__)
[docs]class Runner(object):
def __init__(self, distributor, cores=0):
"""
Args:
distributor: the name of the distribution method, example multiproc
"""
logger.debug("Using %s distribution method" % distributor)
self.distributor = distributor
self.mod_path = __name__ + '.' + distributor
try:
self.module = importlib.import_module(self.mod_path)
self.tasks = importlib.import_module(self.mod_path + '.tasks')
except ImportError:
raise NotImplementedError("%s not found" % self.mod_path)
if not hasattr(self.module, 'map'):
raise NotImplementedError("%s misses map function" % self.mod_path)
self.module.set_cores(cores)
[docs] def map(self, func_name, iterable, args=[]):
"""
args:
func: The function to be called
iterable: a list of objects to iterate over
arguments: list of arguments to give to the function
returns:
the results of all mapped functions
"""
func = self.get_func(func_name)
return self.module.map(func, iterable, args)
[docs] def get_func(self, func_name):
try:
return getattr(self.tasks, func_name)
except AttributeError:
raise NotImplementedError('%s not implemented for %s' %
(func_name, self.mod_path))