deeppavlov/evolve.py
def run_population(population, evolution, gpus):
    """
    Change save and load paths for obtained population, save config.json with model config,
    run population via current python executor (with which evolve.py already run)
    and on given devices (-1 means CPU, other integeres - visible for evolve.py GPUs)
    Args:
        population: list of dictionaries - configs of current population
        evolution: ParamsEvolution
        gpus: list of given devices (list of integers)

    Returns:
        None
    """
    population_size = len(population)
    for k in range(population_size // len(gpus) + 1):
        procs = []
        for j in range(len(gpus)):
            i = k * len(gpus) + j
            if i < population_size:
                save_path = expand_path(
                    evolution.get_value_from_config(parse_config(population[i]),
                                                    evolution.path_to_models_save_path))

                save_path.mkdir(parents=True, exist_ok=True)
                f_name = save_path / "config.json"
                save_json(population[i], f_name)

                with save_path.joinpath('out.txt').open('w', encoding='utf8') as outlog,\
                        save_path.joinpath('err.txt').open('w', encoding='utf8') as errlog:
                    env = dict(os.environ)
                    if len(gpus) > 1 or gpus[0] != -1:
                        env['CUDA_VISIBLE_DEVICES'] = str(gpus[j])

                    procs.append(Popen("{} -m deeppavlov train {}".format(sys.executable, str(f_name)),
                                       shell=True, stdout=outlog, stderr=errlog, env=env))
        for j, proc in enumerate(procs):
            i = k * len(gpus) + j
            log.info(f'Waiting on {i}th proc')
            if proc.wait() != 0:
                save_path = expand_path(
                    evolution.get_value_from_config(parse_config(population[i]),
                                                    evolution.path_to_models_save_path))
                with save_path.joinpath('err.txt').open(encoding='utf8') as errlog:
                    log.warning(f'Population {i} returned an error code {proc.returncode} and an error log:\n' +
                                errlog.read())
    return None
def run_population(population, evolution, gpus):
    """
    Change save and load paths for obtained population, save config.json with model config,
    run population via current python executor (with which evolve.py already run)
    and on given devices (-1 means CPU, other integeres - visible for evolve.py GPUs)
    Args:
        population: list of dictionaries - configs of current population
        evolution: ParamsEvolution
        gpus: list of given devices (list of integers)

    Returns:
        None
    """
    population_size = len(population)
    for k in range(population_size // len(gpus) + 1):
        procs = []
        for j in range(len(gpus)):
            i = k * len(gpus) + j
            if i < population_size:
                save_path = expand_path(
                    evolution.get_value_from_config(parse_config(population[i]),
                                                    evolution.path_to_models_save_path))

                save_path.mkdir(parents=True, exist_ok=True)
                f_name = save_path / "config.json"
                save_json(population[i], f_name)

                with save_path.joinpath('out.txt').open('w', encoding='utf8') as outlog,\
                        save_path.joinpath('err.txt').open('w', encoding='utf8') as errlog:
                    env = dict(os.environ)
                    if len(gpus) > 1 or gpus[0] != -1:
                        env['CUDA_VISIBLE_DEVICES'] = str(gpus[j])

                    procs.append(Popen("{} -m deeppavlov train {}".format(sys.executable, str(f_name)),
                                       shell=True, stdout=outlog, stderr=errlog, env=env))
        for j, proc in enumerate(procs):
            i = k * len(gpus) + j
            log.info(f'Waiting on {i}th proc')
            if proc.wait() != 0:
                save_path = expand_path(
                    evolution.get_value_from_config(parse_config(population[i]),
                                                    evolution.path_to_models_save_path))
                with save_path.joinpath('err.txt').open(encoding='utf8') as errlog:
                    log.warning(f'Population {i} returned an error code {proc.returncode} and an error log:\n' +
                                errlog.read())
    return None
def run_population ( population , evolution , gpus ) : population_size = len ( population ) for k in range ( population_size // len ( gpus ) + 1 ) : procs = [ ] for j in range ( len ( gpus ) ) : i = k * len ( gpus ) + j if i < population_size : save_path = expand_path ( evolution . get_value_from_config ( parse_config ( population [ i ] ) , evolution . path_to_models_save_path ) ) save_path . mkdir ( parents = True , exist_ok = True ) f_name = save_path / "config.json" save_json ( population [ i ] , f_name ) with save_path . joinpath ( 'out.txt' ) . open ( 'w' , encoding = 'utf8' ) as outlog , save_path . joinpath ( 'err.txt' ) . open ( 'w' , encoding = 'utf8' ) as errlog : env = dict ( os . environ ) if len ( gpus ) > 1 or gpus [ 0 ] != - 1 : env [ 'CUDA_VISIBLE_DEVICES' ] = str ( gpus [ j ] ) procs . append ( Popen ( "{} -m deeppavlov train {}" . format ( sys . executable , str ( f_name ) ) , shell = True , stdout = outlog , stderr = errlog , env = env ) ) for j , proc in enumerate ( procs ) : i = k * len ( gpus ) + j log . info ( f'Waiting on {i}th proc' ) if proc . wait ( ) != 0 : save_path = expand_path ( evolution . get_value_from_config ( parse_config ( population [ i ] ) , evolution . path_to_models_save_path ) ) with save_path . joinpath ( 'err.txt' ) . open ( encoding = 'utf8' ) as errlog : log . warning ( f'Population {i} returned an error code {proc.returncode} and an error log:\n' + errlog . read ( ) ) return None
Change save and load paths for obtained population, save config.json with model config,
    run population via current python executor (with which evolve.py already run)
    and on given devices (-1 means CPU, other integeres - visible for evolve.py GPUs)
    Args:
        population: list of dictionaries - configs of current population
        evolution: ParamsEvolution
        gpus: list of given devices (list of integers)

    Returns:
        None
Change save and load paths for obtained population save config . json with model config run population via current python executor ( with which evolve . py already run ) and on given devices ( - 1 means CPU other integeres - visible for evolve . py GPUs ) Args : population : list of dictionaries - configs of current population evolution : ParamsEvolution gpus : list of given devices ( list of integers )