deeppavlov/core/commands/train.py
def read_data_by_config(config: dict):
    """Read data by dataset_reader from specified config."""
    dataset_config = config.get('dataset', None)

    if dataset_config:
        config.pop('dataset')
        ds_type = dataset_config['type']
        if ds_type == 'classification':
            reader = {'class_name': 'basic_classification_reader'}
            iterator = {'class_name': 'basic_classification_iterator'}
            config['dataset_reader'] = {**dataset_config, **reader}
            config['dataset_iterator'] = {**dataset_config, **iterator}
        else:
            raise Exception("Unsupported dataset type: {}".format(ds_type))

    try:
        reader_config = dict(config['dataset_reader'])
    except KeyError:
        raise ConfigError("No dataset reader is provided in the JSON config.")

    reader = get_model(reader_config.pop('class_name'))()
    data_path = reader_config.pop('data_path', '')
    if isinstance(data_path, list):
        data_path = [expand_path(x) for x in data_path]
    else:
        data_path = expand_path(data_path)

    return reader.read(data_path, **reader_config)
def read_data_by_config(config: dict):
    """Read data by dataset_reader from specified config."""
    dataset_config = config.get('dataset', None)

    if dataset_config:
        config.pop('dataset')
        ds_type = dataset_config['type']
        if ds_type == 'classification':
            reader = {'class_name': 'basic_classification_reader'}
            iterator = {'class_name': 'basic_classification_iterator'}
            config['dataset_reader'] = {**dataset_config, **reader}
            config['dataset_iterator'] = {**dataset_config, **iterator}
        else:
            raise Exception("Unsupported dataset type: {}".format(ds_type))

    try:
        reader_config = dict(config['dataset_reader'])
    except KeyError:
        raise ConfigError("No dataset reader is provided in the JSON config.")

    reader = get_model(reader_config.pop('class_name'))()
    data_path = reader_config.pop('data_path', '')
    if isinstance(data_path, list):
        data_path = [expand_path(x) for x in data_path]
    else:
        data_path = expand_path(data_path)

    return reader.read(data_path, **reader_config)
def read_data_by_config ( config : dict ) : dataset_config = config . get ( 'dataset' , None ) if dataset_config : config . pop ( 'dataset' ) ds_type = dataset_config [ 'type' ] if ds_type == 'classification' : reader = { 'class_name' : 'basic_classification_reader' } iterator = { 'class_name' : 'basic_classification_iterator' } config [ 'dataset_reader' ] = { * * dataset_config , * * reader } config [ 'dataset_iterator' ] = { * * dataset_config , * * iterator } else : raise Exception ( "Unsupported dataset type: {}" . format ( ds_type ) ) try : reader_config = dict ( config [ 'dataset_reader' ] ) except KeyError : raise ConfigError ( "No dataset reader is provided in the JSON config." ) reader = get_model ( reader_config . pop ( 'class_name' ) ) ( ) data_path = reader_config . pop ( 'data_path' , '' ) if isinstance ( data_path , list ) : data_path = [ expand_path ( x ) for x in data_path ] else : data_path = expand_path ( data_path ) return reader . read ( data_path , * * reader_config )
Read data by dataset_reader from specified config.
Read data by dataset_reader from specified config .