luminoso_api/v4_json_stream.py
def _read_csv(reader, header, encode_fn):
    """
    Given a constructed CSV reader object, a header row that we've read, and
    a detected encoding, yield its rows as dictionaries.
    """
    for row in reader:
        if len(row) == 0:
            continue
        row = [encode_fn(cell) for cell in row]
        row_list = zip(header, row)
        row_dict = dict(row_list)
        if len(row_dict['text']) == 0:
            continue
        row_dict['text'] = unicodedata.normalize(
            'NFKC', row_dict['text'].strip()
        )
        if row_dict.get('title') == '':
            del row_dict['title']
        if 'date' in row_dict:
            # We handle dates further in open_json_or_csv_somehow
            if row_dict['date'] == '':
                del row_dict['date']
        if 'subset' in row_dict:
            subsets = [cell[1] for cell in row_list
                       if cell[1] != '' and cell[0] == 'subset']
            if subsets:
                row_dict['subsets'] = subsets
            if 'subset' in row_dict:
                del row_dict['subset']
        yield row_dict
def _read_csv(reader, header, encode_fn):
    """
    Given a constructed CSV reader object, a header row that we've read, and
    a detected encoding, yield its rows as dictionaries.
    """
    for row in reader:
        if len(row) == 0:
            continue
        row = [encode_fn(cell) for cell in row]
        row_list = zip(header, row)
        row_dict = dict(row_list)
        if len(row_dict['text']) == 0:
            continue
        row_dict['text'] = unicodedata.normalize(
            'NFKC', row_dict['text'].strip()
        )
        if row_dict.get('title') == '':
            del row_dict['title']
        if 'date' in row_dict:
            # We handle dates further in open_json_or_csv_somehow
            if row_dict['date'] == '':
                del row_dict['date']
        if 'subset' in row_dict:
            subsets = [cell[1] for cell in row_list
                       if cell[1] != '' and cell[0] == 'subset']
            if subsets:
                row_dict['subsets'] = subsets
            if 'subset' in row_dict:
                del row_dict['subset']
        yield row_dict
def _read_csv ( reader , header , encode_fn ) : for row in reader : if len ( row ) == 0 : continue row = [ encode_fn ( cell ) for cell in row ] row_list = zip ( header , row ) row_dict = dict ( row_list ) if len ( row_dict [ 'text' ] ) == 0 : continue row_dict [ 'text' ] = unicodedata . normalize ( 'NFKC' , row_dict [ 'text' ] . strip ( ) ) if row_dict . get ( 'title' ) == '' : del row_dict [ 'title' ] if 'date' in row_dict : # We handle dates further in open_json_or_csv_somehow if row_dict [ 'date' ] == '' : del row_dict [ 'date' ] if 'subset' in row_dict : subsets = [ cell [ 1 ] for cell in row_list if cell [ 1 ] != '' and cell [ 0 ] == 'subset' ] if subsets : row_dict [ 'subsets' ] = subsets if 'subset' in row_dict : del row_dict [ 'subset' ] yield row_dict
Given a constructed CSV reader object, a header row that we've read, and
    a detected encoding, yield its rows as dictionaries.
Given a constructed CSV reader object a header row that we ve read and a detected encoding yield its rows as dictionaries .