luminoso_api/v5_download.py
def iterate_docs(client, expanded=False, progress=False):
    """
    Yield each document in a Luminoso project in turn. Requires a client whose
    URL points to a project.

    If expanded=True, it will include additional fields that Luminoso added in
    its analysis, such as 'terms' and 'vector'.

    Otherwise, it will contain only the fields necessary to reconstruct the
    document: 'title', 'text', and 'metadata'.

    Shows a progress bar if progress=True.
    """
    # Get total number of docs from the project record
    num_docs = client.get()['document_count']
    progress_bar = None
    try:
        if progress:
            progress_bar = tqdm(desc='Downloading documents', total=num_docs)

        for offset in range(0, num_docs, DOCS_PER_BATCH):
            response = client.get('docs', offset=offset, limit=DOCS_PER_BATCH)
            docs = response['result']
            for doc in docs:
                # Get the appropriate set of fields for each document
                if expanded:
                    for field in UNNECESSARY_FIELDS:
                        doc.pop(field, None)
                else:
                    doc = {field: doc[field] for field in CONCISE_FIELDS}

                if progress:
                    progress_bar.update()
                yield doc

    finally:
        if progress:
            progress_bar.close()
def iterate_docs(client, expanded=False, progress=False):
    """
    Yield each document in a Luminoso project in turn. Requires a client whose
    URL points to a project.

    If expanded=True, it will include additional fields that Luminoso added in
    its analysis, such as 'terms' and 'vector'.

    Otherwise, it will contain only the fields necessary to reconstruct the
    document: 'title', 'text', and 'metadata'.

    Shows a progress bar if progress=True.
    """
    # Get total number of docs from the project record
    num_docs = client.get()['document_count']
    progress_bar = None
    try:
        if progress:
            progress_bar = tqdm(desc='Downloading documents', total=num_docs)

        for offset in range(0, num_docs, DOCS_PER_BATCH):
            response = client.get('docs', offset=offset, limit=DOCS_PER_BATCH)
            docs = response['result']
            for doc in docs:
                # Get the appropriate set of fields for each document
                if expanded:
                    for field in UNNECESSARY_FIELDS:
                        doc.pop(field, None)
                else:
                    doc = {field: doc[field] for field in CONCISE_FIELDS}

                if progress:
                    progress_bar.update()
                yield doc

    finally:
        if progress:
            progress_bar.close()
def iterate_docs ( client , expanded = False , progress = False ) : # Get total number of docs from the project record num_docs = client . get ( ) [ 'document_count' ] progress_bar = None try : if progress : progress_bar = tqdm ( desc = 'Downloading documents' , total = num_docs ) for offset in range ( 0 , num_docs , DOCS_PER_BATCH ) : response = client . get ( 'docs' , offset = offset , limit = DOCS_PER_BATCH ) docs = response [ 'result' ] for doc in docs : # Get the appropriate set of fields for each document if expanded : for field in UNNECESSARY_FIELDS : doc . pop ( field , None ) else : doc = { field : doc [ field ] for field in CONCISE_FIELDS } if progress : progress_bar . update ( ) yield doc finally : if progress : progress_bar . close ( )
Yield each document in a Luminoso project in turn. Requires a client whose
    URL points to a project.

    If expanded=True, it will include additional fields that Luminoso added in
    its analysis, such as 'terms' and 'vector'.

    Otherwise, it will contain only the fields necessary to reconstruct the
    document: 'title', 'text', and 'metadata'.

    Shows a progress bar if progress=True.
Yield each document in a Luminoso project in turn . Requires a client whose URL points to a project .