deeppavlov/models/tokenizers/utils.py
def ngramize(items: List[str], ngram_range=(1, 1)) -> Generator[List[str], Any, None]:
    """
    Make ngrams from a list of tokens/lemmas
    :param items: list of tokens, lemmas or other strings to form ngrams
    :param ngram_range: range for producing ngrams, ex. for unigrams + bigrams should be set to
    (1, 2), for bigrams only should be set to (2, 2)
    :return: ngrams (as strings) generator
    """

    ngrams = []
    ranges = [(0, i) for i in range(ngram_range[0], ngram_range[1] + 1)]
    for r in ranges:
        ngrams += list(zip(*[items[j:] for j in range(*r)]))

    formatted_ngrams = [' '.join(item) for item in ngrams]

    yield formatted_ngrams
def ngramize(items: List[str], ngram_range=(1, 1)) -> Generator[List[str], Any, None]:
    """
    Make ngrams from a list of tokens/lemmas
    :param items: list of tokens, lemmas or other strings to form ngrams
    :param ngram_range: range for producing ngrams, ex. for unigrams + bigrams should be set to
    (1, 2), for bigrams only should be set to (2, 2)
    :return: ngrams (as strings) generator
    """

    ngrams = []
    ranges = [(0, i) for i in range(ngram_range[0], ngram_range[1] + 1)]
    for r in ranges:
        ngrams += list(zip(*[items[j:] for j in range(*r)]))

    formatted_ngrams = [' '.join(item) for item in ngrams]

    yield formatted_ngrams
def ngramize ( items : List [ str ] , ngram_range = ( 1 , 1 ) ) -> Generator [ List [ str ] , Any , None ] : ngrams = [ ] ranges = [ ( 0 , i ) for i in range ( ngram_range [ 0 ] , ngram_range [ 1 ] + 1 ) ] for r in ranges : ngrams += list ( zip ( * [ items [ j : ] for j in range ( * r ) ] ) ) formatted_ngrams = [ ' ' . join ( item ) for item in ngrams ] yield formatted_ngrams
Make ngrams from a list of tokens/lemmas
    :param items: list of tokens, lemmas or other strings to form ngrams
    :param ngram_range: range for producing ngrams, ex. for unigrams + bigrams should be set to
    (1, 2), for bigrams only should be set to (2, 2)
    :return: ngrams (as strings) generator
Make ngrams from a list of tokens / lemmas : param items : list of tokens lemmas or other strings to form ngrams : param ngram_range : range for producing ngrams ex . for unigrams + bigrams should be set to ( 1 2 ) for bigrams only should be set to ( 2 2 ) : return : ngrams ( as strings ) generator