splearn/feature_extraction/text.py
def _count_vocab(self, analyzed_docs):
        """Create sparse feature matrix, and vocabulary where fixed_vocab=False
        """
        vocabulary = self.vocabulary_
        j_indices = _make_int_array()
        indptr = _make_int_array()
        indptr.append(0)
        for doc in analyzed_docs:
            for feature in doc:
                try:
                    j_indices.append(vocabulary[feature])
                except KeyError:
                    # Ignore out-of-vocabulary items for fixed_vocab=True
                    continue
            indptr.append(len(j_indices))

        j_indices = frombuffer_empty(j_indices, dtype=np.intc)
        indptr = np.frombuffer(indptr, dtype=np.intc)
        values = np.ones(len(j_indices))

        X = sp.csr_matrix((values, j_indices, indptr),
                          shape=(len(indptr) - 1, len(vocabulary)),
                          dtype=self.dtype)
        X.sum_duplicates()

        if self.binary:
            X.data.fill(1)

        return X
def _count_vocab(self, analyzed_docs):
        """Create sparse feature matrix, and vocabulary where fixed_vocab=False
        """
        vocabulary = self.vocabulary_
        j_indices = _make_int_array()
        indptr = _make_int_array()
        indptr.append(0)
        for doc in analyzed_docs:
            for feature in doc:
                try:
                    j_indices.append(vocabulary[feature])
                except KeyError:
                    # Ignore out-of-vocabulary items for fixed_vocab=True
                    continue
            indptr.append(len(j_indices))

        j_indices = frombuffer_empty(j_indices, dtype=np.intc)
        indptr = np.frombuffer(indptr, dtype=np.intc)
        values = np.ones(len(j_indices))

        X = sp.csr_matrix((values, j_indices, indptr),
                          shape=(len(indptr) - 1, len(vocabulary)),
                          dtype=self.dtype)
        X.sum_duplicates()

        if self.binary:
            X.data.fill(1)

        return X
def _count_vocab ( self , analyzed_docs ) : vocabulary = self . vocabulary_ j_indices = _make_int_array ( ) indptr = _make_int_array ( ) indptr . append ( 0 ) for doc in analyzed_docs : for feature in doc : try : j_indices . append ( vocabulary [ feature ] ) except KeyError : # Ignore out-of-vocabulary items for fixed_vocab=True continue indptr . append ( len ( j_indices ) ) j_indices = frombuffer_empty ( j_indices , dtype = np . intc ) indptr = np . frombuffer ( indptr , dtype = np . intc ) values = np . ones ( len ( j_indices ) ) X = sp . csr_matrix ( ( values , j_indices , indptr ) , shape = ( len ( indptr ) - 1 , len ( vocabulary ) ) , dtype = self . dtype ) X . sum_duplicates ( ) if self . binary : X . data . fill ( 1 ) return X
Create sparse feature matrix, and vocabulary where fixed_vocab=False
Create sparse feature matrix and vocabulary where fixed_vocab = False