goose3/extractors/content.py
def get_siblings_content(self, current_sibling, baselinescore_siblings_para):
        """
        adds any siblings that may have a decent score to this node
        """
        if current_sibling.tag == 'p' and self.parser.getText(current_sibling):
            tmp = current_sibling
            if tmp.tail:
                tmp = deepcopy(tmp)
                tmp.tail = ''
            return [tmp]
        else:
            potential_paragraphs = self.parser.getElementsByTag(current_sibling, tag='p')
            if potential_paragraphs is None:
                return None

            paragraphs = list()
            for first_paragraph in potential_paragraphs:
                text = self.parser.getText(first_paragraph)
                if text:  # no len(text) > 0
                    word_stats = self.stopwords_class(language=self.get_language()).get_stopword_count(text)
                    paragraph_score = word_stats.get_stopword_count()
                    sibling_baseline_score = float(.30)
                    high_link_density = self.is_highlink_density(first_paragraph)
                    score = float(baselinescore_siblings_para * sibling_baseline_score)
                    if score < paragraph_score and not high_link_density:
                        para = self.parser.createElement(tag='p', text=text, tail=None)
                        paragraphs.append(para)
            return paragraphs
def get_siblings_content(self, current_sibling, baselinescore_siblings_para):
        """
        adds any siblings that may have a decent score to this node
        """
        if current_sibling.tag == 'p' and self.parser.getText(current_sibling):
            tmp = current_sibling
            if tmp.tail:
                tmp = deepcopy(tmp)
                tmp.tail = ''
            return [tmp]
        else:
            potential_paragraphs = self.parser.getElementsByTag(current_sibling, tag='p')
            if potential_paragraphs is None:
                return None

            paragraphs = list()
            for first_paragraph in potential_paragraphs:
                text = self.parser.getText(first_paragraph)
                if text:  # no len(text) > 0
                    word_stats = self.stopwords_class(language=self.get_language()).get_stopword_count(text)
                    paragraph_score = word_stats.get_stopword_count()
                    sibling_baseline_score = float(.30)
                    high_link_density = self.is_highlink_density(first_paragraph)
                    score = float(baselinescore_siblings_para * sibling_baseline_score)
                    if score < paragraph_score and not high_link_density:
                        para = self.parser.createElement(tag='p', text=text, tail=None)
                        paragraphs.append(para)
            return paragraphs
def get_siblings_content ( self , current_sibling , baselinescore_siblings_para ) : if current_sibling . tag == 'p' and self . parser . getText ( current_sibling ) : tmp = current_sibling if tmp . tail : tmp = deepcopy ( tmp ) tmp . tail = '' return [ tmp ] else : potential_paragraphs = self . parser . getElementsByTag ( current_sibling , tag = 'p' ) if potential_paragraphs is None : return None paragraphs = list ( ) for first_paragraph in potential_paragraphs : text = self . parser . getText ( first_paragraph ) if text : # no len(text) > 0 word_stats = self . stopwords_class ( language = self . get_language ( ) ) . get_stopword_count ( text ) paragraph_score = word_stats . get_stopword_count ( ) sibling_baseline_score = float ( .30 ) high_link_density = self . is_highlink_density ( first_paragraph ) score = float ( baselinescore_siblings_para * sibling_baseline_score ) if score < paragraph_score and not high_link_density : para = self . parser . createElement ( tag = 'p' , text = text , tail = None ) paragraphs . append ( para ) return paragraphs
adds any siblings that may have a decent score to this node
adds any siblings that may have a decent score to this node