lib/web/wikipedia.py
def plain(self, markup):
        
        """ Strips Wikipedia markup from given text.
        
        This creates a "plain" version of the markup,
        stripping images and references and the like.
        Does some commonsense maintenance as well,
        like collapsing multiple spaces.
        If you specified full_strip=False for WikipediaPage instance,
        some markup is preserved as HTML (links, bold, italic).
        
        """
        
        # Strip bold and italic.
        if self.full_strip:
            markup = markup.replace("'''", "")
            markup = markup.replace("''", "")
        else:
            markup = re.sub("'''([^']*?)'''", "<b>\\1</b>", markup)
            markup = re.sub("''([^']*?)''", "<i>\\1</i>", markup)
        
        # Strip image gallery sections.
        markup = re.sub(self.re["gallery"], "", markup)
        
        # Strip tables.
        markup = re.sub(self.re["table"], "", markup)
        markup = markup.replace("||", "")
        markup = markup.replace("|}", "")
        
        # Strip links, keeping the display alias.
        # We'll strip the ending ]] later.
        if self.full_strip:
            markup = re.sub(r"\[\[[^\]]*?\|", "", markup)
        else:
            markup = re.sub(r"\[\[([^]|]*|)\]\]", '<a href="\\1">\\1</a>', markup)
            markup = re.sub(r"\[\[([^]|]*|)\|([^]]*)\]\]", '<a href="\\1">\\2</a>', markup)    

        # Strip translations, users, etc.
        markup = re.sub(self.re["translation"], "", markup)
        
        # This math TeX is not supported:
        markup = markup.replace("\displaytyle", "")
        markup = markup.replace("\textstyle", "")
        markup = markup.replace("\scriptstyle", "")
        markup = markup.replace("\scriptscriptstyle", "")
        
        # Before stripping [ and ] brackets,
        # make sure they are retained inside <math></math> equations.
        markup = re.sub("(<math>.*?)\[(.*?</math>)", "\\1MATH___OPEN\\2", markup)
        markup = re.sub("(<math>.*?)\](.*?</math>)", "\\1MATH___CLOSE\\2", markup)
        markup = markup.replace("[", "")
        markup = markup.replace("]", "")
        markup = markup.replace("MATH___OPEN", "[")
        markup = markup.replace("MATH___CLOSE", "]")
        
        # a) Strip references.
        # b) Strip <ref></ref> tags.
        # c) Strip <ref name="" /> tags.
        # d) Replace --REF--(12) by [12].
        # e) Remove space between [12] and trailing punctuation .,
        # f) Remove HTML comment <!-- -->
        # g) Keep the Latin Extended-B template: {{latinx| }}
        # h) Strip Middle-Earth references.
        # i) Keep quotes: {{quote| }}
        # j) Remove templates
        markup = re.sub(self.re["reference"], "", markup)                  # a
        markup = re.sub("</{0,1}ref.*?>", "", markup)                      # b
        markup = re.sub("<ref name=\".*?\" {0,1}/>", "", markup)           # c
        markup = re.sub(self.ref+"\(([0-9]*?)\)", "[\\1] ", markup)        # d
        markup = re.sub("\] ([,.\"\?\)])", "]\\1", markup)                 # e
        markup = re.sub(self.re["comment"], "", markup)                    # f
        markup = re.sub("\{\{latinx\|(.*?)\}\}", "\\1", markup)            # g
        markup = re.sub("\{\{ME-ref.*?\}\}", "", markup)                   # h
        markup = re.sub("\{\{quote\|(.*?)\}\}", "\"\\1\"", markup)         # i
        markup = re.sub(re.compile("\{\{.*?\}\}", re.DOTALL), "", markup)  # j
        markup = markup.replace("}}", "")
        
        # Collapse multiple spaces between words,
        # unless they appear in preformatted text.
        markup = re.sub("<br.*?/{0,1}>", " ", markup)
        markup = markup.split("\n")
        for i in range(len(markup)):
            if not markup[i].startswith(" "):
                markup[i] = re.sub(r"[ ]+", " ", markup[i])
        markup = "\n".join(markup)
        markup = markup.replace(" .", ".")
        
        # Strip all HTML except <math> tags.
        if self.full_strip:
            markup = strip_tags(markup, exclude=["math"], linebreaks=True)
        
        markup = markup.strip()
        return markup
def plain(self, markup):
        
        """ Strips Wikipedia markup from given text.
        
        This creates a "plain" version of the markup,
        stripping images and references and the like.
        Does some commonsense maintenance as well,
        like collapsing multiple spaces.
        If you specified full_strip=False for WikipediaPage instance,
        some markup is preserved as HTML (links, bold, italic).
        
        """
        
        # Strip bold and italic.
        if self.full_strip:
            markup = markup.replace("'''", "")
            markup = markup.replace("''", "")
        else:
            markup = re.sub("'''([^']*?)'''", "<b>\\1</b>", markup)
            markup = re.sub("''([^']*?)''", "<i>\\1</i>", markup)
        
        # Strip image gallery sections.
        markup = re.sub(self.re["gallery"], "", markup)
        
        # Strip tables.
        markup = re.sub(self.re["table"], "", markup)
        markup = markup.replace("||", "")
        markup = markup.replace("|}", "")
        
        # Strip links, keeping the display alias.
        # We'll strip the ending ]] later.
        if self.full_strip:
            markup = re.sub(r"\[\[[^\]]*?\|", "", markup)
        else:
            markup = re.sub(r"\[\[([^]|]*|)\]\]", '<a href="\\1">\\1</a>', markup)
            markup = re.sub(r"\[\[([^]|]*|)\|([^]]*)\]\]", '<a href="\\1">\\2</a>', markup)    

        # Strip translations, users, etc.
        markup = re.sub(self.re["translation"], "", markup)
        
        # This math TeX is not supported:
        markup = markup.replace("\displaytyle", "")
        markup = markup.replace("\textstyle", "")
        markup = markup.replace("\scriptstyle", "")
        markup = markup.replace("\scriptscriptstyle", "")
        
        # Before stripping [ and ] brackets,
        # make sure they are retained inside <math></math> equations.
        markup = re.sub("(<math>.*?)\[(.*?</math>)", "\\1MATH___OPEN\\2", markup)
        markup = re.sub("(<math>.*?)\](.*?</math>)", "\\1MATH___CLOSE\\2", markup)
        markup = markup.replace("[", "")
        markup = markup.replace("]", "")
        markup = markup.replace("MATH___OPEN", "[")
        markup = markup.replace("MATH___CLOSE", "]")
        
        # a) Strip references.
        # b) Strip <ref></ref> tags.
        # c) Strip <ref name="" /> tags.
        # d) Replace --REF--(12) by [12].
        # e) Remove space between [12] and trailing punctuation .,
        # f) Remove HTML comment <!-- -->
        # g) Keep the Latin Extended-B template: {{latinx| }}
        # h) Strip Middle-Earth references.
        # i) Keep quotes: {{quote| }}
        # j) Remove templates
        markup = re.sub(self.re["reference"], "", markup)                  # a
        markup = re.sub("</{0,1}ref.*?>", "", markup)                      # b
        markup = re.sub("<ref name=\".*?\" {0,1}/>", "", markup)           # c
        markup = re.sub(self.ref+"\(([0-9]*?)\)", "[\\1] ", markup)        # d
        markup = re.sub("\] ([,.\"\?\)])", "]\\1", markup)                 # e
        markup = re.sub(self.re["comment"], "", markup)                    # f
        markup = re.sub("\{\{latinx\|(.*?)\}\}", "\\1", markup)            # g
        markup = re.sub("\{\{ME-ref.*?\}\}", "", markup)                   # h
        markup = re.sub("\{\{quote\|(.*?)\}\}", "\"\\1\"", markup)         # i
        markup = re.sub(re.compile("\{\{.*?\}\}", re.DOTALL), "", markup)  # j
        markup = markup.replace("}}", "")
        
        # Collapse multiple spaces between words,
        # unless they appear in preformatted text.
        markup = re.sub("<br.*?/{0,1}>", " ", markup)
        markup = markup.split("\n")
        for i in range(len(markup)):
            if not markup[i].startswith(" "):
                markup[i] = re.sub(r"[ ]+", " ", markup[i])
        markup = "\n".join(markup)
        markup = markup.replace(" .", ".")
        
        # Strip all HTML except <math> tags.
        if self.full_strip:
            markup = strip_tags(markup, exclude=["math"], linebreaks=True)
        
        markup = markup.strip()
        return markup
def plain ( self , markup ) : # Strip bold and italic. if self . full_strip : markup = markup . replace ( "'''" , "" ) markup = markup . replace ( "''" , "" ) else : markup = re . sub ( "'''([^']*?)'''" , "<b>\\1</b>" , markup ) markup = re . sub ( "''([^']*?)''" , "<i>\\1</i>" , markup ) # Strip image gallery sections. markup = re . sub ( self . re [ "gallery" ] , "" , markup ) # Strip tables. markup = re . sub ( self . re [ "table" ] , "" , markup ) markup = markup . replace ( "||" , "" ) markup = markup . replace ( "|}" , "" ) # Strip links, keeping the display alias. # We'll strip the ending ]] later. if self . full_strip : markup = re . sub ( r"\[\[[^\]]*?\|" , "" , markup ) else : markup = re . sub ( r"\[\[([^]|]*|)\]\]" , '<a href="\\1">\\1</a>' , markup ) markup = re . sub ( r"\[\[([^]|]*|)\|([^]]*)\]\]" , '<a href="\\1">\\2</a>' , markup ) # Strip translations, users, etc. markup = re . sub ( self . re [ "translation" ] , "" , markup ) # This math TeX is not supported: markup = markup . replace ( "\displaytyle" , "" ) markup = markup . replace ( "\textstyle" , "" ) markup = markup . replace ( "\scriptstyle" , "" ) markup = markup . replace ( "\scriptscriptstyle" , "" ) # Before stripping [ and ] brackets, # make sure they are retained inside <math></math> equations. markup = re . sub ( "(<math>.*?)\[(.*?</math>)" , "\\1MATH___OPEN\\2" , markup ) markup = re . sub ( "(<math>.*?)\](.*?</math>)" , "\\1MATH___CLOSE\\2" , markup ) markup = markup . replace ( "[" , "" ) markup = markup . replace ( "]" , "" ) markup = markup . replace ( "MATH___OPEN" , "[" ) markup = markup . replace ( "MATH___CLOSE" , "]" ) # a) Strip references. # b) Strip <ref></ref> tags. # c) Strip <ref name="" /> tags. # d) Replace --REF--(12) by [12]. # e) Remove space between [12] and trailing punctuation ., # f) Remove HTML comment <!-- --> # g) Keep the Latin Extended-B template: {{latinx| }} # h) Strip Middle-Earth references. # i) Keep quotes: {{quote| }} # j) Remove templates markup = re . sub ( self . re [ "reference" ] , "" , markup ) # a markup = re . sub ( "</{0,1}ref.*?>" , "" , markup ) # b markup = re . sub ( "<ref name=\".*?\" {0,1}/>" , "" , markup ) # c markup = re . sub ( self . ref + "\(([0-9]*?)\)" , "[\\1] " , markup ) # d markup = re . sub ( "\] ([,.\"\?\)])" , "]\\1" , markup ) # e markup = re . sub ( self . re [ "comment" ] , "" , markup ) # f markup = re . sub ( "\{\{latinx\|(.*?)\}\}" , "\\1" , markup ) # g markup = re . sub ( "\{\{ME-ref.*?\}\}" , "" , markup ) # h markup = re . sub ( "\{\{quote\|(.*?)\}\}" , "\"\\1\"" , markup ) # i markup = re . sub ( re . compile ( "\{\{.*?\}\}" , re . DOTALL ) , "" , markup ) # j markup = markup . replace ( "}}" , "" ) # Collapse multiple spaces between words, # unless they appear in preformatted text. markup = re . sub ( "<br.*?/{0,1}>" , " " , markup ) markup = markup . split ( "\n" ) for i in range ( len ( markup ) ) : if not markup [ i ] . startswith ( " " ) : markup [ i ] = re . sub ( r"[ ]+" , " " , markup [ i ] ) markup = "\n" . join ( markup ) markup = markup . replace ( " ." , "." ) # Strip all HTML except <math> tags. if self . full_strip : markup = strip_tags ( markup , exclude = [ "math" ] , linebreaks = True ) markup = markup . strip ( ) return markup
Strips Wikipedia markup from given text.
        
        This creates a "plain" version of the markup,
        stripping images and references and the like.
        Does some commonsense maintenance as well,
        like collapsing multiple spaces.
        If you specified full_strip=False for WikipediaPage instance,
        some markup is preserved as HTML (links, bold, italic).
Strips Wikipedia markup from given text . This creates a plain version of the markup stripping images and references and the like . Does some commonsense maintenance as well like collapsing multiple spaces . If you specified full_strip = False for WikipediaPage instance some markup is preserved as HTML ( links bold italic ) .