lib/web/BeautifulSoup.py
def _detectEncoding(self, xml_data, isHTML=False):
        """Given a document, tries to detect its XML encoding."""
        xml_encoding = sniffed_xml_encoding = None
        try:
            if xml_data[:4] == '\x4c\x6f\xa7\x94':
                # EBCDIC
                xml_data = self._ebcdic_to_ascii(xml_data)
            elif xml_data[:4] == '\x00\x3c\x00\x3f':
                # UTF-16BE
                sniffed_xml_encoding = 'utf-16be'
                xml_data = unicode(xml_data, 'utf-16be').encode('utf-8')
            elif (len(xml_data) >= 4) and (xml_data[:2] == '\xfe\xff') \
                     and (xml_data[2:4] != '\x00\x00'):
                # UTF-16BE with BOM
                sniffed_xml_encoding = 'utf-16be'
                xml_data = unicode(xml_data[2:], 'utf-16be').encode('utf-8')
            elif xml_data[:4] == '\x3c\x00\x3f\x00':
                # UTF-16LE
                sniffed_xml_encoding = 'utf-16le'
                xml_data = unicode(xml_data, 'utf-16le').encode('utf-8')
            elif (len(xml_data) >= 4) and (xml_data[:2] == '\xff\xfe') and \
                     (xml_data[2:4] != '\x00\x00'):
                # UTF-16LE with BOM
                sniffed_xml_encoding = 'utf-16le'
                xml_data = unicode(xml_data[2:], 'utf-16le').encode('utf-8')
            elif xml_data[:4] == '\x00\x00\x00\x3c':
                # UTF-32BE
                sniffed_xml_encoding = 'utf-32be'
                xml_data = unicode(xml_data, 'utf-32be').encode('utf-8')
            elif xml_data[:4] == '\x3c\x00\x00\x00':
                # UTF-32LE
                sniffed_xml_encoding = 'utf-32le'
                xml_data = unicode(xml_data, 'utf-32le').encode('utf-8')
            elif xml_data[:4] == '\x00\x00\xfe\xff':
                # UTF-32BE with BOM
                sniffed_xml_encoding = 'utf-32be'
                xml_data = unicode(xml_data[4:], 'utf-32be').encode('utf-8')
            elif xml_data[:4] == '\xff\xfe\x00\x00':
                # UTF-32LE with BOM
                sniffed_xml_encoding = 'utf-32le'
                xml_data = unicode(xml_data[4:], 'utf-32le').encode('utf-8')
            elif xml_data[:3] == '\xef\xbb\xbf':
                # UTF-8 with BOM
                sniffed_xml_encoding = 'utf-8'
                xml_data = unicode(xml_data[3:], 'utf-8').encode('utf-8')
            else:
                sniffed_xml_encoding = 'ascii'
                pass
        except:
            xml_encoding_match = None
        xml_encoding_match = re.compile(
            '^<\?.*encoding=[\'"](.*?)[\'"].*\?>').match(xml_data)
        if not xml_encoding_match and isHTML:
            regexp = re.compile('<\s*meta[^>]+charset=([^>]*?)[;\'">]', re.I)
            xml_encoding_match = regexp.search(xml_data)
        if xml_encoding_match is not None:
            xml_encoding = xml_encoding_match.groups()[0].lower()
            if isHTML:
                self.declaredHTMLEncoding = xml_encoding
            if sniffed_xml_encoding and \
               (xml_encoding in ('iso-10646-ucs-2', 'ucs-2', 'csunicode',
                                 'iso-10646-ucs-4', 'ucs-4', 'csucs4',
                                 'utf-16', 'utf-32', 'utf_16', 'utf_32',
                                 'utf16', 'u16')):
                xml_encoding = sniffed_xml_encoding
        return xml_data, xml_encoding, sniffed_xml_encoding
def _detectEncoding(self, xml_data, isHTML=False):
        """Given a document, tries to detect its XML encoding."""
        xml_encoding = sniffed_xml_encoding = None
        try:
            if xml_data[:4] == '\x4c\x6f\xa7\x94':
                # EBCDIC
                xml_data = self._ebcdic_to_ascii(xml_data)
            elif xml_data[:4] == '\x00\x3c\x00\x3f':
                # UTF-16BE
                sniffed_xml_encoding = 'utf-16be'
                xml_data = unicode(xml_data, 'utf-16be').encode('utf-8')
            elif (len(xml_data) >= 4) and (xml_data[:2] == '\xfe\xff') \
                     and (xml_data[2:4] != '\x00\x00'):
                # UTF-16BE with BOM
                sniffed_xml_encoding = 'utf-16be'
                xml_data = unicode(xml_data[2:], 'utf-16be').encode('utf-8')
            elif xml_data[:4] == '\x3c\x00\x3f\x00':
                # UTF-16LE
                sniffed_xml_encoding = 'utf-16le'
                xml_data = unicode(xml_data, 'utf-16le').encode('utf-8')
            elif (len(xml_data) >= 4) and (xml_data[:2] == '\xff\xfe') and \
                     (xml_data[2:4] != '\x00\x00'):
                # UTF-16LE with BOM
                sniffed_xml_encoding = 'utf-16le'
                xml_data = unicode(xml_data[2:], 'utf-16le').encode('utf-8')
            elif xml_data[:4] == '\x00\x00\x00\x3c':
                # UTF-32BE
                sniffed_xml_encoding = 'utf-32be'
                xml_data = unicode(xml_data, 'utf-32be').encode('utf-8')
            elif xml_data[:4] == '\x3c\x00\x00\x00':
                # UTF-32LE
                sniffed_xml_encoding = 'utf-32le'
                xml_data = unicode(xml_data, 'utf-32le').encode('utf-8')
            elif xml_data[:4] == '\x00\x00\xfe\xff':
                # UTF-32BE with BOM
                sniffed_xml_encoding = 'utf-32be'
                xml_data = unicode(xml_data[4:], 'utf-32be').encode('utf-8')
            elif xml_data[:4] == '\xff\xfe\x00\x00':
                # UTF-32LE with BOM
                sniffed_xml_encoding = 'utf-32le'
                xml_data = unicode(xml_data[4:], 'utf-32le').encode('utf-8')
            elif xml_data[:3] == '\xef\xbb\xbf':
                # UTF-8 with BOM
                sniffed_xml_encoding = 'utf-8'
                xml_data = unicode(xml_data[3:], 'utf-8').encode('utf-8')
            else:
                sniffed_xml_encoding = 'ascii'
                pass
        except:
            xml_encoding_match = None
        xml_encoding_match = re.compile(
            '^<\?.*encoding=[\'"](.*?)[\'"].*\?>').match(xml_data)
        if not xml_encoding_match and isHTML:
            regexp = re.compile('<\s*meta[^>]+charset=([^>]*?)[;\'">]', re.I)
            xml_encoding_match = regexp.search(xml_data)
        if xml_encoding_match is not None:
            xml_encoding = xml_encoding_match.groups()[0].lower()
            if isHTML:
                self.declaredHTMLEncoding = xml_encoding
            if sniffed_xml_encoding and \
               (xml_encoding in ('iso-10646-ucs-2', 'ucs-2', 'csunicode',
                                 'iso-10646-ucs-4', 'ucs-4', 'csucs4',
                                 'utf-16', 'utf-32', 'utf_16', 'utf_32',
                                 'utf16', 'u16')):
                xml_encoding = sniffed_xml_encoding
        return xml_data, xml_encoding, sniffed_xml_encoding
def _detectEncoding ( self , xml_data , isHTML = False ) : xml_encoding = sniffed_xml_encoding = None try : if xml_data [ : 4 ] == '\x4c\x6f\xa7\x94' : # EBCDIC xml_data = self . _ebcdic_to_ascii ( xml_data ) elif xml_data [ : 4 ] == '\x00\x3c\x00\x3f' : # UTF-16BE sniffed_xml_encoding = 'utf-16be' xml_data = unicode ( xml_data , 'utf-16be' ) . encode ( 'utf-8' ) elif ( len ( xml_data ) >= 4 ) and ( xml_data [ : 2 ] == '\xfe\xff' ) and ( xml_data [ 2 : 4 ] != '\x00\x00' ) : # UTF-16BE with BOM sniffed_xml_encoding = 'utf-16be' xml_data = unicode ( xml_data [ 2 : ] , 'utf-16be' ) . encode ( 'utf-8' ) elif xml_data [ : 4 ] == '\x3c\x00\x3f\x00' : # UTF-16LE sniffed_xml_encoding = 'utf-16le' xml_data = unicode ( xml_data , 'utf-16le' ) . encode ( 'utf-8' ) elif ( len ( xml_data ) >= 4 ) and ( xml_data [ : 2 ] == '\xff\xfe' ) and ( xml_data [ 2 : 4 ] != '\x00\x00' ) : # UTF-16LE with BOM sniffed_xml_encoding = 'utf-16le' xml_data = unicode ( xml_data [ 2 : ] , 'utf-16le' ) . encode ( 'utf-8' ) elif xml_data [ : 4 ] == '\x00\x00\x00\x3c' : # UTF-32BE sniffed_xml_encoding = 'utf-32be' xml_data = unicode ( xml_data , 'utf-32be' ) . encode ( 'utf-8' ) elif xml_data [ : 4 ] == '\x3c\x00\x00\x00' : # UTF-32LE sniffed_xml_encoding = 'utf-32le' xml_data = unicode ( xml_data , 'utf-32le' ) . encode ( 'utf-8' ) elif xml_data [ : 4 ] == '\x00\x00\xfe\xff' : # UTF-32BE with BOM sniffed_xml_encoding = 'utf-32be' xml_data = unicode ( xml_data [ 4 : ] , 'utf-32be' ) . encode ( 'utf-8' ) elif xml_data [ : 4 ] == '\xff\xfe\x00\x00' : # UTF-32LE with BOM sniffed_xml_encoding = 'utf-32le' xml_data = unicode ( xml_data [ 4 : ] , 'utf-32le' ) . encode ( 'utf-8' ) elif xml_data [ : 3 ] == '\xef\xbb\xbf' : # UTF-8 with BOM sniffed_xml_encoding = 'utf-8' xml_data = unicode ( xml_data [ 3 : ] , 'utf-8' ) . encode ( 'utf-8' ) else : sniffed_xml_encoding = 'ascii' pass except : xml_encoding_match = None xml_encoding_match = re . compile ( '^<\?.*encoding=[\'"](.*?)[\'"].*\?>' ) . match ( xml_data ) if not xml_encoding_match and isHTML : regexp = re . compile ( '<\s*meta[^>]+charset=([^>]*?)[;\'">]' , re . I ) xml_encoding_match = regexp . search ( xml_data ) if xml_encoding_match is not None : xml_encoding = xml_encoding_match . groups ( ) [ 0 ] . lower ( ) if isHTML : self . declaredHTMLEncoding = xml_encoding if sniffed_xml_encoding and ( xml_encoding in ( 'iso-10646-ucs-2' , 'ucs-2' , 'csunicode' , 'iso-10646-ucs-4' , 'ucs-4' , 'csucs4' , 'utf-16' , 'utf-32' , 'utf_16' , 'utf_32' , 'utf16' , 'u16' ) ) : xml_encoding = sniffed_xml_encoding return xml_data , xml_encoding , sniffed_xml_encoding
Given a document, tries to detect its XML encoding.
Given a document tries to detect its XML encoding .