文本和字节序列
1. 大纲：
2. 字符
str1 = 'café'
#café字符串有4个Unicode字符
print(len(str1))
#使用UTF-8把str编码成bytes对象
byte_str1 = str1.encode('utf-8')
#bytes字面量以b开头
print(byte_str1)
#字节序列有5个字节（在UTF-8中，é的码位编码成两个字节）
print(len(byte_str1))
#使用UTF-8把bytes对象解码成str对象
print(byte_str1.decode('utf-8'))
3. 字节
#bytes对象可以从str对象使用给定的编码构建
cafe = bytes('café', encoding='UTF-8')
print(cafe)
#各个元素是range(256)内的整数
print(cafe[0])
#这里注意，bytes对象的切片还是bytes对象
print(cafe[:1])
cafe_arr = bytearray(cafe)
#bytearray对象没有字面量句法，而是以bytearray()和字节序列字面量参数的形式显示
print(cafe_arr)
#各个元素是range(256)内的整数
print(cafe_arr[0])
#bytearray对象的切片还是bytearray对象
print(cafe_arr[-1:])
print(bytes.fromhex('31 4B CE A9'))
4. 处理UnicodeEncodeError，UnicodeDecodeError,SyntaxError
octets = b'Montr\xe9al'
#print(octets.decode('utf_8')
#编码时制定errors='relpace'，把无法编码的字符替换成"?"，官方上表示未知编码
print(octets.decode('utf_8', errors='replace'))
5. 检测文本的编码
6. 处理文本文件
import sys, locale
expressions = """
locale.getpreferredencoding()
type(my_file)
my_file.encoding
sys.stdout.isatty()
sys.stdout.encoding
sys.stdin.isatty()
sys.stdin.encoding
sys.stderr.isatty()
sys.stderr.encoding
sys.getdefaultencoding()
sys.getfilesystemencoding()
"""
print(expressions)
my_file = open('dummy', 'w')
for expression in expressions.split():
#eval() 函数用来执行一个字符串表达式，并返回表达式的值。
value = eval(expression)
print(expression.rjust(30), '->', repr(value))
7. Unicode规范化字符串
from unicodedata import normalize
s1 = 'café'
s2 = 'cafe\u0301'
print(len(s1), len(s2)) #4 5
print('s1 == s2: ',s1 == s2) #False
#规范化
s3 = normalize('NFC', s1)
s4 = normalize('NFC', s2)
print(len(s3), len(s4)) #4 4
print(s3 == s4) #True
from unicodedata import normalize
s1 = 'café'
s2 = 'cafe\u0301'
print(len(s1), len(s2)) #4 5
print('s1 == s2: ',s1 == s2) #False
#规范化
s3 = normalize('NFD', s1)
s4 = normalize('NFD', s2)
print(len(s3), len(s4)) #5 5
print(s3 == s4) #True
#比较规范化Unicode字符串
from unicodedata import normalize
def nfc_equal(str1, str2):
return normalize('NFC', str1) == normalize('NFC', str2)
def fold_equal(str1, str2):
return (normalize('NFC', str1).casefold() ==
normalize('NFC', str2).casefold())
8. 极端规范化：去掉变音符号
def shave_marks(txt):
"""Remove all diacritic marks"""
norm_txt = unicodedata.normalize('NFD', txt)
shaved = ''.join(c for c in norm_txt
if not unicodedata.combining(c))
return unicodedata.normalize('NFC', shaved)
9. 支持字符串和字节序列的双模式API
import os
print(os.listdir('.'))
print(os.listdir(b'.'))