文字コードを宣言しないページをエージェントに読ませる前に、UTF-8 のテキストにして渡す小さなスクリプト

UnicodeDecodeError: 'utf-8' codec can't decode byte 0xed in position 50: invalid continuation byte

b = "<p>“quoted”</p>".encode("cp1252")
print(repr(b.decode("latin-1")))   # '<p>\x93quoted\x94</p>'
print(repr(b.decode("cp1252")))    # '<p>“quoted”</p>'

#!/usr/bin/env python3
"""fetch_utf8.py — 文字コード不明のページを UTF-8 のテキストにして保存する。"""
import codecs, re, sys, urllib.request
from html.parser import HTMLParser
 
MAX_BYTES = 2_000_000
META = re.compile(rb'''<meta[^>]+charset\s*=\s*["']?\s*([A-Za-z0-9_\-:.]+)''', re.I)
HDR = re.compile(r'charset\s*=\s*["\']?([A-Za-z0-9_\-:.]+)', re.I)
 
def normalize(label):
    """ラベルを Python のコーデック名へ。WHATWG 流に latin-1 は cp1252 として扱う。"""
    try:
        name = codecs.lookup(label.strip()).name
    except LookupError:
        return None
    return "cp1252" if name in ("iso8859-1", "latin-1") else name
 
def detect(body, content_type):
    m = HDR.search(content_type or "")
    if m and normalize(m.group(1)):
        return normalize(m.group(1)), "http-header"
    if body.startswith(codecs.BOM_UTF8):
        return "utf-8-sig", "bom"
    m = META.search(body[:1024])
    if m and normalize(m.group(1).decode("ascii", "ignore")):
        return normalize(m.group(1).decode("ascii", "ignore")), "meta"
    try:
        body.decode("utf-8")
        return "utf-8", "utf8-valid"
    except UnicodeDecodeError:
        return "cp1252", "fallback"
 
class Text(HTMLParser):
    def __init__(self):
        super().__init__(); self.out = []; self.skip = 0
    def handle_starttag(self, tag, attrs):
        if tag in ("script", "style"): self.skip += 1
    def handle_endtag(self, tag):
        if tag in ("script", "style") and self.skip: self.skip -= 1
        if tag in ("p", "div", "li", "h1", "h2", "h3", "br", "tr", "title"): self.out.append("\n")
    def handle_data(self, d):
        if not self.skip: self.out.append(d)
 
def main(url, dest):
    req = urllib.request.Request(url, headers={"User-Agent": "fetch-utf8/1.0"})
    with urllib.request.urlopen(req, timeout=20) as r:
        body = r.read(MAX_BYTES + 1)
        ctype = r.headers.get("Content-Type", "")
    if len(body) > MAX_BYTES:
        sys.exit("too large: " + url)
    enc, why = detect(body, ctype)
    html = body.decode(enc, errors="replace")
    p = Text(); p.feed(html)
    text = re.sub(r"\n{3,}", "\n\n", "".join(p.out)).strip()
    with open(dest, "w", encoding="utf-8") as f:
        f.write(text + "\n")
    if why == "fallback":
        print("warning: 文字コードを判定できず cp1252 と仮定しました。結果を目で確認してください", file=sys.stderr)
    print(f"encoding={enc} via={why} replaced={html.count(chr(0xFFFD))} -> {dest}")
 
if __name__ == "__main__":
    if len(sys.argv) != 3: sys.exit("usage: fetch_utf8.py URL OUT.txt")
    main(*sys.argv[1:])

from http.server import BaseHTTPRequestHandler, HTTPServer
PAGES = {
 "/latin1": ("text/html", "<html><head><meta charset=iso-8859-1><title>Asesoría</title></head><body><p>Año nuevo, señor.</p></body></html>".encode("iso-8859-1")),
 "/sjis":   ("text/html", "<html><head><meta http-equiv='Content-Type' content='text/html; charset=Shift_JIS'></head><body><p>壁紙アプリの更新履歴です。</p></body></html>".encode("shift_jis")),
 "/utf8":   ("text/html", "<html><body><p>日本語 UTF-8 そのまま</p></body></html>".encode("utf-8")),
 "/hdr":    ("text/html; charset=euc-jp", "<html><body><p>ヘッダ優先の確認</p></body></html>".encode("euc-jp")),
 "/nometa": ("text/html", "<html><body><p>Año sin宣言</p></body></html>".encode("cp1252", "replace")),
}
class H(BaseHTTPRequestHandler):
    def do_GET(self):
        ct, b = PAGES[self.path]
        self.send_response(200); self.send_header("Content-Type", ct); self.send_header("Content-Length", str(len(b))); self.end_headers(); self.wfile.write(b)
    def log_message(self, *a): pass
HTTPServer(("127.0.0.1", 8765), H).serve_forever()

python3 serve.py &
for p in latin1 sjis utf8 hdr nometa; do
  python3 fetch_utf8.py http://127.0.0.1:8765/$p out_$p.txt
done

[latin1]  encoding=cp1252 via=meta        -> Asesoría / Año nuevo, señor.
[sjis]    encoding=shift_jis via=meta     -> 壁紙アプリの更新履歴です。
[utf8]    encoding=utf-8 via=utf8-valid   -> 日本語 UTF-8 そのまま
[hdr]     encoding=euc_jp via=http-header -> ヘッダ優先の確認
[nometa]  encoding=cp1252 via=fallback    -> 警告を表示