From 72471cc8c6c470c9f355e51585ea6ffe37f6b868 Mon Sep 17 00:00:00 2001 From: harbinresearcher Date: Wed, 7 Oct 2026 10:07:27 +0800 Subject: [PATCH] Accept normalized UTF-8 declarations with a BOM --- babel/util.py | 3 ++- tests/test_util.py | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/babel/util.py b/babel/util.py index a2bf728cc..ca53baba1 100644 --- a/babel/util.py +++ b/babel/util.py @@ -91,7 +91,8 @@ def parse_encoding(fp: IO[bytes]) -> str | None: if has_bom: if m: magic_comment_encoding = m.group(1).decode('latin-1') - if magic_comment_encoding != 'utf-8': + normalized_encoding = magic_comment_encoding.lower().replace('_', '-') + if normalized_encoding != 'utf-8' and not normalized_encoding.startswith('utf-8-'): raise SyntaxError(f"encoding problem: {magic_comment_encoding} with BOM") return 'utf-8' elif m: diff --git a/tests/test_util.py b/tests/test_util.py index f51626e25..07056e938 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -12,6 +12,7 @@ import __future__ +import codecs from io import BytesIO import pytest @@ -73,6 +74,26 @@ def test_parse_encoding_non_ascii(): assert parse_encoding('K\xf6ln') is None +@pytest.mark.parametrize('encoding', ['utf-8', 'UTF-8', 'utf_8', 'UTF_8', 'utf-8-sig']) +@pytest.mark.parametrize('prefix', [b'', b'# source file\n']) +def test_parse_encoding_bom(encoding, prefix): + source = codecs.BOM_UTF8 + prefix + f'# coding: {encoding}\n'.encode('ascii') + compile(source, '', 'exec') + stream = BytesIO(source) + stream.seek(3) + assert util.parse_encoding(stream) == 'utf-8' + assert stream.tell() == 3 + + +@pytest.mark.parametrize('encoding', ['latin-1', 'utf8', 'cp65001']) +def test_parse_encoding_bom_incompatible(encoding): + stream = BytesIO(codecs.BOM_UTF8 + f'# coding: {encoding}\n'.encode('ascii')) + stream.seek(3) + with pytest.raises(SyntaxError, match=f'encoding problem: {encoding} with BOM'): + util.parse_encoding(stream) + assert stream.tell() == 3 + + @pytest.mark.parametrize(('source', 'result'), [ (''' from __future__ import print_function,