From 4addda1a2d0eaf5bf8c0edfbb535a651ec99279c Mon Sep 17 00:00:00 2001 From: Marco De Vincenzi Date: Tue, 18 Aug 2026 19:21:57 -0400 Subject: [PATCH] fix: specify utf-8 encoding for all text file I/O Signed-off-by: Marco De Vincenzi --- .../darnit_baseline/attestation/generator.py | 2 +- .../threat_model/dependencies.py | 2 +- .../src/darnit_baseline/tools.py | 2 +- .../src/darnit_testchecks/adapters/builtin.py | 8 +++--- packages/darnit/src/darnit/cli.py | 8 +++--- .../darnit/src/darnit/config/discovery.py | 4 +-- .../darnit/src/darnit/context/dot_project.py | 8 +++--- .../src/darnit/context/dot_project_org.py | 4 +-- packages/darnit/src/darnit/context/sieve.py | 14 +++++----- .../darnit/src/darnit/core/audit_cache.py | 4 +-- packages/darnit/src/darnit/core/handlers.py | 2 +- .../darnit/src/darnit/core/verification.py | 4 +-- .../darnit/harness/interactive_resolver.py | 2 +- .../darnit/src/darnit/remediation/executor.py | 2 +- .../darnit/src/darnit/remediation/helpers.py | 2 +- .../darnit/server/tools/test_repository.py | 6 ++--- scripts/create-example-test-repo.py | 26 +++++++++++-------- scripts/validate_sync.py | 6 ++--- tests/darnit/remediation/test_helpers.py | 2 +- 19 files changed, 56 insertions(+), 52 deletions(-) diff --git a/packages/darnit-baseline/src/darnit_baseline/attestation/generator.py b/packages/darnit-baseline/src/darnit_baseline/attestation/generator.py index f73a4deb..6fb06660 100644 --- a/packages/darnit-baseline/src/darnit_baseline/attestation/generator.py +++ b/packages/darnit-baseline/src/darnit_baseline/attestation/generator.py @@ -135,7 +135,7 @@ def generate_attestation_from_results( # Save the attestation try: os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True) - with open(output_path, 'w') as f: + with open(output_path, 'w', encoding="utf-8") as f: f.write(output) logger.info(f"Attestation saved to: {output_path}") except OSError as e: diff --git a/packages/darnit-baseline/src/darnit_baseline/threat_model/dependencies.py b/packages/darnit-baseline/src/darnit_baseline/threat_model/dependencies.py index 493af067..bc5350e1 100644 --- a/packages/darnit-baseline/src/darnit_baseline/threat_model/dependencies.py +++ b/packages/darnit-baseline/src/darnit_baseline/threat_model/dependencies.py @@ -189,7 +189,7 @@ def _parse_go_mod(repo_path: str, declared: set[str]) -> None: def _read_file(path: str) -> str: - with open(path, errors="ignore") as f: + with open(path, errors="ignore", encoding="utf-8") as f: return f.read() diff --git a/packages/darnit-baseline/src/darnit_baseline/tools.py b/packages/darnit-baseline/src/darnit_baseline/tools.py index f0fce210..e6029749 100644 --- a/packages/darnit-baseline/src/darnit_baseline/tools.py +++ b/packages/darnit-baseline/src/darnit_baseline/tools.py @@ -1089,7 +1089,7 @@ def generate_threat_model( target = repo_path / output_path target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(content) + target.write_text(content, encoding="utf-8") return ( f"Threat model written to {output_path} ({len(content)} bytes, " f"{len(emitted)} findings, {overflow.total} trimmed)" diff --git a/packages/darnit-testchecks/src/darnit_testchecks/adapters/builtin.py b/packages/darnit-testchecks/src/darnit_testchecks/adapters/builtin.py index 0a63fa8c..6d48f36c 100644 --- a/packages/darnit-testchecks/src/darnit_testchecks/adapters/builtin.py +++ b/packages/darnit-testchecks/src/darnit_testchecks/adapters/builtin.py @@ -98,7 +98,7 @@ def check_pattern_not_found( for file_path in repo.glob(file_pattern): if file_path.is_file(): try: - content = file_path.read_text(errors="ignore") + content = file_path.read_text(errors="ignore", encoding="utf-8") for pattern_name, regex in regex_patterns.items(): matches = re.findall(regex, content, re.MULTILINE) if matches: @@ -134,7 +134,7 @@ def check_pattern_found( for file_path in repo.glob(file_pattern): if file_path.is_file(): try: - content = file_path.read_text(errors="ignore") + content = file_path.read_text(errors="ignore", encoding="utf-8") for pattern_name, regex in regex_patterns.items(): if re.search(regex, content, re.MULTILINE): found[pattern_name] = True @@ -589,7 +589,7 @@ def _create_readme( source="testchecks", ) - readme_path.write_text(content) + readme_path.write_text(content, encoding="utf-8") return RemediationResult( control_id="TEST-DOC-01", success=True, @@ -641,7 +641,7 @@ def _create_gitignore(self, local_path: str, dry_run: bool) -> RemediationResult source="testchecks", ) - gitignore_path.write_text(content) + gitignore_path.write_text(content, encoding="utf-8") return RemediationResult( control_id="TEST-IGN-01", success=True, diff --git a/packages/darnit/src/darnit/cli.py b/packages/darnit/src/darnit/cli.py index d0bb3bee..af1cb9cc 100644 --- a/packages/darnit/src/darnit/cli.py +++ b/packages/darnit/src/darnit/cli.py @@ -429,7 +429,7 @@ def cmd_init(args: argparse.Namespace) -> int: # check = {{ adapter = "kusari" }} ''' - baseline_path.write_text(template) + baseline_path.write_text(template, encoding="utf-8") logger.info(f"βœ“ Created {baseline_path}") return 0 @@ -570,7 +570,7 @@ def cmd_install(args: argparse.Namespace) -> int: config = {} if settings_path.exists(): try: - config = json.loads(settings_path.read_text()) + config = json.loads(settings_path.read_text(encoding="utf-8")) except json.JSONDecodeError as e: logger.error(f"Invalid JSON in settings file: {settings_path}: {e}") return 1 @@ -594,7 +594,7 @@ def cmd_install(args: argparse.Namespace) -> int: try: json.dumps(config) # validate before write - settings_path.write_text(json.dumps(config, indent=2) + "\n") + settings_path.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8") except Exception as e: logger.error(f"Failed to write settings file: {e}") return 1 @@ -765,7 +765,7 @@ def cmd_harness(args: argparse.Namespace) -> int: ) return int(HarnessExitCode.SETUP_ERROR) try: - _tty_probe = open("/dev/tty", "r+", buffering=1) # noqa: SIM115 + _tty_probe = open("/dev/tty", "r+", buffering=1, encoding="utf-8") # noqa: SIM115 _tty_probe.close() except OSError as exc: _emit_exit_summary( diff --git a/packages/darnit/src/darnit/config/discovery.py b/packages/darnit/src/darnit/config/discovery.py index c7f32801..7420331c 100644 --- a/packages/darnit/src/darnit/config/discovery.py +++ b/packages/darnit/src/darnit/config/discovery.py @@ -141,7 +141,7 @@ def discover_project_name(local_path: str) -> str | None: try: pkg_path = os.path.join(local_path, "package.json") if os.path.exists(pkg_path): - with open(pkg_path) as f: + with open(pkg_path, encoding="utf-8") as f: data = json.load(f) if "name" in data: return data["name"] @@ -176,7 +176,7 @@ def discover_project_name(local_path: str) -> str | None: try: go_mod_path = os.path.join(local_path, "go.mod") if os.path.exists(go_mod_path): - with open(go_mod_path) as f: + with open(go_mod_path, encoding="utf-8") as f: first_line = f.readline().strip() if first_line.startswith("module "): module_path = first_line[7:].strip() diff --git a/packages/darnit/src/darnit/context/dot_project.py b/packages/darnit/src/darnit/context/dot_project.py index bf73fe3d..accf4473 100644 --- a/packages/darnit/src/darnit/context/dot_project.py +++ b/packages/darnit/src/darnit/context/dot_project.py @@ -364,7 +364,7 @@ def read(self) -> ProjectConfig: yaml = YAML() yaml.preserve_quotes = True - with open(self.project_yaml) as f: + with open(self.project_yaml, encoding="utf-8") as f: data = yaml.load(f) if data is None: @@ -405,7 +405,7 @@ def _read_maintainers_into(self, config: ProjectConfig) -> None: from ruamel.yaml import YAML yaml = YAML() - with open(self.maintainers_yaml) as f: + with open(self.maintainers_yaml, encoding="utf-8") as f: data = yaml.load(f) if data is None: @@ -905,7 +905,7 @@ def update(self, updates: dict[str, Any]) -> None: # Read existing content or create new if self.project_yaml.exists(): - with open(self.project_yaml) as f: + with open(self.project_yaml, encoding="utf-8") as f: data = yaml.load(f) if data is None: data = {} @@ -918,7 +918,7 @@ def update(self, updates: dict[str, Any]) -> None: self._deep_update(data, updates) # Write back - with open(self.project_yaml, "w") as f: + with open(self.project_yaml, "w", encoding="utf-8") as f: yaml.dump(data, f) logger.info("Updated .project/project.yaml") diff --git a/packages/darnit/src/darnit/context/dot_project_org.py b/packages/darnit/src/darnit/context/dot_project_org.py index a3d4feaf..fb55e3dc 100644 --- a/packages/darnit/src/darnit/context/dot_project_org.py +++ b/packages/darnit/src/darnit/context/dot_project_org.py @@ -168,7 +168,7 @@ def _fetch_org_project(self, owner: str) -> ProjectConfig | None: ) if project_content: - (project_dir / "project.yaml").write_text(project_content) + (project_dir / "project.yaml").write_text(project_content, encoding="utf-8") else: logger.debug("No project.yaml found in %s/.project", owner) return None @@ -182,7 +182,7 @@ def _fetch_org_project(self, owner: str) -> ProjectConfig | None: owner, ".project/maintainers.yaml" ) if maintainers_content: - (project_dir / "maintainers.yaml").write_text(maintainers_content) + (project_dir / "maintainers.yaml").write_text(maintainers_content, encoding="utf-8") # Parse using existing reader reader = DotProjectReader(tmpdir) diff --git a/packages/darnit/src/darnit/context/sieve.py b/packages/darnit/src/darnit/context/sieve.py index 30cd91cd..632d513b 100644 --- a/packages/darnit/src/darnit/context/sieve.py +++ b/packages/darnit/src/darnit/context/sieve.py @@ -232,7 +232,7 @@ def _detect_maintainers_deterministic(self, local_path: str) -> list[ContextSign filepath = path / filename if filepath.exists(): try: - content = filepath.read_text() + content = filepath.read_text(encoding="utf-8") maintainers = self._parse_maintainers_file(content) if maintainers: signals.append(ContextSignal( @@ -251,7 +251,7 @@ def _detect_maintainers_deterministic(self, local_path: str) -> list[ContextSign filepath = path / codeowners_path if filepath.exists(): try: - content = filepath.read_text() + content = filepath.read_text(encoding="utf-8") owners = self._parse_codeowners(content) if owners: signals.append(ContextSignal( @@ -282,7 +282,7 @@ def _detect_maintainers_heuristic(self, local_path: str) -> list[ContextSignal]: package_json = path / "package.json" if package_json.exists(): try: - data = json.loads(package_json.read_text()) + data = json.loads(package_json.read_text(encoding="utf-8")) authors = [] # Get author @@ -332,7 +332,7 @@ def _detect_maintainers_heuristic(self, local_path: str) -> list[ContextSignal]: if tomllib: try: - data = tomllib.loads(pyproject.read_text()) + data = tomllib.loads(pyproject.read_text(encoding="utf-8")) authors = [] # Get authors from [project] section @@ -437,7 +437,7 @@ def _detect_security_contact_deterministic(self, local_path: str) -> list[Contex filepath = path / filename if filepath.exists(): try: - content = filepath.read_text() + content = filepath.read_text(encoding="utf-8") contact = self._parse_security_contact(content) if contact: signals.append(ContextSignal( @@ -462,7 +462,7 @@ def _detect_security_contact_heuristic(self, local_path: str) -> list[ContextSig filepath = path / filename if filepath.exists(): try: - content = filepath.read_text() + content = filepath.read_text(encoding="utf-8") # Look for security section security_section = re.search( r"(?:^|\n)#+\s*Security[^\n]*\n([\s\S]*?)(?=\n#+|\Z)", @@ -498,7 +498,7 @@ def _detect_governance_deterministic(self, local_path: str) -> list[ContextSigna filepath = path / filename if filepath.exists(): try: - content = filepath.read_text() + content = filepath.read_text(encoding="utf-8") model = self._parse_governance_model(content) if model: signals.append(ContextSignal( diff --git a/packages/darnit/src/darnit/core/audit_cache.py b/packages/darnit/src/darnit/core/audit_cache.py index 041c3da6..08076a50 100644 --- a/packages/darnit/src/darnit/core/audit_cache.py +++ b/packages/darnit/src/darnit/core/audit_cache.py @@ -135,7 +135,7 @@ def write_audit_cache( # Atomic write: write to temp file in same directory, then rename. fd, tmp_path = tempfile.mkstemp(dir=str(cache_dir), suffix=".tmp", prefix="audit-cache-") try: - with os.fdopen(fd, "w") as f: + with os.fdopen(fd, "w", encoding="utf-8") as f: json.dump(envelope, f, indent=2) os.replace(tmp_path, str(cache_path)) logger.debug("Wrote audit cache to %s", cache_path) @@ -167,7 +167,7 @@ def read_audit_cache(local_path: str, ttl_seconds: int = 3600) -> dict[str, Any] # Parse JSON try: - with open(cache_path) as f: + with open(cache_path, encoding="utf-8") as f: data = json.load(f) except (json.JSONDecodeError, OSError) as exc: logger.debug("Corrupt or unreadable audit cache: %s", exc) diff --git a/packages/darnit/src/darnit/core/handlers.py b/packages/darnit/src/darnit/core/handlers.py index bc5c3791..8833ae61 100644 --- a/packages/darnit/src/darnit/core/handlers.py +++ b/packages/darnit/src/darnit/core/handlers.py @@ -101,7 +101,7 @@ class TemplateInfo: def load_content(self) -> str: """Load template content from file.""" if self.content is None: - self.content = self.path.read_text() + self.content = self.path.read_text(encoding="utf-8") return self.content diff --git a/packages/darnit/src/darnit/core/verification.py b/packages/darnit/src/darnit/core/verification.py index dd544f8e..8f46983c 100644 --- a/packages/darnit/src/darnit/core/verification.py +++ b/packages/darnit/src/darnit/core/verification.py @@ -223,7 +223,7 @@ def get(self, package_name: str, version: str) -> VerificationResult | None: return None try: - with open(cache_path) as f: + with open(cache_path, encoding="utf-8") as f: data = json.load(f) # Check expiration @@ -298,7 +298,7 @@ def set( "version": version, } - with open(cache_path, "w") as f: + with open(cache_path, "w", encoding="utf-8") as f: json.dump(data, f) logger.debug(f"Cached verification result for {package_name}:{version}") diff --git a/packages/darnit/src/darnit/harness/interactive_resolver.py b/packages/darnit/src/darnit/harness/interactive_resolver.py index fcd311ec..bb428cdd 100644 --- a/packages/darnit/src/darnit/harness/interactive_resolver.py +++ b/packages/darnit/src/darnit/harness/interactive_resolver.py @@ -71,7 +71,7 @@ def _ensure_streams(self) -> tuple[TextIO, TextIO]: from darnit.harness.driver import HarnessSetupError try: - self._tty = open("/dev/tty", "r+", buffering=1) # noqa: SIM115 + self._tty = open("/dev/tty", "r+", buffering=1, encoding="utf-8") # noqa: SIM115 except OSError as exc: raise HarnessSetupError( "interactive channel unavailable (/dev/tty not openable): " diff --git a/packages/darnit/src/darnit/remediation/executor.py b/packages/darnit/src/darnit/remediation/executor.py index 1d224a72..c2024cc8 100644 --- a/packages/darnit/src/darnit/remediation/executor.py +++ b/packages/darnit/src/darnit/remediation/executor.py @@ -338,7 +338,7 @@ def _get_template_content(self, template_name: str) -> str | None: ) from err try: - with open(resolved) as f: + with open(resolved, encoding="utf-8") as f: return f.read() except OSError as e: logger.warning(f"Failed to read template file {resolved}: {e}") diff --git a/packages/darnit/src/darnit/remediation/helpers.py b/packages/darnit/src/darnit/remediation/helpers.py index b62ef469..0d58a05f 100644 --- a/packages/darnit/src/darnit/remediation/helpers.py +++ b/packages/darnit/src/darnit/remediation/helpers.py @@ -50,7 +50,7 @@ def write_file_safe(path: str, content: str) -> tuple[bool, str]: Tuple of (success: bool, message: str) """ try: - with open(path, 'w') as f: + with open(path, 'w', encoding="utf-8") as f: f.write(content) return True, f"Successfully wrote {path}" except OSError as e: diff --git a/packages/darnit/src/darnit/server/tools/test_repository.py b/packages/darnit/src/darnit/server/tools/test_repository.py index 9cf32eb9..01b628e8 100644 --- a/packages/darnit/src/darnit/server/tools/test_repository.py +++ b/packages/darnit/src/darnit/server/tools/test_repository.py @@ -55,7 +55,7 @@ def create_test_repository_impl( } } """ - with open(os.path.join(repo_path, "package.json"), "w") as f: + with open(os.path.join(repo_path, "package.json"), "w", encoding="utf-8") as f: f.write(package_json) # Create src/index.js @@ -67,7 +67,7 @@ def create_test_repository_impl( console.log('Run an OpenSSF Baseline audit to see what is missing:'); console.log(chalk.cyan(' audit_openssf_baseline(local_path=".")')); """ - with open(os.path.join(repo_path, "src", "index.js"), "w") as f: + with open(os.path.join(repo_path, "src", "index.js"), "w", encoding="utf-8") as f: f.write(index_js) # Create minimal .gitignore (intentionally missing security exclusions) @@ -75,7 +75,7 @@ def create_test_repository_impl( # This is MISSING important security exclusions! node_modules/ """ - with open(os.path.join(repo_path, ".gitignore"), "w") as f: + with open(os.path.join(repo_path, ".gitignore"), "w", encoding="utf-8") as f: f.write(gitignore) # Initialize git diff --git a/scripts/create-example-test-repo.py b/scripts/create-example-test-repo.py index 55265961..478e7321 100755 --- a/scripts/create-example-test-repo.py +++ b/scripts/create-example-test-repo.py @@ -71,7 +71,7 @@ def create_repo( version = "0.0.1" requires-python = ">=3.10" """ - (repo_path / "pyproject.toml").write_text(pyproject) + (repo_path / "pyproject.toml").write_text(pyproject, encoding="utf-8") # -- src/main.py ---------------------------------------------------------- main_py = """\ @@ -84,7 +84,7 @@ def main(): if __name__ == "__main__": main() """ - (repo_path / "src" / "main.py").write_text(main_py) + (repo_path / "src" / "main.py").write_text(main_py, encoding="utf-8") # -- .mcp.json (points back to darnit workspace) ------------------------- mcp_config = { @@ -103,7 +103,7 @@ def main(): } } } - (repo_path / ".mcp.json").write_text(json.dumps(mcp_config, indent=2) + "\n") + (repo_path / ".mcp.json").write_text(json.dumps(mcp_config, indent=2) + "\n", encoding="utf-8") # -- CLAUDE.md (instructions for Claude Code) ----------------------------- claude_md = """\ @@ -133,7 +133,7 @@ def main(): | PH-QA-01 | CONTRIBUTING.md | | PH-CI-01 | .github/workflows/*.yml | """ - (repo_path / "CLAUDE.md").write_text(claude_md) + (repo_path / "CLAUDE.md").write_text(claude_md, encoding="utf-8") # -- Initialize git ------------------------------------------------------- try: @@ -209,42 +209,45 @@ def remediate_repo(repo_path_str: str) -> None: if not readme.exists(): readme.write_text( "# hygiene-test-repo\n\n" - "A test project for the Project Hygiene Standard demo.\n" + "A test project for the Project Hygiene Standard demo.\n", + encoding="utf-8", ) created.append("README.md") # PH-DOC-02: LICENSE license_file = repo_path / "LICENSE" if not license_file.exists(): - license_file.write_text("MIT License - test project for demo purposes.\n") + license_file.write_text("MIT License - test project for demo purposes.\n", encoding="utf-8") created.append("LICENSE") # PH-SEC-01: SECURITY.md security = repo_path / "SECURITY.md" if not security.exists(): security.write_text( - "# Security Policy\n\nTo report a vulnerability, open an issue.\n" + "# Security Policy\n\nTo report a vulnerability, open an issue.\n", + encoding="utf-8", ) created.append("SECURITY.md") # PH-CFG-01: .gitignore gitignore = repo_path / ".gitignore" if not gitignore.exists(): - gitignore.write_text("*.pyc\n__pycache__/\n") + gitignore.write_text("*.pyc\n__pycache__/\n", encoding="utf-8") created.append(".gitignore") # PH-CFG-02: .editorconfig editorconfig = repo_path / ".editorconfig" if not editorconfig.exists(): editorconfig.write_text( - "root = true\n\n[*]\nindent_style = space\nindent_size = 4\n" + "root = true\n\n[*]\nindent_style = space\nindent_size = 4\n", + encoding="utf-8", ) created.append(".editorconfig") # PH-QA-01: CONTRIBUTING.md contributing = repo_path / "CONTRIBUTING.md" if not contributing.exists(): - contributing.write_text("# Contributing\n\nOpen a pull request.\n") + contributing.write_text("# Contributing\n\nOpen a pull request.\n", encoding="utf-8") created.append("CONTRIBUTING.md") # PH-CI-01: CI config @@ -259,7 +262,8 @@ def remediate_repo(repo_path_str: str) -> None: " check:\n" " runs-on: ubuntu-latest\n" " steps:\n" - " - uses: actions/checkout@v4\n" + " - uses: actions/checkout@v4\n", + encoding="utf-8", ) created.append(".github/workflows/ci.yml") diff --git a/scripts/validate_sync.py b/scripts/validate_sync.py index 8a727a28..5b7cd3af 100644 --- a/scripts/validate_sync.py +++ b/scripts/validate_sync.py @@ -121,7 +121,7 @@ def validate_pass_types_sync() -> ValidationResult: message="Cannot validate handler names: spec not found", ) - spec_content = SPEC_PATH.read_text() + spec_content = SPEC_PATH.read_text(encoding="utf-8") # Expected built-in handler names from spec (registered handler names only) spec_handler_names = set() @@ -138,7 +138,7 @@ def validate_pass_types_sync() -> ValidationResult: details=str(handlers_file), ) - code_content = handlers_file.read_text() + code_content = handlers_file.read_text(encoding="utf-8") missing_in_code = [] for handler_name in spec_handler_names: @@ -173,7 +173,7 @@ def validate_sarif_reads_from_toml() -> ValidationResult: details=str(sarif_path), ) - content = sarif_path.read_text() + content = sarif_path.read_text(encoding="utf-8") # Check for TOML loading if "_load_framework_config" not in content: diff --git a/tests/darnit/remediation/test_helpers.py b/tests/darnit/remediation/test_helpers.py index d18b72e3..07d3a992 100644 --- a/tests/darnit/remediation/test_helpers.py +++ b/tests/darnit/remediation/test_helpers.py @@ -83,7 +83,7 @@ def test_writes_unicode(self, temp_dir: Path): content = "Hello, δΈ–η•Œ! 🌍" success, msg = write_file_safe(str(filepath), content) assert success is True - assert filepath.read_text() == content + assert filepath.read_text(encoding="utf-8") == content @pytest.mark.unit def test_invalid_path(self, temp_dir: Path):