diff --git a/CMakeLists.txt b/CMakeLists.txt index eefcaa9..7a10c80 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,6 +59,7 @@ add_executable(mithril src/keyweak.cpp src/keycorpus.cpp src/keys.cpp + src/configcred.cpp src/walk.cpp src/json.cpp src/human.cpp @@ -137,6 +138,7 @@ if(Python3_Interpreter_FOUND) # These take the freshly built binary as argv[1]. set(_mithril_py_tests test_secrets + test_config_cred test_der_keys test_keys test_boot diff --git a/src/configcred.cpp b/src/configcred.cpp new file mode 100644 index 0000000..b2e19eb --- /dev/null +++ b/src/configcred.cpp @@ -0,0 +1,283 @@ +#include "configcred.hpp" + +#include +#include +#include +#include +#include + +#include "jsonparse.hpp" +#include "validators.hpp" + +namespace ft { + +namespace { + +constexpr size_t kMaxFileSize = 1u << 20; // 1 MiB: real config files are tiny +constexpr size_t kMaxValueLen = 4096; // ignore absurdly long "values" +constexpr size_t kMaxFindings = 256; // cap a pathological config + +// Strong credential key names, normalized (lowercased, non-alnum stripped). +// Two sub-tiers differ only in how the value is judged: +// pin-like — device access codes / keys where a short value is legitimate +// (a 4-digit PIN, a short passphrase). Short + numeric accepted. +// token-like — API keys / tokens / secrets, where a real value is long or +// decodes; a bare short word or integer there is noise, rejected. +bool is_pin_like_key(std::string_view k) { + static const std::string_view keys[] = { + "passwd", "password", "passcode", "pass", "passphrase", "pin", "pincode", + "psk", "presharedkey", "sharedkey", "wpakey", "wepkey", "wifikey", "wlankey", + }; + return std::find(std::begin(keys), std::end(keys), k) != std::end(keys); +} +bool is_token_like_key(std::string_view k) { + static const std::string_view keys[] = { + "secret", "apikey", "token", "authtoken", "accesstoken", "refreshtoken", + "privatekey", "credential", "credentials", "clientsecret", "consumersecret", + "appsecret", "sessionkey", "authkey", + }; + return std::find(std::begin(keys), std::end(keys), k) != std::end(keys); +} + +std::string normalize_key(std::string_view k) { + std::string out; + out.reserve(k.size()); + for (unsigned char c : k) + if (std::isalnum(c)) out += static_cast(std::tolower(c)); + return out; +} + +std::string lower(std::string s) { + for (char& c : s) c = static_cast(std::tolower(static_cast(c))); + return s; +} + +bool all_same_char(const std::string& v) { + return !v.empty() && std::all_of(v.begin(), v.end(), [&](char c) { return c == v[0]; }); +} + +// Values that are clearly unset / defaults / enums, never a real credential. +bool is_placeholder_value(const std::string& v) { + if (v.empty()) return true; + if (all_same_char(v)) return true; // "0000", "xxxx", "----" + if (is_false_positive(v)) return true; // example/sample/changeme/your_/dummy/... + static const char* words[] = { + "admin", "administrator", "default", "password", "passwd", "pass", "none", + "null", "nil", "na", "n/a", "root", "guest", "user", "enable", "disable", + "enabled", "disabled", "true", "false", "yes", "no", "on", "off", "test", + "unknown", "undefined", "empty", "set", "unset", "value", + }; + std::string lv = lower(v); + for (const char* w : words) + if (lv == w) return true; + return false; +} + +bool whitespace_only(const std::string& v) { + return std::all_of(v.begin(), v.end(), + [](unsigned char c) { return std::isspace(c) != 0; }); +} + +// Shape tests for the decode bump. base64: >=4 chars, multiple of 4 with 0-2 '=' +// padding, body in the standard alphabet. hex: even length >=4, all hex digits. +bool looks_base64(const std::string& v) { + if (v.size() < 4 || v.size() % 4 != 0) return false; + size_t pad = 0; + while (pad < v.size() && v[v.size() - 1 - pad] == '=') ++pad; + if (pad > 2) return false; + for (size_t i = 0; i + pad < v.size(); ++i) { + char c = v[i]; + if (!(std::isalnum(static_cast(c)) || c == '+' || c == '/')) return false; + } + return true; +} +bool looks_hex(const std::string& v) { + if (v.size() < 4 || v.size() % 2 != 0) return false; + return std::all_of(v.begin(), v.end(), + [](unsigned char c) { return std::isxdigit(c) != 0; }); +} + +std::vector hex_decode(const std::string& v) { + auto nib = [](char c) -> int { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + return -1; + }; + std::vector out; + for (size_t i = 0; i + 1 < v.size(); i += 2) { + int hi = nib(v[i]), lo = nib(v[i + 1]); + if (hi < 0 || lo < 0) return {}; + out.push_back(static_cast((hi << 4) | lo)); + } + return out; +} + +bool mostly_printable(const std::vector& b) { + if (b.empty()) return false; + size_t printable = 0; + for (uint8_t c : b) + if (c == '\t' || (c >= 0x20 && c < 0x7F)) ++printable; + return printable * 100 >= b.size() * 80; +} + +std::string to_str(const std::vector& b) { + return std::string(reinterpret_cast(b.data()), b.size()); +} + +// Finding type by key family (keeps crypto-ish PSKs separate from app secrets). +std::string type_for(std::string_view nk, bool pin_like) { + if (nk == "psk" || nk == "presharedkey" || nk == "sharedkey" || nk == "wpakey" || + nk == "wepkey" || nk == "wifikey" || nk == "wlankey") + return "config-wifi-key"; + return pin_like ? "config-password" : "config-secret"; +} + +struct Ctx { + std::vector* out; + size_t count = 0; +}; + +// Judge one (key, scalar-value) pair and emit at most one finding. +void consider(Ctx& ctx, const std::string& keypath, std::string_view raw_key, + const std::string& value, bool numeric) { + if (ctx.count >= kMaxFindings) return; + if (value.empty() || value.size() > kMaxValueLen) return; + + std::string nk = normalize_key(raw_key); + bool pin_like = is_pin_like_key(nk); + bool token_like = is_token_like_key(nk); + if (!pin_like && !token_like) return; // strong-key gate (v1 is strong-only) + + if (whitespace_only(value) || is_placeholder_value(value)) return; + + // Try a transform. A strong key whose value cleanly base64/hex-decodes to + // printable text is strong evidence of an encoded credential (the encoded + // short-PIN case), and is the structural confidence bump. + std::string decoded; + const char* enc = nullptr; + if (!numeric && looks_base64(value)) { + auto d = base64_decode(std::span( + reinterpret_cast(value.data()), value.size())); + if (mostly_printable(d)) { decoded = to_str(d); enc = "base64"; } + } + if (!enc && !numeric && looks_hex(value)) { + auto d = hex_decode(value); + if (mostly_printable(d)) { decoded = to_str(d); enc = "hex"; } + } + // The decoded form must itself not be an obvious placeholder. + if (enc && (decoded.empty() || is_placeholder_value(decoded))) { enc = nullptr; decoded.clear(); } + + // token-like keys need a substantive value: long enough, or it decoded. + // A bare short word / integer under "token"/"secret"/"key" is rejected. + if (token_like && !enc) { + if (numeric) return; // "token": 1003 is a log/enum code, not a secret + if (value.size() < 8) return; // too short to be a real token without a decode + } + + // Partition with the flat-stream engine to avoid a double report: a long, + // no-decode raw value under a key the generic-secret rule ALSO anchors on + // (password/passwd/secret/api_key/access_token/auth_token/private_key) is + // that rule's job — it catches >=16 chars at >=3.5 bits. Keys the flat-stream + // rule has no anchor for (psk, pin, wpakey, token, ...) are only ever caught + // here, so they are never deferred regardless of length. + auto generic_engine_covers = [](std::string_view k) { + static const std::string_view a[] = {"password", "passwd", "secret", + "apikey", "accesstoken", "authtoken", + "privatekey"}; + return std::find(std::begin(a), std::end(a), k) != std::end(a); + }; + if (!enc && value.size() >= 16 && token_entropy(value) >= 3.5 && generic_engine_covers(nk)) + return; + + Finding f; + f.offset = 0; // file-scoped; the key path localizes it + f.size = value.size(); + f.type = type_for(nk, pin_like); + f.category = "config-credential"; + f.label = enc ? redact(value) + " (" + enc + ")" : redact(value); + if (enc) + f.description = "Credential field `" + keypath + "` (" + enc + "-encoded value)"; + else + f.description = "Credential field `" + keypath + "`"; + + if (enc) + f.set_confidence(Confidence::Structural, + std::string("config key + ") + enc + "-decoded value"); + else + f.set_confidence(Confidence::Pattern, "config key + non-placeholder value"); + ctx.out->push_back(std::move(f)); + ++ctx.count; +} + +// Format an integral JSON number back to text (for numeric PIN values). +std::optional integral_number(double d) { + if (d != d || d < 0 || d > 1e12) return std::nullopt; // NaN / huge / negative + if (d != static_cast(static_cast(d))) return std::nullopt; // has a fraction + char buf[32]; + std::snprintf(buf, sizeof(buf), "%lld", static_cast(d)); + return std::string(buf); +} + +void walk(Ctx& ctx, const JsonValue& v, const std::string& path, std::string_view inherited_key); + +// Dispatch a scalar/compound value that sits under `key` at `path`. +void dispatch(Ctx& ctx, const JsonValue& v, const std::string& path, std::string_view key) { + switch (v.type) { + case JsonValue::Type::String: + consider(ctx, path, key, v.str, /*numeric=*/false); + break; + case JsonValue::Type::Number: + if (auto s = integral_number(v.num)) consider(ctx, path, key, *s, /*numeric=*/true); + break; + case JsonValue::Type::Object: + case JsonValue::Type::Array: + walk(ctx, v, path, key); + break; + default: + break; // Null / Bool: never a credential + } +} + +void walk(Ctx& ctx, const JsonValue& v, const std::string& path, std::string_view inherited_key) { + if (ctx.count >= kMaxFindings) return; + if (v.type == JsonValue::Type::Object) { + for (const auto& m : v.obj) { + std::string child = path.empty() ? m.key : path + "." + m.key; + dispatch(ctx, m.value, child, m.key); // the member key is authoritative + } + } else if (v.type == JsonValue::Type::Array) { + // Scalar elements inherit the array's own key, so `"pins":["8604"]` + // is covered; object/array elements carry their own keys as usual. + for (const auto& e : v.arr) { + std::string child = path + "[]"; + dispatch(ctx, e, child, inherited_key); + } + } +} + +} // namespace + +std::vector scan_config_credentials(const std::string& relpath, + std::span data) { + (void)relpath; // v1 keys off content, not extension; relpath reserved for later tiers + std::vector out; + if (data.empty() || data.size() > kMaxFileSize) return out; + + // Cheap sniff: a JSON document starts with '{' or '[' after whitespace. + size_t i = 0; + while (i < data.size() && (data[i] == ' ' || data[i] == '\t' || data[i] == '\n' || + data[i] == '\r')) + ++i; + if (i >= data.size() || (data[i] != '{' && data[i] != '[')) return out; + + auto parsed = json_parse(std::string_view(reinterpret_cast(data.data()), + data.size())); + if (!parsed) return out; // not well-formed JSON -> not our file (strong gate) + + Ctx ctx{&out, 0}; + walk(ctx, *parsed, "", ""); + return out; +} + +} // namespace ft diff --git a/src/configcred.hpp b/src/configcred.hpp new file mode 100644 index 0000000..69f7f2b --- /dev/null +++ b/src/configcred.hpp @@ -0,0 +1,33 @@ +// configcred.hpp — structure-aware config-credential detector (JSON, v1). +// +// The flat-stream secret engine (rules_builtin.cpp `generic-secret`) is tuned +// for long, high-entropy opaque tokens: it floors the value at 16 chars and +// gates on >=3.5 bits/byte entropy, so a short low-entropy credential assigned +// to a credential-named field (a device PIN, a base64-wrapped passcode) is a +// deliberate false-negative there. Lowering those floors on a flat byte stream +// reintroduces a false-positive storm, because nothing structural separates a +// real `"passwd":"8604"` from `passwd=%s` / `oldpasswddir=...` / `"password":""`. +// +// This pass parses the file as structured config first, so the key name, the +// value, and the key/value relationship are exact. Confidence then comes from a +// strong credential key + a present, non-placeholder value (and a structural +// bump when the value cleanly base64/hex-decodes), never from entropy. The +// parse itself is a strong gate: binary, code, and malformed text never enter. +#pragma once + +#include +#include +#include +#include + +#include "finding.hpp" + +namespace ft { + +// Scan one file for config-credential fields. Returns findings with category +// "config-credential"; empty for anything that is not a small, cleanly-parsing +// structured-config document (v1: JSON). Values are redacted in the result. +std::vector scan_config_credentials(const std::string& relpath, + std::span data); + +} // namespace ft diff --git a/src/walk.cpp b/src/walk.cpp index 4a7f8ec..908e82d 100644 --- a/src/walk.cpp +++ b/src/walk.cpp @@ -13,6 +13,7 @@ #include "binver.hpp" #include "boot.hpp" +#include "configcred.hpp" #include "credstore.hpp" #include "derkey.hpp" #include "keys.hpp" @@ -136,6 +137,12 @@ FileHits scan_one(const fs::path& path, const fs::path& root, const Passes& pass auto derkeys = scan_der_private_keys(reader); fh.secrets.insert(fh.secrets.end(), std::make_move_iterator(derkeys.begin()), std::make_move_iterator(derkeys.end())); + // Structure-aware config credentials (JSON): short / encoded credential + // values in a parsed config that the flat-stream engine deliberately + // floors out. Category "config-credential". + auto cfg = scan_config_credentials(fh.path, fm.span()); + fh.secrets.insert(fh.secrets.end(), std::make_move_iterator(cfg.begin()), + std::make_move_iterator(cfg.end())); } if (passes.sbom) { fh.components = scan_sbom(fh.path, fm.span()); // M4 package DBs diff --git a/tests/test_config_cred.py b/tests/test_config_cred.py new file mode 100644 index 0000000..4eef3bf --- /dev/null +++ b/tests/test_config_cred.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""test_config_cred.py — integration test for the config-credential pass (JSON). + +Builds a synthetic tree of config files with known credential fields and known +bait, runs `mithril -j --secrets`, and asserts the config-credential pass flags +exactly the real credential fields (category "config-credential") and none of +the placeholders, enums, non-credential keys, or non-JSON files. Also asserts a +high-entropy value under a generic-anchor key is reported once (by the existing +flat-stream engine), not duplicated by this pass. + +Usage: tests/test_config_cred.py [path/to/mithril] (default: build/mithril) +""" +import base64 +import json +import os +import subprocess +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(HERE) + + +def b64(s): + return base64.b64encode(s.encode()).decode() + + +def build_tree(d): + os.makedirs(os.path.join(d, "etc")) + os.makedirs(os.path.join(d, "cfg")) + + # POSITIVE 1: short disarm-style codes as base64 passwd fields nested in an + # array of user objects. Short + low-entropy + encoded: the flat-stream + # engine floors all of these out. + with open(os.path.join(d, "user.json"), "w") as f: + json.dump({"users": [ + {"name": "svc", "passwd": b64("1379")}, + {"name": "op", "passwd": b64("5082")}, + {"name": "tech", "passwd": b64("9513")}, + ]}, f) + + # POSITIVE 2: a wifi PSK — key the flat-stream engine has no anchor for, so + # this pass is the only chance to catch it. Long value, Pattern tier (no decode). + with open(os.path.join(d, "cfg/wifi.json"), "w") as f: + json.dump({"wlan0": {"ssid": "HomeNet", "psk": "correcthorsebatterystaple7"}}, f) + + # POSITIVE 3: a short numeric PIN as a JSON number (not a string). + with open(os.path.join(d, "cfg/lock.json"), "w") as f: + json.dump({"door": {"pin": 2468}}, f) + + # POSITIVE 4: scalar elements in an array inherit the array's key. + with open(os.path.join(d, "cfg/keys.json"), "w") as f: + json.dump({"wpakey": [b64("Zone1Secret!")]}, f) + + # DEFER: a high-entropy value under a generic-anchor key ("password") must be + # caught once by the flat-stream generic-secret rule, NOT also by this pass. + with open(os.path.join(d, "cfg/app.json"), "w") as f: + json.dump({"db": {"password": "S3cr3t_Pa55w0rd_9xQ7zLmNqVt"}}, f) + + # NEGATIVES — every one of these must produce zero config-credential findings. + with open(os.path.join(d, "cfg/noise.json"), "w") as f: + json.dump({ + "password": "", # empty + "passwd": "changeme", # placeholder + "secret": "default", # placeholder enum + "apikey": "", # placeholder + "pin": "0000", # all-same-char + "token": 1003, # token-like + numeric -> log/enum code + "rebootDelay_sec": 30, # non-credential key + "DefaultLanguage": "English", # non-credential key + "enabled": True, # bool + "entry_time": [0, 30000, 30000], # array of non-cred numbers + }, f) + + # NEGATIVE: not valid JSON (a JS object literal) -> the parse gate rejects it + # before any key is examined, even though it starts with '{' and says passwd. + with open(os.path.join(d, "cfg/bundle.json"), "w") as f: + f.write("{ var passwd = function(){ return 'x'; }; }\n" + ("/*pad*/" * 2000)) + + # NEGATIVE: a plain non-config text file naming passwd (not JSON at all). + with open(os.path.join(d, "etc/readme.txt"), "w") as f: + f.write("Set your passwd to something strong. password = see the manual.\n") + + expected_cfgcred = { + ("user.json", "config-password"), + ("cfg/wifi.json", "config-wifi-key"), + ("cfg/lock.json", "config-password"), + ("cfg/keys.json", "config-wifi-key"), + } + return expected_cfgcred + + +def main(): + binary = sys.argv[1] if len(sys.argv) > 1 else os.path.join(ROOT, "build", "mithril") + if not os.path.exists(binary): + print(f"FAIL: binary not found: {binary} (build first)") + return 1 + + with tempfile.TemporaryDirectory() as d: + expected = build_tree(d) + out = subprocess.check_output([binary, "-j", "--secrets", d]) + rep = json.loads(out) + secrets = rep["secrets"] + fails = 0 + + cfg = [h for h in secrets if h.get("category") == "config-credential"] + got_paths = {(h["path"], h["type"]) for h in cfg} + + # 1. every expected real credential field is flagged, with the right path+type. + missing = expected - got_paths + if missing: + print(f"FAIL: expected config-credentials not found: {sorted(missing)}") + fails += 1 + + # 2. user.json yields exactly the three disarm codes, base64-decoded (structural). + upw = [h for h in cfg if h["path"] == "user.json"] + if len(upw) != 3: + print(f"FAIL: expected 3 passwd fields in user.json, got {len(upw)}: {upw}") + fails += 1 + for h in upw: + if h.get("confidence_tier") != "structural" or "base64" not in h.get("label", ""): + print(f"FAIL: user.json passwd not reported as base64/structural: {h}") + fails += 1 + + # 3. NO config-credential finding comes from any negative file. + bad = [h for h in cfg if h["path"] in + ("cfg/noise.json", "cfg/bundle.json", "etc/readme.txt", "cfg/app.json")] + if bad: + print(f"FAIL: config-credential false positives: {bad}") + fails += 1 + + # 4. the deferred high-entropy password is reported once, by the flat-stream + # engine (generic-secret), and not duplicated as a config-credential. + app_hits = [h for h in secrets if h["path"] == "cfg/app.json"] + app_cfg = [h for h in app_hits if h.get("category") == "config-credential"] + app_gen = [h for h in app_hits if h["type"] == "generic-secret"] + if app_cfg: + print(f"FAIL: app.json password double-reported as config-credential: {app_cfg}") + fails += 1 + if not app_gen: + print(f"FAIL: app.json password not caught by the generic-secret engine: {app_hits}") + fails += 1 + + # 5. values are redacted (the raw PIN/base64 is never emitted verbatim). + for h in cfg: + if "5082" in json.dumps(h) or "correcthorsebatterystaple7" in json.dumps(h): + print(f"FAIL: config-credential leaked a raw value: {h}") + fails += 1 + + if fails == 0: + print(f"PASS: {len(cfg)} config-credentials flagged across " + f"{rep['file_count']} files, placeholders/enums/non-JSON silent, " + f"high-entropy value not double-reported, values redacted") + return 0 + return 1 + + +if __name__ == "__main__": + sys.exit(main())