GitCorner

Code

ossartcraftcraftanything๐ŸŒ
642d5a256791
//! Historical network captures (HAR): requests, responses, timings, hosts and an API surface
//! grouped by path template, with privacy built in.
//!
//! What is never kept: cookie values, `Authorization`/`Proxy-Authorization`/API-key style header
//! values, URL user-info, credential-looking query values, and any body value โ€” JSON bodies are
//! reduced to their outline (keys and types). Secrets the caller names are redacted wherever
//! they appear in kept text, in raw, percent-encoded and JSON-escaped spellings; a sensitive value
//! used as a JSON property name drops that property.

use craft_anything_formats::har::{self, HarEntry};
use craft_anything_formats::sourcemap::base64_decode;
use serde_json::{Map, Value, json};
use std::collections::{BTreeMap, BTreeSet};

pub(crate) const CREDENTIAL_HEADERS: &[&str] = &[
    "authorization",
    "proxy-authorization",
    "cookie",
    "set-cookie",
    "x-api-key",
    "api-key",
    "x-auth-token",
    "x-access-token",
    "x-csrf-token",
    "x-xsrf-token",
    "x-amz-security-token",
    "x-goog-api-key",
    "x-session-id",
];

const CREDENTIAL_PARAMS: &[&str] = &[
    "token",
    "access_token",
    "id_token",
    "refresh_token",
    "auth",
    "authorization",
    "key",
    "api_key",
    "apikey",
    "secret",
    "client_secret",
    "password",
    "passwd",
    "pwd",
    "session",
    "sessionid",
    "sid",
    "sig",
    "signature",
    "code",
    "x-amz-signature",
    "x-amz-credential",
    "x-amz-security-token",
];

#[derive(Debug, Clone, Default)]
pub struct HarOptions {
    pub hidden_values: Vec<String>,
    pub offset: usize,
    pub limit: usize,
    /// Only these entries (0-based), in full detail.
    pub record_numbers: Option<Vec<usize>>,
    pub max_entries: usize,
}

pub(crate) struct Redactor {
    /// Every spelling of every declared value, longest first.
    pub(crate) needles: Vec<String>,
    log: Vec<Value>,
}

/// Percent-encode `s`; when `form` is true, use WHATWG `application/x-www-form-urlencoded` rules
/// (space becomes `+`, `*` stays literal, `~` is percent-encoded); otherwise keep `~` and encode `*`.
pub fn encode_query(s: &str, form: bool, upper_hex: bool) -> String {
    let mut out = String::with_capacity(s.len());
    for b in s.bytes() {
        if form && b == b' ' {
            out.push('+');
        } else if b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.') || (form && b == b'*') || (!form && b == b'~') {
            out.push(char::from(b));
        } else if upper_hex {
            out.push_str(&format!("%{b:02X}"));
        } else {
            out.push_str(&format!("%{b:02x}"));
        }
    }
    out
}

/// Percent-encode `s` using uppercase hexadecimal escapes (`%20` for spaces).
fn percent_encode(s: &str) -> String {
    encode_query(s, false, true)
}

/// True when `haystack` contains `needle` under ASCII case-insensitive comparison.
pub fn contains_ascii_ci(haystack: &str, needle: &str) -> bool {
    !needle.is_empty() && haystack.as_bytes().windows(needle.len()).any(|w| w.eq_ignore_ascii_case(needle.as_bytes()))
}

/// Replace ASCII case-insensitive occurrences of `needles` in `s` with `replacement`, merging
/// overlapping matched spans so an earlier shorter match cannot expose a longer secret's suffix.
pub fn replace_needles_with(s: &str, needles: &[String], replacement: &str) -> (String, u64) {
    if needles.is_empty() || !needles.iter().any(|n| contains_ascii_ci(s, n)) {
        return (s.to_string(), 0);
    }
    let mut spans: Vec<(usize, usize)> = Vec::new();
    for (start, _) in s.char_indices() {
        let Some(tail) = s.get(start..) else { continue };
        let max_len = needles
            .iter()
            .filter(|n| {
                !n.is_empty()
                    && tail.len() >= n.len()
                    && s.is_char_boundary(start + n.len())
                    && tail.as_bytes().get(..n.len()).is_some_and(|w| w.eq_ignore_ascii_case(n.as_bytes()))
            })
            .map(String::len)
            .max();
        if let Some(len) = max_len {
            let end = start + len;
            if let Some((_, prev_end)) = spans.last_mut()
                && start < *prev_end
            {
                *prev_end = (*prev_end).max(end);
            } else {
                spans.push((start, end));
            }
        }
    }
    let mut out = String::with_capacity(s.len());
    let mut count = 0u64;
    let mut cursor = 0usize;
    for (start, end) in spans {
        if let Some(prefix) = s.get(cursor..start) {
            out.push_str(prefix);
        }
        out.push_str(replacement);
        count = count.saturating_add(1);
        cursor = end;
    }
    if let Some(rest) = s.get(cursor..) {
        out.push_str(rest);
    }
    (out, count)
}

/// Replace ASCII case-insensitive occurrences of `needles` (merging overlapping matches) with `[redacted]`.
fn replace_needles(s: &str, needles: &[String]) -> String {
    replace_needles_with(s, needles, "[redacted]").0
}

impl Redactor {
    pub(crate) fn new(values: &[String]) -> Self {
        let mut needles: Vec<String> = Vec::new();
        for v in values.iter().filter(|v| !v.is_empty()) {
            let json_escaped = serde_json::to_string(v).ok().and_then(|q| q.get(1..q.len().saturating_sub(1)).map(String::from)).unwrap_or_default();
            let pct_up = percent_encode(v);
            let pct_lo = encode_query(v, false, false);
            let form_up = encode_query(v, true, true);
            let form_lo = encode_query(v, true, false);
            let quote_plus_up = pct_up.replace("%20", "+");
            let quote_plus_lo = pct_lo.replace("%20", "+");
            for form in [
                v.clone(),
                v.to_ascii_lowercase(),
                v.to_ascii_uppercase(),
                pct_up.clone(),
                pct_lo,
                pct_up.to_ascii_lowercase(),
                form_up.clone(),
                form_lo,
                form_up.to_ascii_lowercase(),
                quote_plus_up,
                quote_plus_lo,
                json_escaped,
                v.replace(' ', "+"),
            ] {
                if !form.is_empty() && !needles.contains(&form) {
                    needles.push(form);
                }
            }
        }
        needles.sort_by_key(|n| std::cmp::Reverse(n.len()));
        Redactor { needles, log: Vec::new() }
    }

    fn contains(&self, s: &str) -> bool {
        self.needles.iter().any(|n| contains_ascii_ci(s, n))
    }

    /// Replace declared values in kept text.
    fn text(&mut self, pointer: &str, s: &str) -> String {
        if !self.contains(s) {
            return s.to_string();
        }
        let out = replace_needles(s, &self.needles);
        self.log.push(json!({"pointer": pointer, "reason": "secret-value"}));
        out
    }
}

/// Split a URL, dropping user-info. Returns (scheme, host, path, query pairs, had_userinfo).
pub(crate) fn split_url(url: &str) -> (String, String, String, Vec<(String, String)>, bool) {
    let (scheme, rest) = url.split_once("://").unwrap_or(("", url));
    let rest = rest.split('#').next().unwrap_or(rest);
    let (authority, path_q) = match rest.find(['/', '?', '#']) {
        Some(i) => (rest.get(..i).unwrap_or(""), rest.get(i..).unwrap_or("/")),
        None => (rest, "/"),
    };
    let (userinfo, host) = match authority.rsplit_once('@') {
        Some((_, h)) => (true, h),
        None => (false, authority),
    };
    let (path, query) = path_q.split_once('?').unwrap_or((path_q, ""));
    let pairs = query
        .split('&')
        .filter(|p| !p.is_empty())
        .take(200)
        .map(|p| {
            let (k, v) = p.split_once('=').unwrap_or((p, ""));
            (k.to_string(), v.to_string())
        })
        .collect();
    (scheme.to_ascii_lowercase(), host.to_ascii_lowercase(), if path.is_empty() { "/".into() } else { path.to_string() }, pairs, userinfo)
}

/// Decode a query name once for privacy checks; malformed escapes stay literal.
pub(crate) fn decode_query_name(name: &str) -> String {
    let mut decoded = Vec::with_capacity(name.len());
    let mut bytes = name.as_bytes().iter().copied().peekable();
    while let Some(b) = bytes.next() {
        if b == b'%' {
            let mut next = bytes.clone();
            let hex = next.next().and_then(|hi| next.next().map(|lo| [hi, lo]));
            if let Some(value) =
                hex.filter(|h| h.iter().all(u8::is_ascii_hexdigit)).and_then(|h| std::str::from_utf8(&h).ok().and_then(|s| u8::from_str_radix(s, 16).ok()))
            {
                decoded.push(value);
                bytes = next;
                continue;
            }
        }
        decoded.push(if b == b'+' { b' ' } else { b });
    }
    String::from_utf8_lossy(&decoded).into_owned()
}

pub(crate) fn is_credential_param(k: &str) -> bool {
    let l = decode_query_name(k).to_ascii_lowercase();
    CREDENTIAL_PARAMS.contains(&l.as_str()) || l.ends_with("_token") || l.ends_with("token") || l.contains("password") || l.contains("secret")
}

/// Rebuild a safe URL: no user-info, credential query values replaced.
fn safe_url(url: &str, pointer: &str, red: &mut Redactor) -> (String, Vec<String>) {
    let (scheme, host, path, pairs, userinfo) = split_url(url);
    if userinfo {
        red.log.push(json!({"pointer": pointer, "reason": "url-userinfo"}));
    }
    let mut q = Vec::new();
    let mut names = Vec::new();
    for (k, v) in pairs {
        if red.contains(&k) || red.contains(&decode_query_name(&k)) {
            red.log.push(json!({"pointer": pointer, "reason": "secret-value", "scope": "query-name"}));
            continue;
        }
        names.push(k.clone());
        if is_credential_param(&k) && !v.is_empty() {
            red.log.push(json!({"pointer": pointer, "reason": "credential-parameter", "parameter": k}));
            q.push(format!("{k}=[redacted]"));
        } else {
            q.push(if v.is_empty() { k } else { format!("{k}={v}") });
        }
    }
    let base = if scheme.is_empty() { format!("{host}{path}") } else { format!("{scheme}://{host}{path}") };
    let full = if q.is_empty() { base } else { format!("{base}?{}", q.join("&")) };
    (red.text(pointer, &full), names)
}

/// Path template: numeric โ†’ {n}, uuid โ†’ {uuid}, long hex โ†’ {hex}, ISO date โ†’ {date}.
pub fn path_template(path: &str) -> String {
    path.split('/')
        .map(|seg| {
            let s = seg;
            if !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()) {
                "{n}".to_string()
            } else if is_uuid(s) {
                "{uuid}".to_string()
            } else if s.len() >= 32 && s.bytes().all(|b| b.is_ascii_hexdigit()) {
                "{hex}".to_string()
            } else if is_iso_date(s) {
                "{date}".to_string()
            } else {
                s.to_string()
            }
        })
        .collect::<Vec<_>>()
        .join("/")
}

pub(crate) fn is_uuid(s: &str) -> bool {
    let parts: Vec<&str> = s.split('-').collect();
    parts.len() == 5 && parts.iter().zip([8, 4, 4, 4, 12]).all(|(p, n)| p.len() == n && p.bytes().all(|b| b.is_ascii_hexdigit()))
}

fn is_iso_date(s: &str) -> bool {
    let b = s.as_bytes();
    if b.len() != 10 || b.get(4) != Some(&b'-') || b.get(7) != Some(&b'-') {
        return false;
    }
    let number = |start, end| s.get(start..end).filter(|v| v.bytes().all(|b| b.is_ascii_digit())).and_then(|v| v.parse::<u32>().ok());
    let (Some(year), Some(month), Some(day)) = (number(0, 4), number(5, 7), number(8, 10)) else { return false };
    let days = match month {
        1 | 3 | 5 | 7 | 8 | 10 | 12 => 31,
        4 | 6 | 9 | 11 => 30,
        2 => 28 + u32::from(year.is_multiple_of(4) && (!year.is_multiple_of(100) || year.is_multiple_of(400))),
        _ => return false,
    };
    (1..=days).contains(&day)
}

/// The outline of a JSON value: types only, never values.
pub fn json_outline(v: &Value, depth: usize, red: &mut OutlineRedactions<'_>, pointer: &str) -> Value {
    if depth > 8 {
        return json!("โ€ฆ");
    }
    match v {
        Value::Null => json!("null"),
        Value::Bool(_) => json!("boolean"),
        Value::Number(n) => json!(if n.is_i64() || n.is_u64() { "integer" } else { "number" }),
        Value::String(_) => json!("string"),
        Value::Array(a) => {
            let mut items: Option<Value> = None;
            for (i, x) in a.iter().take(20).enumerate() {
                let s = json_outline(x, depth + 1, red, &format!("{pointer}/{i}"));
                items = Some(match items {
                    None => s,
                    Some(prev) => merge_outlines(prev, s),
                });
            }
            json!({"type": "array", "items": items.unwrap_or(json!("unknown")), "length": a.len()})
        }
        Value::Object(o) => {
            let mut props = Map::new();
            for (k, x) in o.iter().take(200) {
                if red.names_hit(k) {
                    red.log.push(json!({"pointer": pointer, "reason": "secret-value", "scope": "property-name"}));
                    continue;
                }
                let child = format!("{pointer}/{}", k.replace('~', "~0").replace('/', "~1"));
                props.insert(k.clone(), json_outline(x, depth + 1, red, &child));
            }
            json!({"type": "object", "properties": props})
        }
    }
}

/// Redaction context for outlines (only property names can leak).
pub struct OutlineRedactions<'r> {
    needles: &'r [String],
    pub log: Vec<Value>,
}

impl OutlineRedactions<'_> {
    fn names_hit(&self, k: &str) -> bool {
        self.needles.iter().any(|n| contains_ascii_ci(k, n))
    }
}

fn merge_outlines(a: Value, b: Value) -> Value {
    if a == b {
        return a;
    }
    match (a, b) {
        (Value::Object(mut x), Value::Object(y)) if x.get("type") == Some(&json!("object")) && y.get("type") == Some(&json!("object")) => {
            let mut props = x.remove("properties").and_then(|p| if let Value::Object(m) = p { Some(m) } else { None }).unwrap_or_default();
            if let Some(Value::Object(yp)) = y.get("properties") {
                for (k, v) in yp {
                    let merged = match props.remove(k) {
                        Some(old) => merge_outlines(old, v.clone()),
                        None => v.clone(),
                    };
                    props.insert(k.clone(), merged);
                }
            }
            json!({"type": "object", "properties": props})
        }
        (Value::Object(x), other) | (other, Value::Object(x)) if x.contains_key("one_of") => {
            let mut list = x.get("one_of").and_then(Value::as_array).cloned().unwrap_or_default();
            if !list.contains(&other) && list.len() < 6 {
                list.push(other);
            }
            json!({"one_of": list})
        }
        (a, b) => json!({"one_of": [a, b]}),
    }
}

fn body_outline(
    mime: Option<&str>,
    text: Option<&str>,
    encoding: Option<&str>,
    params: &[(String, Option<String>)],
    red: &mut Redactor,
    pointer: &str,
) -> Value {
    let mime_l = mime.unwrap_or("").to_ascii_lowercase();
    let decoded_bytes = match (text, encoding) {
        (Some(t), Some("base64")) => base64_decode(t),
        (Some(t), _) => Some(t.as_bytes().to_vec()),
        _ => None,
    };
    let bytes = decoded_bytes.as_ref().map_or(0, Vec::len);
    let decoded = decoded_bytes.as_deref().map(|b| String::from_utf8_lossy(b).into_owned());
    let json_like = mime_l.contains("json") || decoded.as_deref().is_some_and(|t| t.trim_start().starts_with('{') || t.trim_start().starts_with('['));
    if json_like && let Some(v) = decoded.as_deref().and_then(|t| serde_json::from_str::<Value>(t).ok()) {
        let mut sr = OutlineRedactions { needles: &red.needles, log: Vec::new() };
        let outline = json_outline(&v, 0, &mut sr, pointer);
        red.log.extend(sr.log);
        return json!({"kind": "json", "bytes": bytes, "outline": outline});
    }
    let filter_form_fields = |names: Vec<String>, red: &mut Redactor| -> Vec<String> {
        let mut kept = Vec::with_capacity(names.len());
        for n in names {
            if red.contains(&n) || red.contains(&decode_query_name(&n)) {
                red.log.push(json!({"pointer": pointer, "reason": "secret-value", "scope": "form-field"}));
            } else {
                kept.push(n);
            }
        }
        kept
    };
    if mime_l.contains("x-www-form-urlencoded") {
        let names: Vec<String> = if params.is_empty() {
            decoded.as_deref().unwrap_or("").split('&').filter_map(|p| p.split('=').next()).filter(|k| !k.is_empty()).map(String::from).take(200).collect()
        } else {
            params.iter().map(|(k, _)| k.clone()).take(200).collect()
        };
        return json!({"kind": "form", "bytes": bytes, "fields": filter_form_fields(names, red)});
    }
    if mime_l.contains("multipart") {
        let names: Vec<String> = params.iter().map(|(k, _)| k.clone()).take(200).collect();
        return json!({"kind": "multipart", "bytes": bytes, "fields": filter_form_fields(names, red)});
    }
    if decoded.is_none() {
        return Value::Null;
    }
    json!({"kind": if mime_l.is_empty() { "unknown".to_string() } else { red.text(&format!("{pointer}/mime"), &mime_l) }, "bytes": bytes})
}

fn headers_view(h: &[(String, String)], pointer: &str, red: &mut Redactor) -> Vec<Value> {
    h.iter()
        .take(200)
        .enumerate()
        .filter_map(|(i, (n, v))| {
            if red.contains(n) {
                red.log.push(json!({"pointer": pointer, "reason": "secret-value", "scope": "header-name"}));
                return None;
            }
            let l = n.to_ascii_lowercase();
            let p = format!("{pointer}/{i}/value");
            if CREDENTIAL_HEADERS.contains(&l.as_str()) {
                red.log.push(json!({"pointer": p, "reason": "auth-header"}));
                Some(json!({"name": n, "value": null, "redacted": "auth-header"}))
            } else if matches!(l.as_str(), "referer" | "origin" | "location") {
                let (u, _) = safe_url(v, &p, red);
                Some(json!({"name": n, "value": u}))
            } else {
                Some(json!({"name": n, "value": red.text(&p, v)}))
            }
        })
        .collect()
}

fn entry_view(e: &HarEntry, red: &mut Redactor) -> Value {
    let base = format!("/log/entries/{}", e.ordinal);
    let (url, query_names) = safe_url(&e.request.url, &format!("{base}/request/url"), red);
    let (_, host, path, _, _) = split_url(&e.request.url);
    let req_body =
        e.request.post.as_ref().map(|p| body_outline(p.mime.as_deref(), p.text.as_deref(), None, &p.params, red, &format!("{base}/request/postData")));
    let res_body =
        body_outline(e.response.mime.as_deref(), e.response.text.as_deref(), e.response.encoding.as_deref(), &[], red, &format!("{base}/response/content"));
    let redirect = e.response.redirect_url.as_deref().map(|u| safe_url(u, &format!("{base}/response/redirectURL"), red).0);
    if !e.request.cookie_names.is_empty() || !e.response.cookie_names.is_empty() {
        red.log.push(json!({"pointer": format!("{base}/cookies"), "reason": "auth-header"}));
    }
    json!({
        "ordinal": e.ordinal,
        "started": e.started.as_deref().map(|value| red.text(&format!("{base}/started"), value)),
        "time_ms": e.time_ms,
        "resource_type": e.resource_type.as_deref().map(|value| red.text(&format!("{base}/resource_type"), value)),
        "method": red.text(&format!("{base}/request/method"), &e.request.method),
        "url": url,
        "host": red.text(&format!("{base}/request/host"), &host),
        "path_template": red.text(&format!("{base}/request/path_template"), &path_template(&path)),
        "query_parameters": query_names,
        "http_version": e.request.http_version.as_deref().map(|value| red.text(&format!("{base}/http_version"), value)),
        "status": e.response.status,
        "status_text": e.response.status_text.as_deref().map(|value| red.text(&format!("{base}/status_text"), value)),
        "request_headers": headers_view(&e.request.headers, &format!("{base}/request/headers"), red),
        "response_headers": headers_view(&e.response.headers, &format!("{base}/response/headers"), red),
        "request_cookie_names": e.request.cookie_names.iter().filter(|n| !red.contains(n)).collect::<Vec<_>>(),
        "response_cookie_names": e.response.cookie_names.iter().filter(|n| !red.contains(n)).collect::<Vec<_>>(),
        "request_body": req_body,
        "response_body": res_body,
        "response_mime": e.response.mime.as_deref().map(|value| red.text(&format!("{base}/response_mime"), value)),
        "response_bytes": e.response.content_size,
        "redirect_url": redirect,
        "timings": e.timings.iter().filter(|(k, _)| !red.contains(k)).map(|(k, v)| (k.clone(), json!(v))).collect::<Map<String, Value>>(),
    })
}

/// HAR uses -1 for an unknown size. A known zero (for example on a 304) still counts.
fn response_sizes(entries: &[HarEntry], field: impl Fn(&har::HarResponse) -> Option<i64>) -> (u64, usize) {
    let mut bytes = 0u64;
    let mut missing = 0;
    for entry in entries {
        match field(&entry.response).and_then(|n| u64::try_from(n).ok()) {
            Some(n) => bytes = bytes.saturating_add(n),
            None => missing += 1,
        }
    }
    (bytes, missing)
}

/// Inspect a HAR capture.
pub fn inspect_har(bytes: &[u8], opts: &HarOptions) -> Result<Value, String> {
    let har = har::parse(bytes, opts.max_entries.max(1)).map_err(|e| e.to_string())?;
    let mut red = Redactor::new(&opts.hidden_values);
    let mut methods: BTreeMap<String, usize> = BTreeMap::new();
    let mut statuses: BTreeMap<String, usize> = BTreeMap::new();
    let mut hosts: BTreeMap<String, usize> = BTreeMap::new();
    let mut mimes: BTreeMap<String, usize> = BTreeMap::new();
    let (body_bytes, missing_body) = response_sizes(&har.entries, |r| r.body_size);
    let (content_bytes, missing_content) = response_sizes(&har.entries, |r| r.content_size);
    let mut first: Option<String> = None;
    let mut last: Option<String> = None;
    let mut slowest: Vec<(f64, usize)> = Vec::new();
    let mut surface: BTreeMap<String, Value> = BTreeMap::new();
    let mut views = Vec::new();
    for e in &har.entries {
        let view = entry_view(e, &mut red);
        *methods.entry(view["method"].as_str().unwrap_or_default().to_ascii_uppercase()).or_default() += 1;
        *statuses.entry(format!("{}xx", e.response.status / 100)).or_default() += 1;
        let host = view["host"].as_str().unwrap_or("").to_string();
        *hosts.entry(red.text("/hosts", &host)).or_default() += 1;
        let mime = view["response_mime"].as_str().unwrap_or("").split(';').next().unwrap_or("").trim().to_string();
        *mimes.entry(mime.clone()).or_default() += 1;
        if let Some(s) = &e.started {
            if first.as_ref().is_none_or(|f| s < f) {
                first = Some(s.clone());
            }
            if last.as_ref().is_none_or(|l| s > l) {
                last = Some(s.clone());
            }
        }
        if let Some(t) = e.time_ms {
            slowest.push((t, e.ordinal));
        }
        // API surface.
        let (scheme, _, _, _, _) = split_url(&e.request.url);
        let key = format!(
            "{} {}://{}{}",
            view["method"].as_str().unwrap_or_default().to_ascii_uppercase(),
            red.text("/scheme", &scheme),
            host,
            view["path_template"].as_str().unwrap_or("")
        );
        let key = red.text("/api_surface", &key);
        let g = surface.entry(key.clone()).or_insert_with(|| json!({"operation": key, "count": 0, "statuses": [], "query_parameters": [], "request_outline": null, "response_outline": null, "response_mimes": []}));
        if let Some(o) = g.as_object_mut() {
            let c = o.get("count").and_then(Value::as_u64).unwrap_or(0) + 1;
            o.insert("count".into(), json!(c));
            add_unique(o, "statuses", json!(e.response.status));
            for q in view["query_parameters"].as_array().into_iter().flatten() {
                add_unique(o, "query_parameters", q.clone());
            }
            add_unique(o, "response_mimes", json!(mime));
            for (field, src) in [("request_outline", &view["request_body"]), ("response_outline", &view["response_body"])] {
                if let Some(outline) = src.get("outline") {
                    let merged = match o.get(field).cloned() {
                        Some(Value::Null) | None => outline.clone(),
                        Some(prev) => merge_outlines(prev, outline.clone()),
                    };
                    o.insert(field.into(), merged);
                }
            }
        }
        views.push(view);
    }
    slowest.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
    let slowest: Vec<Value> = slowest
        .iter()
        .take(5)
        .filter_map(|(t, i)| views.get(*i).map(|v| json!({"ordinal": i, "time_ms": t, "method": v["method"], "url": v["url"]})))
        .collect();
    let selected: Vec<Value> = match &opts.record_numbers {
        Some(ords) => views.iter().filter(|v| v["ordinal"].as_u64().is_some_and(|o| ords.contains(&(o as usize)))).cloned().collect(),
        None => views.iter().skip(opts.offset).take(opts.limit.max(1)).cloned().collect(),
    };
    let ops: Vec<Value> = surface.into_values().collect();
    let version = har.version.as_deref().map(|v| red.text("/har_version", v));
    let creator = har.creator.as_deref().map(|v| red.text("/creator", v));
    let first = first.as_deref().map(|v| red.text("/summary/first_request", v));
    let last = last.as_deref().map(|v| red.text("/summary/last_request", v));
    let mut redactions = red.log;
    redactions.truncate(5000);
    Ok(json!({
        "format": "har",
        "har_version": version,
        "creator": creator,
        "pages": har.pages,
        "summary": {
            "entries": har.entries.len(),
            "declared_entries": har.declared_entries,
            "methods": methods, "statuses": statuses, "hosts": hosts, "response_mimes": mimes,
            "response_body_bytes": body_bytes, "responses_without_body_size": missing_body,
            "response_content_bytes": content_bytes, "responses_without_content_size": missing_content,
            "first_request": first, "last_request": last, "slowest": slowest,
        },
        "api_surface": ops,
        "entries": selected,
        "entries_total": views.len(),
        "redactions": redactions,
        "sensitive_value_declarations": opts.hidden_values.len(),
        "truncated": har.truncated,
    }))
}

fn add_unique(o: &mut Map<String, Value>, key: &str, v: Value) {
    if let Some(Value::Array(a)) = o.get_mut(key)
        && !a.contains(&v)
        && a.len() < 64
    {
        a.push(v);
    }
}

/// Every set of distinct values seen for a key in a JSON tree โ€” used by tests to prove nothing leaks.
pub fn all_strings(v: &Value, out: &mut BTreeSet<String>) {
    match v {
        Value::String(s) => {
            out.insert(s.clone());
        }
        Value::Array(a) => a.iter().for_each(|x| all_strings(x, out)),
        Value::Object(o) => o.iter().for_each(|(k, x)| {
            out.insert(k.clone());
            all_strings(x, out);
        }),
        _ => {}
    }
}

/// A HAR 1.2 recording of browser network entries, as the browser tools return them (method, url,
/// status, MIME type and any response body they kept). Nothing is invented: a header reduced to its
/// length, a body that was left out and a body that came back as bytes are all left out here too.
/// The result is what `web_standin_build` reads, so a rebuilt front end can be given recorded
/// answers.
pub fn har_from_network(requests: &[Value], started: &str) -> Value {
    let entries: Vec<Value> = requests.iter().map(|r| har_entry(r, started)).collect();
    json!({
        "log": {
            "version": "1.2",
            "creator": {"name": "Craft Anything", "version": env!("CARGO_PKG_VERSION")},
            "entries": entries,
        }
    })
}

/// Read a flat header map or the retained values in an outlined one.
fn har_header_map(value: Option<&Value>) -> Option<&Map<String, Value>> {
    let map = value?.as_object()?;
    Some(map.get("kept").and_then(Value::as_object).unwrap_or(map))
}

/// Write retained textual headers as HAR name-value pairs.
fn har_headers(headers: Option<&Map<String, Value>>) -> Vec<Value> {
    headers.into_iter().flatten().filter_map(|(name, v)| v.as_str().map(|v| json!({"name": name, "value": v}))).collect()
}

/// The text of a body the tools kept, if it is whole: a body cut at the cap would be replayed as
/// an incomplete answer (invalid JSON, a file missing its end), so only a complete one is used.
fn har_body<'a>(r: &'a Value, key: &str) -> Option<&'a str> {
    let body = r.get(key)?;
    if body.get("state").and_then(Value::as_str) != Some("captured") || body.get("truncated").and_then(Value::as_bool) == Some(true) {
        return None;
    }
    body.get("text").and_then(Value::as_str)
}

fn har_entry(r: &Value, started: &str) -> Value {
    let text = |key: &str| r.get(key).and_then(Value::as_str).unwrap_or_default();
    let request_headers = har_header_map(r.get("request_header_values").or_else(|| r.get("request_headers")));
    let mut request = json!({
        "method": if text("method").is_empty() { "GET" } else { text("method") },
        "url": text("url"),
        "httpVersion": "HTTP/1.1",
        "headers": har_headers(request_headers),
        "queryString": [],
        "cookies": [],
        "headersSize": -1,
        "bodySize": -1,
    });
    if let Some(body) = har_body(r, "request_body") {
        let mime = request_headers
            .into_iter()
            .flatten()
            .filter(|(name, _)| name.eq_ignore_ascii_case("content-type"))
            .find_map(|(_, value)| value.as_str())
            .unwrap_or_default();
        request["postData"] = json!({"mimeType": mime, "text": body});
    }
    let mut content = json!({"size": 0, "mimeType": text("mime_type")});
    if let Some(body) = har_body(r, "response_body") {
        content["text"] = json!(body);
        content["size"] = json!(r.get("response_body").and_then(|b| b.get("bytes")).and_then(Value::as_u64).unwrap_or(body.len() as u64));
    }
    json!({
        "startedDateTime": started,
        "time": 0,
        "request": request,
        "response": {
            "status": r.get("status").and_then(Value::as_i64).unwrap_or(0),
            "statusText": "",
            "httpVersion": "HTTP/1.1",
            "headers": har_headers(har_header_map(r.get("response_header_values").or_else(|| r.get("response_headers")))),
            "cookies": [],
            "content": content,
            "redirectURL": "",
            "headersSize": -1,
            "bodySize": -1,
        },
        "cache": {},
        "timings": {"send": 0, "wait": 0, "receive": 0},
    })
}

/// Now, as an ISO 8601 UTC stamp (`2026-11-14T09:30:00Z`), for a recording's `startedDateTime`.
pub fn utc_iso_now() -> String {
    let secs = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0);
    utc_iso(secs)
}

/// A Unix time as an ISO 8601 UTC stamp, without a calendar crate: days since the epoch to a civil
/// date, by Howard Hinnant's algorithm.
fn utc_iso(secs: u64) -> String {
    let days = secs / 86_400;
    let rest = secs % 86_400;
    let z = days as i64 + 719_468;
    let era = z.div_euclid(146_097);
    let doe = z - era * 146_097;
    let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
    let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
    let mp = (5 * doy + 2) / 153;
    let day = doy - (153 * mp + 2) / 5 + 1;
    let month = if mp < 10 { mp + 3 } else { mp - 9 };
    let year = if month <= 2 { yoe + era * 400 + 1 } else { yoe + era * 400 };
    format!("{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}Z", rest / 3600, (rest % 3600) / 60, rest % 60)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn split_url_authority_cases() {
        let mut failures = Vec::new();
        for (row, url, host, path, userinfo) in [
            ("at in path", "https://example.test/@profile", "example.test", "/@profile", false),
            ("at in file name", "https://cdn.example.test/icon@2x.png", "cdn.example.test", "/icon@2x.png", false),
            ("at in query", "https://example.test/?q=a@b", "example.test", "/", false),
            ("at in fragment", "https://example.test/#a@b", "example.test", "/", false),
            ("user-info and at in path", "https://user:pw@example.test/@x", "example.test", "/@x", true),
            ("IPv6 and port", "https://[::1]:8080/@x", "[::1]:8080", "/@x", false),
            ("query with slash, no path", "https://example.test?q=a/b@c", "example.test", "/", false),
            ("port and at in path", "http://example.test:8080/a@b", "example.test:8080", "/a@b", false),
            ("fragment, no path", "https://example.test#a@b", "example.test", "/", false),
            ("user-info only", "https://user@example.test/p", "example.test", "/p", true),
            ("plain", "https://example.test/path", "example.test", "/path", false),
            ("percent-encoded at in user-info", "https://u%40x:pw@example.test/", "example.test", "/", true),
            ("two at signs in user-info", "https://a@b@EXAMPLE.test/p", "example.test", "/p", true),
        ] {
            let (_, actual_host, actual_path, _, actual_userinfo) = split_url(url);
            let actual = (actual_host.as_str(), actual_path.as_str(), actual_userinfo);
            let expected = (host, path, userinfo);
            if actual != expected {
                failures.push(format!("{row}: expected {expected:?}, got {actual:?}"));
            }
        }
        assert!(failures.is_empty(), "{}", failures.join("\n"));
    }

    const HAR: &str = r#"{"log":{"version":"1.2","creator":{"name":"t","version":"1"},"entries":[
      {"startedDateTime":"2026-01-01T00:00:00Z","time":120,
       "request":{"method":"GET","url":"https://bob:hunter2@api.example.test/v1/users/42?access_token=SECRET123&page=2","headers":[{"name":"Authorization","value":"Bearer SECRET123"},{"name":"Accept","value":"application/json"},{"name":"X-Trace","value":"trace-hunter2"}],"cookies":[{"name":"sid","value":"COOKIEVAL"}]},
       "response":{"status":200,"headers":[{"name":"Set-Cookie","value":"sid=COOKIEVAL"}],"content":{"size":40,"mimeType":"application/json","text":"{\"id\":42,\"name\":\"Ann\",\"tags\":[\"a\"],\"hunter2\":true}"}},"timings":{"wait":100}},
      {"startedDateTime":"2026-01-01T00:00:01Z","time":30,
       "request":{"method":"GET","url":"https://api.example.test/v1/users/7","headers":[]},
       "response":{"status":404,"headers":[],"content":{"size":2,"mimeType":"application/json","text":"eyJlcnJvciI6Im5vIn0=","encoding":"base64"}},"timings":{}},
      {"startedDateTime":"2026-01-01T00:00:02Z","time":5,
       "request":{"method":"POST","url":"https://api.example.test/v1/login","headers":[],"postData":{"mimeType":"application/x-www-form-urlencoded","text":"user=ann&password=hunter2"}},
       "response":{"status":302,"headers":[{"name":"Location","value":"https://u:p@api.example.test/home"}],"content":{"size":0,"mimeType":""},"redirectURL":"https://api.example.test/home?token=abc"},"timings":{}}
    ]}}"#;

    fn run(sensitive: &[&str]) -> Value {
        inspect_har(
            HAR.as_bytes(),
            &HarOptions { hidden_values: sensitive.iter().map(|s| s.to_string()).collect(), limit: 50, max_entries: 1000, ..Default::default() },
        )
        .unwrap()
    }

    #[test]
    fn nothing_secret_survives() {
        let v = run(&["hunter2"]);
        let mut all = BTreeSet::new();
        all_strings(&v, &mut all);
        for s in &all {
            for secret in ["SECRET123", "COOKIEVAL", "hunter2", "bob:", "u:p@", "Ann"] {
                assert!(!s.contains(secret), "{secret} leaked in {s:?}");
            }
        }
        assert_eq!(v["summary"]["entries"], 3);
        assert_eq!(v["summary"]["methods"]["GET"], 2);
        assert_eq!(v["entries"][0]["url"], "https://api.example.test/v1/users/42?access_token=[redacted]&page=2");
        assert_eq!(v["entries"][0]["request_headers"][0]["value"], Value::Null);
        assert_eq!(v["entries"][0]["request_headers"][2]["value"], "trace-[redacted]");
        assert_eq!(v["entries"][2]["request_body"]["fields"], json!(["user", "password"]));
        let outline = &v["entries"][0]["response_body"]["outline"];
        assert_eq!(outline["properties"]["id"], "integer");
        assert!(outline["properties"].get("hunter2").is_none(), "sensitive property name dropped");
        assert_eq!(v["entries"][1]["response_body"]["outline"]["properties"]["error"], "string");
    }

    #[test]
    fn api_surface_groups_by_template() {
        let v = run(&[]);
        let ops = v["api_surface"].as_array().unwrap();
        let users = ops.iter().find(|o| o["operation"] == "GET https://api.example.test/v1/users/{n}").unwrap();
        assert_eq!(users["count"], 2);
        assert_eq!(users["statuses"], json!([200, 404]));
        assert_eq!(path_template("/a/123e4567-e89b-12d3-a456-426614174000/2026-01-02/deadbeefdeadbeefdeadbeefdeadbeef"), "/a/{uuid}/{date}/{hex}");
    }

    #[test]
    fn encoded_forms_are_redacted() {
        let r = Redactor::new(&["a b/c\"".into()]);
        assert!(r.needles.iter().any(|n| n == "a%20b%2Fc%22"));
        assert!(r.needles.iter().any(|n| n == "a b/c\\\""));
        let mut r = r;
        assert_eq!(r.text("/x", "q=a%20b%2Fc%22"), "q=[redacted]");
        assert!(inspect_har(b"not json", &HarOptions::default()).is_err());
    }

    /// A recording written from what a browser run kept reads back as the same requests, and a
    /// stand-in built from it knows them.
    #[test]
    fn a_recording_of_kept_requests_reads_back() {
        let requests = vec![
            json!({"method": "GET", "url": "https://api.example.test/v1/items/42", "status": 200, "mime_type": "application/json",
                   "response_headers": {"content-type": "application/json"},
                   "response_body": {"state": "captured", "bytes": 11, "text": "{\"title\":1}"},
                   "request_headers": {"accept": "application/json", "authorization": {"value_length": 12}}}),
            json!({"method": "POST", "url": "https://api.example.test/v1/items", "status": 201, "mime_type": "application/json",
                   "request_body": {"state": "captured", "bytes": 9, "text": "{\"a\":true}"},
                   "response_body": {"state": "not_exposed"}}),
        ];
        let har = har_from_network(&requests, "2026-11-14T09:30:00Z");
        let bytes = serde_json::to_vec(&har).unwrap();
        let parsed = har::parse(&bytes, 10).unwrap();
        assert_eq!(parsed.entries.len(), 2);
        assert_eq!(parsed.entries[0].request.method, "GET");
        assert_eq!(parsed.entries[0].response.text.as_deref(), Some("{\"title\":1}"));
        assert_eq!(parsed.entries[1].request.post.as_ref().and_then(|p| p.text.as_deref()), Some("{\"a\":true}"));
        assert_eq!(parsed.entries[1].response.status, 201);
        assert_eq!(parsed.entries[1].response.text, None, "a body that wasn't kept isn't invented");
        // A header whose value was reduced to its length stays out.
        let headers = har["log"]["entries"][0]["request"]["headers"].as_array().unwrap();
        assert_eq!(headers.len(), 1);
        let standin = crate::standin::build(&parsed, &crate::standin::BuildOptions::default());
        assert_eq!(standin.routes.len(), 2, "{:?}", standin.routes.iter().map(|r| &r.template).collect::<Vec<_>>());
        assert_eq!(standin.hosts, vec!["api.example.test".to_string()]);
    }

    /// #1838: the browser keeps headers under `response_headers.kept`; a recording has to read
    /// that shape or a `Link` a front end paginates with disappears from replayed answers.
    #[test]
    fn a_recording_reads_the_nested_header_shape_the_browser_keeps() {
        let requests = vec![json!({
            "method": "GET", "url": "https://api.example.test/v1/items?page=2", "status": 200,
            "mime_type": "application/json",
            "response_headers": {
                "kept": {"content-type": "application/json", "link": "</v1/items?page=3>; rel=\"next\"", "x-long": {"value_length": 40}},
                "credentials_present": ["set-cookie"],
            },
            "response_body": {"state": "captured", "bytes": 2, "text": "{}"},
        })];
        let har = har_from_network(&requests, "2026-11-14T09:30:00Z");
        let headers = har["log"]["entries"][0]["response"]["headers"].as_array().unwrap();
        let link = headers.iter().find(|h| h["name"] == "link").expect("the kept Link header");
        assert_eq!(link["value"], "</v1/items?page=3>; rel=\"next\"");
        assert!(headers.iter().any(|h| h["name"] == "content-type"));
        assert!(!headers.iter().any(|h| h["name"] == "x-long"), "a length isn't a value");
        let parsed = har::parse(&serde_json::to_vec(&har).unwrap(), 10).unwrap();
        assert!(parsed.entries[0].response.headers.iter().any(|(name, _)| name == "link"), "{:?}", parsed.entries[0].response.headers);
    }

    /// #1838: a body cut at the cap would replay as an incomplete answer, so it isn't one.
    #[test]
    fn a_recording_leaves_out_a_truncated_body() {
        let requests = vec![json!({
            "method": "GET", "url": "https://api.example.test/v1/big", "status": 200, "mime_type": "application/json",
            "request_body": {"state": "captured", "bytes": 70_000, "truncated": true, "text": "{\"q\":"},
            "response_body": {"state": "captured", "bytes": 70_000, "truncated": true, "text": "{\"a\":"},
        })];
        let har = har_from_network(&requests, "2026-11-14T09:30:00Z");
        let entry = &har["log"]["entries"][0];
        assert!(entry["response"]["content"].get("text").is_none(), "{}", entry["response"]);
        assert!(entry["request"].get("postData").is_none(), "{}", entry["request"]);
    }

    /// #1838: two GraphQL operations to one endpoint are kept apart by the body they sent, so the
    /// stand-in answers each with its own.
    #[test]
    fn two_operations_to_one_endpoint_are_told_apart_by_their_bodies() {
        let op = |query: &str, title: &str| {
            let answer = format!("{{\"data\":{{\"{title}\":[]}}}}");
            json!({"method": "POST", "url": "https://api.example.test/graphql", "status": 200, "mime_type": "application/json",
                   "request_body": {"state": "captured", "bytes": query.len(), "text": query},
                   "response_body": {"state": "captured", "bytes": answer.len(), "text": answer}})
        };
        let requests = vec![op("{\"query\":\"query Items { items { id } }\"}", "items"), op("{\"query\":\"query Users { users { id } }\"}", "users")];
        let har = har_from_network(&requests, "2026-11-14T09:30:00Z");
        let parsed = har::parse(&serde_json::to_vec(&har).unwrap(), 10).unwrap();
        let standin = crate::standin::build(&parsed, &crate::standin::BuildOptions::default());
        let route = standin.routes.iter().find(|r| r.template == "/graphql").expect("one route");
        let keys: Vec<Option<String>> = route.samples.iter().map(|s| s.request_key.clone()).collect();
        assert_eq!(keys.len(), 2, "{keys:?}");
        assert!(keys.iter().all(Option::is_some), "both bodies are request keys: {keys:?}");
        assert_ne!(keys[0], keys[1], "the two operations are told apart");
    }

    #[test]
    fn a_unix_time_is_written_as_a_utc_stamp() {
        assert_eq!(utc_iso(0), "1970-01-01T00:00:00Z");
        assert_eq!(utc_iso(86_399), "1970-01-01T23:59:59Z");
        assert_eq!(utc_iso(951_782_400), "2000-02-29T00:00:00Z");
    }
}