642d5a256791
html.rs
//! HTML pages: script references, inline scripts, module preloads and the Content Security Policy.
//! A tolerant tag scanner, not a full HTML parser.
use serde::{Deserialize, Serialize};
const MAX_TAGS: usize = 2_000;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct HtmlScript {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub src: Option<String>,
pub module: bool,
/// Byte range of inline script text.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub inline: Option<(u32, u32)>,
pub line: u32,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct HtmlFacts {
pub scripts: Vec<HtmlScript>,
pub preloads: Vec<String>,
/// Nonempty meta policies in document order, separated by a comma and space.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub csp: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub title: Option<String>,
}
/// Attributes and end offset of a complete tag, starting just after its name.
fn attributes(src: &str, lower: &str, mut i: usize) -> Option<(Vec<(String, String)>, usize)> {
let b = lower.as_bytes();
let mut out = Vec::new();
while i < b.len() {
while b.get(i).is_some_and(|c| c.is_ascii_whitespace() || *c == b'/') {
i += 1;
}
match b.get(i) {
None => break,
Some(b'>') => return Some((out, i + 1)),
_ => {}
}
let ns = i;
while b.get(i).is_some_and(|c| !c.is_ascii_whitespace() && *c != b'=' && *c != b'>') {
i += 1;
}
let name = lower.get(ns..i).unwrap_or("").to_string();
while b.get(i).is_some_and(u8::is_ascii_whitespace) {
i += 1;
}
let mut value = String::new();
if b.get(i) == Some(&b'=') {
i += 1;
while b.get(i).is_some_and(u8::is_ascii_whitespace) {
i += 1;
}
match b.get(i) {
Some(q @ (b'"' | b'\'')) => {
let q = *q;
let vs = i + 1;
i = vs;
while b.get(i).is_some_and(|c| *c != q) {
i += 1;
}
value = src.get(vs..i).unwrap_or("").to_string();
i += 1;
}
_ => {
let vs = i;
while b.get(i).is_some_and(|c| !c.is_ascii_whitespace() && *c != b'>') {
i += 1;
}
value = src.get(vs..i).unwrap_or("").to_string();
}
}
}
if name.is_empty() {
i += 1;
} else if out.len() < 64 {
out.push((name, decode_entities(&value)));
}
}
None
}
fn decode_entities(s: &str) -> String {
let mut out = String::with_capacity(s.len());
let mut rest = s;
while let Some(at) = rest.find('&') {
out.push_str(rest.get(..at).unwrap_or_default());
rest = rest.get(at + 1..).unwrap_or_default();
let decoded = rest
.strip_prefix('#')
.and_then(|number| {
let (digits, radix) = number.strip_prefix(['x', 'X']).map_or((number, 10), |n| (n, 16));
let count = digits.bytes().take_while(|c| if radix == 16 { c.is_ascii_hexdigit() } else { c.is_ascii_digit() }).count();
if count == 0 {
return None;
}
let value = digits.get(..count).and_then(|s| u32::from_str_radix(s, radix).ok()).unwrap_or(u32::MAX);
let consumed = rest.len() - digits.len() + count;
let consumed = consumed + usize::from(rest.as_bytes().get(consumed) == Some(&b';'));
// HTML maps the numeric C1 references through the Windows-1252 table.
const C1: [u32; 32] = [
0x20ac, 0x81, 0x201a, 0x192, 0x201e, 0x2026, 0x2020, 0x2021, 0x2c6, 0x2030, 0x160, 0x2039, 0x152, 0x8d, 0x17d, 0x8f, 0x90, 0x2018, 0x2019,
0x201c, 0x201d, 0x2022, 0x2013, 0x2014, 0x2dc, 0x2122, 0x161, 0x203a, 0x153, 0x9d, 0x17e, 0x178,
];
let value = value.checked_sub(0x80).and_then(|i| C1.get(i as usize)).copied().unwrap_or(value);
Some((char::from_u32(value).filter(|c| *c != '\0').unwrap_or('\u{fffd}'), consumed))
})
.or_else(|| {
[("amp;", '&'), ("quot;", '"'), ("apos;", '\''), ("lt;", '<'), ("gt;", '>')]
.into_iter()
.find_map(|(name, value)| rest.starts_with(name).then_some((value, name.len())))
});
if let Some((c, consumed)) = decoded {
out.push(c);
rest = rest.get(consumed..).unwrap_or_default();
} else {
out.push('&');
}
}
out.push_str(rest);
out
}
fn get<'v>(attrs: &'v [(String, String)], k: &str) -> Option<&'v str> {
attrs.iter().find(|(n, _)| n == k).map(|(_, v)| v.as_str())
}
/// Find an actual closing tag, not a longer name such as `</scripture>`.
fn closing_tag(lower: &str, from: usize, name: &str) -> Option<usize> {
let needle = format!("</{name}");
lower.get(from..)?.match_indices(&needle).find_map(|(offset, _)| {
let at = from + offset;
lower.as_bytes().get(at + needle.len()).filter(|c| c.is_ascii_whitespace() || matches!(c, b'/' | b'>')).map(|_| at)
})
}
/// The end of a comment, measured from just after `<!--`.
fn comment_end(rest: &str) -> usize {
if rest.starts_with('>') {
return 1;
}
if rest.starts_with("->") {
return 2;
}
rest.match_indices('>')
.find_map(|(i, _)| {
let before = rest.get(..i)?;
(before.ends_with("--") || before.ends_with("--!")).then_some(i + 1)
})
.unwrap_or(rest.len())
}
/// Scan an HTML document.
pub fn scan_html(src: &str) -> HtmlFacts {
let lower = src.to_ascii_lowercase();
let lines = crate::util::LineIndex::new(src);
let mut out = HtmlFacts::default();
let mut pos = 0usize;
let mut tags = 0usize;
while let Some(rel) = lower.get(pos..).and_then(|r| r.find('<')) {
tags += 1;
if tags > MAX_TAGS * 50 || out.scripts.len() >= MAX_TAGS {
break;
}
let at = pos + rel;
let rest = lower.get(at + 1..).unwrap_or("");
// Inspect only the names we handle: malformed runs of `<` must not rescan the remaining document.
let name = ["script", "link", "meta", "title", "style", "textarea", "xmp", "iframe", "noembed", "noframes", "plaintext"]
.into_iter()
.find(|name| rest.starts_with(*name) && rest.as_bytes().get(name.len()).is_none_or(|c| c.is_ascii_whitespace() || matches!(*c, b'/' | b'>')))
.unwrap_or_default();
let after = at + 1 + name.len();
match name {
"script" => {
let Some((attrs, end)) = attributes(src, &lower, after) else { break };
let close = closing_tag(&lower, end, "script").unwrap_or(lower.len());
let module = get(&attrs, "type").is_some_and(|t| t.eq_ignore_ascii_case("module"));
let line = lines.line(u32::try_from(at).unwrap_or(u32::MAX));
let src_attr = get(&attrs, "src").map(String::from);
let js = javascript_type(get(&attrs, "type"), get(&attrs, "language"));
let inline = (src_attr.is_none() && close > end && js && src.get(end..close).is_some_and(|t| !t.trim().is_empty()))
.then(|| (u32::try_from(end).unwrap_or(u32::MAX), u32::try_from(close).unwrap_or(u32::MAX)));
if js && (src_attr.is_some() || inline.is_some()) {
out.scripts.push(HtmlScript { src: src_attr, module, inline, line });
}
pos = close.max(end).max(at + 1);
}
"style" | "textarea" | "xmp" | "iframe" | "noembed" | "noframes" => {
let Some((_, end)) = attributes(src, &lower, after) else { break };
pos = closing_tag(&lower, end, name).unwrap_or(lower.len()).max(at + 1);
}
"plaintext" => break,
"link" => {
let Some((attrs, end)) = attributes(src, &lower, after) else { break };
let rel_attr = get(&attrs, "rel").unwrap_or("").to_ascii_lowercase();
let script_destination = get(&attrs, "as").is_some_and(|value| value.eq_ignore_ascii_case("script"));
if rel_attr.split_ascii_whitespace().any(|token| token == "modulepreload" || (token == "preload" && script_destination))
&& let Some(h) = get(&attrs, "href")
&& out.preloads.len() < MAX_TAGS
{
out.preloads.push(h.to_string());
}
pos = end.max(at + 1);
}
"meta" => {
let Some((attrs, end)) = attributes(src, &lower, after) else { break };
if get(&attrs, "http-equiv").is_some_and(|h| h.eq_ignore_ascii_case("content-security-policy"))
&& let Some(policy) = get(&attrs, "content").filter(|s| !s.trim_matches(|c: char| c.is_ascii_whitespace()).is_empty())
{
let combined = out.csp.get_or_insert_with(String::new);
if !combined.is_empty() {
combined.push_str(", ");
}
combined.push_str(policy);
}
pos = end.max(at + 1);
}
"title" => {
let Some((_, end)) = attributes(src, &lower, after) else { break };
let close = closing_tag(&lower, end, "title").unwrap_or(lower.len());
out.title = src.get(end..close).map(|t| decode_entities(t).trim().chars().take(200).collect());
pos = close.max(at + 1);
}
_ if let Some(comment) = rest.strip_prefix("!--") => {
pos = at + 4 + comment_end(comment);
}
_ if (rest.starts_with('!') && !rest.starts_with("!doctype") && !rest.starts_with("![cdata["))
|| rest.strip_prefix('/').and_then(|s| s.as_bytes().first()).is_some_and(|c| !c.is_ascii_alphabetic()) =>
{
// The bogus comment state ends at the first greater-than sign, even inside quotes.
pos = rest.find('>').map_or(lower.len(), |end| at + 2 + end);
}
_ if rest.as_bytes().first().is_some_and(u8::is_ascii_alphabetic)
|| rest.strip_prefix('/').and_then(|s| s.as_bytes().first()).is_some_and(u8::is_ascii_alphabetic) =>
{
let Some((_, end)) = attributes(src, &lower, after) else { break };
pos = end.max(at + 1);
}
_ => pos = at + 1,
}
}
out
}
const JAVASCRIPT_MIME_TYPES: &[&str] = &[
"text/javascript",
"text/javascript1.0",
"text/javascript1.1",
"text/javascript1.2",
"text/javascript1.3",
"text/javascript1.4",
"text/javascript1.5",
"application/javascript",
"application/ecmascript",
"text/ecmascript",
"application/x-javascript",
"application/x-ecmascript",
"text/x-javascript",
"text/x-ecmascript",
"text/jscript",
"text/livescript",
];
/// Whether a script is executable code to inventory, including explicit in-browser Babel types.
fn javascript_type(t: Option<&str>, language: Option<&str>) -> bool {
let whitespace = |c: char| c.is_ascii_whitespace();
if let Some(t) = t {
if t.is_empty() || t.eq_ignore_ascii_case("module") {
return true;
}
let essence = t.trim_matches(whitespace);
return JAVASCRIPT_MIME_TYPES.iter().chain(&["text/babel", "text/jsx"]).any(|m| m.eq_ignore_ascii_case(essence));
}
let Some(language) = language.filter(|language| !language.is_empty()) else { return true };
// The browser matches `text/<language>` directly for a legacy language value.
JAVASCRIPT_MIME_TYPES.iter().filter_map(|m| m.strip_prefix("text/")).any(|m| m.eq_ignore_ascii_case(language))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn finds_scripts_and_csp() {
let h = r#"<!doctype html><html><head><meta http-equiv="Content-Security-Policy" content="default-src 'self'">
<title>Notes</title><link rel="modulepreload" href="/assets/vendor-abc.js">
<script type="module" src="./app.js"></script><!-- <script src="no.js"></script> -->
<script>window.x = 1;</script><script type="application/json">{}</script></head></html>"#;
let f = scan_html(h);
assert_eq!(f.scripts.len(), 2);
assert_eq!(f.scripts[0].src.as_deref(), Some("./app.js"));
assert!(f.scripts[0].module);
let (s, e) = f.scripts[1].inline.unwrap();
assert_eq!(&h[s as usize..e as usize], "window.x = 1;");
assert_eq!(f.preloads, vec!["/assets/vendor-abc.js"]);
assert_eq!(f.csp.as_deref(), Some("default-src 'self'"));
assert_eq!(f.title.as_deref(), Some("Notes"));
}
/// #285: import maps, speculation rules and HTML templates were read as JavaScript.
#[test]
fn only_javascript_types_are_scripts() {
for t in ["importmap", "speculationrules", "text/html", "application/json", "text/x-template"] {
let f = scan_html(&format!(r#"<script type="{t}">{{"imports": {{}}}}</script><script type="{t}" src="data.json"></script>"#));
assert!(f.scripts.is_empty(), "{t}: {:?}", f.scripts);
}
for t in ["", "module", "text/javascript", "Application/JavaScript", "text/babel"] {
assert_eq!(scan_html(&format!(r#"<script type="{t}">fetch("/real-api");</script>"#)).scripts.len(), 1, "{t}");
}
}
#[test]
fn junk_does_not_panic() {
for s in ["<", "<script", "<script src=", "<script src='x", "<<<<>>>", "<script>é", "<meta http-equiv=content-security-policy content=\"é", "<!--"] {
let _ = scan_html(s);
}
}
}
#[cfg(test)]
mod title_entities_regression {
use super::scan_html;
#[test]
fn displayed_title_decodes_references_once() {
let facts = scan_html("<title>Fish & Chips 🐟 &quot;</title>");
assert_eq!(facts.title.as_deref(), Some("Fish & Chips 🐟 ""));
assert_eq!(scan_html("<title>A &unknown; B</title>").title.as_deref(), Some("A &unknown; B"));
}
#[test]
fn title_limit_counts_displayed_characters() {
let html = format!("<title>{}</title>", "&".repeat(201));
assert_eq!(scan_html(&html).title, Some("&".repeat(200)));
}
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.