GitCorner

Code

ossartcraftfigcraft🌐
b45255bb6fe2
use super::hosting::{Open, spawn};
use super::limits::{PerAddress, ip_key};
use super::queue::{Blocks, HubIn, InFlight, Line, Out, wide_block};
use super::{
    Arc, AtomicBool, AtomicUsize, Bucket, Bytes, ClientMsg, CloseCode, CloseFrame, Duplex, Duration, ErrorResponse, Instant, IpAddr, Message, Meter,
    NetLimits, Ordering, Read, Readers, Receiver, RecvTimeoutError, Request, Response, Role, Sender, ServerMsg, SocketAddr, TcpListener, TcpStream,
    WebSocket, Write, config, io, message, shutdown, sync_channel,
};

/// How long a closing connection waits for the peer to hang up.
const LINGER: Duration = Duration::from_secs(2);

/// Signals (cursor chat, spotlight) a client may send per second on average, and in a burst;
/// beyond that they are dropped.
pub const SIGNAL_RATE: f64 = 20.0;

pub const SIGNAL_BURST: f64 = 40.0;

/// Longest request line an uptime check may have.
const MAX_CHECK_LINE: usize = 2048;

/// An uptime check: a plain `GET` of a path ending in `/healthz` is answered `200 ok` and closed.
/// True when it was one (the connection is done). Anything else is only peeked at, so a WebSocket
/// handshake goes on untouched.
fn health_check(stream: &mut TcpStream, until: Instant) -> bool {
    let mut buf = [0u8; MAX_CHECK_LINE];
    let line = loop {
        let n = match stream.peek(&mut buf) {
            Ok(0) | Err(_) => return false,
            Ok(n) => n,
        };
        let seen = buf.get(..n).unwrap_or_default();
        if !seen.starts_with(b"GET ") && !b"GET ".starts_with(seen) {
            return false;
        }
        if let Some(end) = seen.windows(2).position(|w| w == b"\r\n") {
            break seen.get(..end).unwrap_or_default().to_vec();
        }
        if n == buf.len() || Instant::now() >= until {
            return false;
        }
        // The rest of the line is on its way (it nearly always arrives in one piece).
        std::thread::sleep(Duration::from_millis(10));
    };
    let path = line.split(|b| *b == b' ').nth(1).unwrap_or_default();
    let path = path.split(|b| *b == b'?').next().unwrap_or_default();
    if !path.ends_with(b"/healthz") {
        return false;
    }
    // Read the request (its headers end with a blank line), so closing doesn't reset it.
    let mut req = Vec::new();
    let mut chunk = [0u8; 1024];
    while !req.windows(4).any(|w| w == b"\r\n\r\n") && req.len() < 16 << 10 {
        // The same deadline as the request line: a peer dripping header bytes would otherwise
        // hold this thread and its connection slot for as long as it liked.
        let left = until.saturating_duration_since(Instant::now());
        if left.is_zero() || stream.set_read_timeout(Some(left)).is_err() {
            break;
        }
        match stream.read(&mut chunk) {
            Ok(0) | Err(_) => break,
            Ok(n) => req.extend_from_slice(chunk.get(..n).unwrap_or_default()),
        }
    }
    let _ = stream
        .write_all(b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 3\r\nCache-Control: no-store\r\nConnection: close\r\n\r\nok\n");
    let _ = stream.shutdown(std::net::Shutdown::Write);
    true
}

/// The client a trusted proxy names: the last address of its last `X-Forwarded-For` (the one the
/// proxy added; any before it are whatever the client sent).
fn forwarded_for(req: &Request) -> Option<IpAddr> {
    let last = req.headers().get_all("x-forwarded-for").iter().next_back()?.to_str().ok()?;
    let entry = last.rsplit(',').next()?.trim();
    entry.parse::<IpAddr>().ok().or_else(|| entry.parse::<SocketAddr>().ok().map(|a| a.ip()))
}

/// What every connection shares: the count per address, the bytes queued for all of them and the
/// bytes in flight to the hub.
#[derive(Clone)]
pub(super) struct Shared {
    pub(super) per_address: PerAddress,
    pub(super) queued: Arc<AtomicUsize>,
    pub(super) in_flight: InFlight,
    pub(super) blocks: Blocks,
    /// Bytes being read into messages, all connections together, and who is reading them.
    pub(super) reading: Arc<AtomicUsize>,
    pub(super) readers: Readers,
}

pub(super) fn accept(listener: &TcpListener, hub: &Sender<HubIn>, stop: &AtomicBool, open: &Arc<AtomicUsize>, limits: NetLimits, shared: Shared) {
    let mut next = 0u64;
    for stream in listener.incoming() {
        if stop.load(Ordering::SeqCst) {
            break;
        }
        let stream = match stream {
            Ok(s) => s,
            Err(e) => {
                // Out of sockets or similar: don't spin.
                log::warn!("live session: accept failed: {e}");
                std::thread::sleep(Duration::from_millis(50));
                continue;
            }
        };
        if open.load(Ordering::Relaxed) >= limits.max_connections {
            log::info!("live session: too many connections; refusing one");
            continue;
        }
        open.fetch_add(1, Ordering::Relaxed);
        let guard = Open(open.clone());
        next += 1;
        let (conn, hub, shared) = (next, hub.clone(), shared.clone());
        if let Err(e) = spawn("live-conn", move || serve(stream, conn, &hub, limits, guard, &shared)) {
            log::warn!("live session: can't start a connection thread: {e}");
        }
    }
}

/// A close reason that fits a WebSocket close frame (123 bytes).
pub(super) fn reason(s: &str) -> String {
    let mut end = s.len().min(123);
    while !s.is_char_boundary(end) {
        end -= 1;
    }
    s.get(..end).unwrap_or_default().to_string()
}

pub(super) fn encode(msg: &ServerMsg) -> Option<Bytes> {
    match message::encode(msg) {
        Ok(b) => Some(Bytes::from(b)),
        Err(e) => {
            log::warn!("live session: can't encode a message: {e}");
            None
        }
    }
}

/// Queue `Refused {reason}` and a close for one connection (small: not counted against its caps).
pub(super) fn refuse(out: &Line, why: &str) {
    refused(out, why);
}

/// [`refuse`], and whether the close was queued (it isn't when the queue is full).
pub(super) fn refused(out: &Line, why: &str) -> bool {
    if let Some(b) = encode(&ServerMsg::Refused { reason: why.to_string() }) {
        let _ = out.tx.try_send(Out::Bytes(b, None));
    }
    out.tx.try_send(Out::Close(CloseCode::Policy, reason(why))).is_ok()
}

/// What the server tells a client when it is too busy to take more from it.
const BUSY: &str = "the server is busy: try again in a moment";

/// Read the next binary message (skipping pings, until `deadline`) and its size, or why the connection should end.
fn next_message(ws: &mut WebSocket<Duplex>, max: usize, deadline: Option<Instant>) -> Result<Option<(ClientMsg, usize)>, String> {
    loop {
        return match ws.read() {
            Ok(Message::Binary(b)) => {
                // Whole now: it counts as in flight from here, no longer as being read.
                ws.get_mut().read_done();
                message::decode::<ClientMsg>(&b, max).map(|m| Some((m, b.len()))).map_err(|_| "that message couldn't be read".to_string())
            }
            Ok(Message::Text(_)) => Err("only binary messages are spoken here".into()),
            // Pings keep the per-read timeout alive, but not the overall deadline (slowloris), and
            // a message trickled in is cut off by it (`Duplex::read_until`).
            Ok(Message::Ping(_) | Message::Pong(_) | Message::Frame(_)) | Err(tungstenite::Error::Io(_))
                if deadline.is_some_and(|d| Instant::now() >= d) =>
            {
                Err("no hello in time".into())
            }
            Ok(Message::Ping(_) | Message::Pong(_) | Message::Frame(_)) => continue,
            Ok(Message::Close(_)) => Ok(None),
            Err(tungstenite::Error::Capacity(_)) => Err(too_large(max)),
            // The reading budget's refusal (see `Meter`).
            Err(tungstenite::Error::Io(e)) if e.kind() == io::ErrorKind::Other => Err(e.to_string()),
            Err(_) => Ok(None),
        };
    }
}

/// Why a message over `max` bytes is refused.
fn too_large(max: usize) -> String {
    match max >> 20 {
        0 => format!("message too large: this server takes at most {} KiB at once here", max >> 10),
        mib => format!("message too large: this server takes at most {mib} MiB at once (a change that big can't be kept here)"),
    }
}

/// One connection's reader (the writer runs on its own thread).
fn serve(mut stream: TcpStream, conn: u64, hub: &Sender<HubIn>, limits: NetLimits, open: Open, shared: &Shared) {
    let _ = stream.set_nodelay(true);
    let hello_deadline = Instant::now() + limits.hello_timeout;
    if stream.set_read_timeout(Some(limits.hello_timeout)).is_err() || stream.set_write_timeout(Some(limits.write_timeout)).is_err() {
        return;
    }
    if health_check(&mut stream, Instant::now() + limits.hello_timeout) {
        return;
    }
    let Ok(peer) = stream.peer_addr().map(|a| a.ip().to_canonical()) else { return };
    let proxied = limits.trusted_proxy.is_some_and(|p| p.to_canonical() == peer);
    let Ok((mut rd, wr, sock)) = Duplex::split(stream, None) else { return };
    // The handshake and the hello must be in by the deadline, however slowly they trickle in.
    rd.read_until(Some(hello_deadline));
    let cfg = config(limits.max_client_message);
    let mut forwarded = None;
    // The handshake callback's type is tungstenite's (its error is an HTTP response).
    #[allow(clippy::result_large_err)]
    let read_client = |req: &Request, resp: Response| -> Result<Response, ErrorResponse> {
        if proxied {
            forwarded = forwarded_for(req);
        }
        Ok(resp)
    };
    let mut ws = match tungstenite::accept_hdr_with_config(rd, read_client, Some(config(limits.max_hello))) {
        Ok(ws) => ws,
        Err(e) => {
            log::debug!("live session: handshake failed: {e}");
            shutdown(&sock);
            return;
        }
    };
    if proxied && forwarded.is_none() {
        log::debug!("live session: connection {conn} came through the proxy without X-Forwarded-For");
    }
    let ip = ip_key(forwarded.unwrap_or(peer));
    let counted = shared.per_address.enter_all(ip, limits.max_per_address, limits.max_connections);
    let wide = wide_block(ip);
    let block = shared.blocks.get(wide);
    ws.get_mut().meter(Meter::new(
        conn,
        &sock,
        &shared.readers,
        block.reading.clone(),
        shared.reading.clone(),
        limits.max_reading_block,
        limits.max_reading_total,
    ));
    let (tx, rx) = sync_channel::<Out>(limits.queue.max(4));
    let out = Line {
        tx,
        queued: Arc::default(),
        waiting: Arc::default(),
        max_presence: limits.queue.max(4) / 2,
        block: block.queued.clone(),
        all: shared.queued.clone(),
        max: limits.max_queued_bytes,
        max_block: limits.max_queued_block,
        max_all: limits.max_queued_total,
    };
    let writer =
        match sock.try_clone().map(|s| spawn("live-write", move || write_loop(WebSocket::from_raw_socket(wr, Role::Server, Some(cfg)), &rx, &s))) {
            Ok(Ok(t)) => t,
            _ => {
                shutdown(&sock);
                return;
            }
        };
    let end = |why: Option<&str>| {
        if let Some(why) = why {
            refuse(&out, why);
        } else {
            let _ = out.tx.try_send(Out::Close(CloseCode::Normal, String::new()));
        }
    };
    if counted.is_none() {
        log::info!("event=refused ip={ip} reason=connections_per_address");
        end(Some("too many connections from your address: close some and try again"));
    } else {
        let hello = next_message(&mut ws, limits.max_hello, Some(hello_deadline));
        // In time or not, the hello's deadline is over: what's read next has timeouts of its own.
        ws.get_mut().read_until(None);
        match hello {
            Ok(Some((
                first @ (ClientMsg::Hello(_)
                | ClientMsg::Manage(_)
                | ClientMsg::OpenLibrary(_)
                | ClientMsg::OpenPlugins(_)
                | ClientMsg::OpenHosting(_)
                | ClientMsg::Resolve(_)
                | ClientMsg::ResolveTeam(_)),
                bytes,
            ))) => {
                // Sessions and the operator may use the last connections; one-shot requests
                // (hosting, libraries, browsing plugins) leave them free.
                let session = matches!(first, ClientMsg::Hello(_) | ClientMsg::Manage(_));
                // A publish uploads a whole file; a large one on a slow link takes the server a
                // while to receive and answer. The manage/hosting connections that carry uploads
                // wait the longer request-reply timeout, not the short request idle.
                let idle = match first {
                    ClientMsg::Manage(_) | ClientMsg::OpenHosting(_) => limits.request_reply_timeout,
                    ClientMsg::Hello(_) => limits.idle_timeout,
                    _ => limits.request_idle,
                };
                let max = match &first {
                    ClientMsg::OpenPlugins(o) if o.key.is_empty() => limits.max_request,
                    _ => limits.max_client_message,
                };
                let reserved = !session && open.0.load(Ordering::Relaxed) > limits.max_connections.saturating_sub(limits.reserve_for_sessions);
                // (In flight is only taken when the connection goes on: the hub gives it back.)
                if reserved || !shared.in_flight.take(bytes, limits.max_in_flight) {
                    end(Some(BUSY));
                } else {
                    let (admitted, verdict) = sync_channel(1);
                    let opened =
                        sock.try_clone().is_ok_and(|s| hub.send(HubIn::Open { conn, ip, first, out: out.clone(), sock: s, bytes, admitted }).is_ok());
                    // Nothing more is read until the hub took the connection in (its token checked):
                    // only then may it send messages of the full size.
                    if opened && verdict.recv().is_ok() && ws.get_ref().set_read_timeout(Some(idle)).is_ok() {
                        ws.set_config(|c| *c = config(max));
                        read_loop(&mut ws, conn, hub, limits, max, &shared.in_flight, &end);
                        let _ = hub.send(HubIn::Gone { conn });
                    }
                }
            }
            Ok(Some(_)) => end(Some("say hello first")),
            Ok(None) => {}
            Err(why) => end(Some(&why)),
        }
    }
    drop(counted);
    // The block's counters go with the last handle on them (the table forgets it then).
    drop(block);
    ws.get_mut().read_until(None);
    linger(&mut ws);
    drop(out);
    // The writer closes the socket once it has sent what's queued (a refusal, a close).
    let _ = writer.join();
    shutdown(&sock);
}

fn read_loop(
    ws: &mut WebSocket<Duplex>,
    conn: u64,
    hub: &Sender<HubIn>,
    limits: NetLimits,
    max: usize,
    in_flight: &InFlight,
    end: &dyn Fn(Option<&str>),
) {
    let mut bucket = Bucket::new(limits.rate, limits.burst);
    let mut signals = Bucket::new(SIGNAL_RATE, SIGNAL_BURST);
    loop {
        match next_message(ws, max, None) {
            Ok(Some((msg, bytes))) => {
                let now = Instant::now();
                if !bucket.take(now) {
                    log::info!("live session: connection {conn} sends too much; disconnecting it");
                    return end(Some("too many messages"));
                }
                // Typing sends a chat signal per keystroke; more than this is dropped.
                if matches!(msg, ClientMsg::Signal(_)) && !signals.take(now) {
                    log::debug!("live session: connection {conn} sends signals too fast; dropping one");
                    continue;
                }
                if !in_flight.take(bytes, limits.max_in_flight) {
                    log::info!("live session: the hub is behind; dropping connection {conn}");
                    return end(Some(BUSY));
                }
                if hub.send(HubIn::Msg { conn, msg, bytes }).is_err() {
                    return;
                }
            }
            Ok(None) => return end(None),
            Err(why) => return end(Some(&why)),
        }
    }
}

fn write_loop(mut ws: WebSocket<Duplex>, rx: &Receiver<Out>, sock: &TcpStream) {
    let mut close = Some((CloseCode::Normal, String::new()));
    while let Ok(o) = rx.recv() {
        match o {
            Out::Bytes(b, _charge) => {
                if ws.send(Message::Binary(b)).is_err() {
                    close = None;
                    break;
                }
            }
            Out::Close(code, why) => {
                close = Some((code, why));
                break;
            }
        }
    }
    if let Some((code, why)) = close {
        // Close politely: the close frame, then no more writing; the reader drains what the peer
        // still sends until it hangs up (closing on unread data would reset the connection and
        // could lose the close frame), or until the linger runs out.
        let _ = ws.close(Some(CloseFrame { code, reason: why.into() }));
        let _ = sock.shutdown(std::net::Shutdown::Write);
        let until = Instant::now() + LINGER;
        while let Ok(_) | Err(RecvTimeoutError::Timeout) = rx.recv_timeout(until.saturating_duration_since(Instant::now())) {
            if Instant::now() >= until {
                break;
            }
        }
    }
    shutdown(sock);
}

/// Read and drop what the peer still sends after the close, until it hangs up (at most
/// [`LINGER`]). Raw bytes, not frames: after a frame over the size limit the WebSocket's reader
/// is mid-frame and mustn't be asked for more.
fn linger(ws: &mut WebSocket<Duplex>) {
    let until = Instant::now() + LINGER;
    let mut buf = [0u8; 4096];
    while Instant::now() < until && ws.get_ref().set_read_timeout(Some(Duration::from_millis(250))).is_ok() {
        match ws.get_mut().read(&mut buf) {
            Ok(0) => break,
            Ok(_) => {}
            Err(e) if matches!(e.kind(), io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut | io::ErrorKind::Interrupted) => {}
            Err(_) => break,
        }
    }
}

#[cfg(test)]
mod health_check_tests {
    use super::*;

    #[test]
    fn a_health_check_dripping_headers_ends_at_the_deadline() {
        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
        let addr = listener.local_addr().unwrap();
        let client = std::thread::spawn(move || {
            let mut c = TcpStream::connect(addr).unwrap();
            c.write_all(b"GET /healthz HTTP/1.1\r\n").unwrap();
            // One header byte every 20 ms, far past the deadline, until the server hangs up.
            for _ in 0..200 {
                if c.write_all(b"X").is_err() {
                    break;
                }
                std::thread::sleep(Duration::from_millis(20));
            }
        });
        let (mut stream, _) = listener.accept().unwrap();
        let start = Instant::now();
        assert!(health_check(&mut stream, start + Duration::from_millis(300)));
        assert!(start.elapsed() < Duration::from_secs(2), "{:?}", start.elapsed());
        drop(stream);
        client.join().unwrap();
    }
}