b45255bb6fe2
library.rs
use figcraft_collab::message::MAX_ASSET_USAGE;
use super::{
ACTIONS, ASSET_ACTIONS, Action, AssetActions, AssetUsage, BUSY_DOWNLOADS, Cursor, Detached, FILES, FileDetached, FileUsage, IpAddr, LIBRARY,
LibraryMeta, LibraryReply, MAX_FIRST_LIBRARY_VERSION, MAX_REPORTED, MAX_USAGE_FILE, MAX_USERS, MAX_WEEK_ROWS, MAX_WEEKS, META, NO_PUBLIC_LIBRARY,
NodeId, Path, PathBuf, Store, USAGE, USAGE_WRITE_EVERY, Usage, UsageCache, UsageEntry, WeekActions, clean_name, disk, fs, now, write_json,
};
/// Which of a file's totals a report brings.
#[derive(Clone, Copy, PartialEq, Eq)]
enum Count {
Inserted,
Detached,
}
const WEEK: u64 = 7 * 86_400;
/// Longest style or variable id a report may name.
const MAX_ASSET_ID: usize = 64;
/// The start of the week `t` falls in (seconds since 1970 UTC, a Monday at 00:00).
fn week_of(t: u64) -> u64 {
let Some(at) = i64::try_from(t).ok().and_then(|t| chrono::DateTime::from_timestamp(t, 0)) else { return 0 };
let monday = at.date_naive().week(chrono::Weekday::Mon).first_day();
u64::try_from(monday.and_time(chrono::NaiveTime::MIN).and_utc().timestamp()).unwrap_or(0)
}
/// A report's counts cleaned: positive, one per key, at most [`MAX_REPORTED`].
fn cleaned<K: Ord>(counts: Vec<(K, u32)>) -> Vec<(K, u32)> {
let mut counts: Vec<_> = counts.into_iter().filter(|(_, n)| *n > 0).take(MAX_REPORTED * 4).collect();
counts.sort();
counts.dedup_by(|a, b| a.0 == b.0);
counts.truncate(MAX_REPORTED);
counts
}
/// A report's style or variable counts: ids of a sensible length, at most [`MAX_ASSET_USAGE`].
fn known_assets(counts: Vec<(String, u32)>) -> Vec<(String, u32)> {
counts.into_iter().filter(|(id, _)| id.len() <= MAX_ASSET_ID).take(MAX_ASSET_USAGE).collect()
}
/// A kept list of statistics, or an empty one when it is missing or damaged (it starts over).
fn read_kept<T: serde::de::DeserializeOwned>(path: &Path) -> Vec<T> {
if !path.exists() {
return Vec::new();
}
disk::read_capped(path, MAX_USAGE_FILE).ok().and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default()
}
/// Raise the highest `kind` totals in `peaks` to `counts`, each from what `last` says was reported
/// before them; what grew past them, by how much.
fn grow<K: PartialEq + Clone>(peaks: &mut Vec<(K, u32, u32)>, counts: &[(K, u32)], kind: Count, last: impl Fn(&K) -> u32) -> Vec<(K, u32)> {
let mut grown = Vec::new();
for (c, n) in counts {
let at = match peaks.iter().position(|p| p.0 == *c) {
Some(at) => at,
None if peaks.len() < MAX_REPORTED => {
peaks.push((c.clone(), 0, 0));
peaks.len() - 1
}
None => continue,
};
let Some(p) = peaks.get_mut(at) else { continue };
let peak = if kind == Count::Inserted { &mut p.1 } else { &mut p.2 };
let base = (*peak).max(last(c));
if *n > base {
grown.push((c.clone(), *n - base));
}
*peak = base.max(*n);
}
grown
}
impl UsageEntry {
/// Raise this file's highest `kind` totals to `counts`; what grew past them, by how much.
/// A detach total reported before peaks were kept starts from that report.
fn grow_peaks(&mut self, counts: &[(NodeId, u32)], kind: Count) -> Vec<(NodeId, u32)> {
let detached = &self.detached;
grow(&mut self.peaks, counts, kind, |c| if kind == Count::Detached { detached.iter().find(|d| d.0 == *c).map_or(0, |d| d.1) } else { 0 })
}
}
/// Add `grown` to this week's `kind` counts in `weeks`, dropping weeks older than [`MAX_WEEKS`]
/// and, past [`MAX_WEEK_ROWS`], the oldest rows.
fn add_to_week<K: PartialEq + Clone>(weeks: &mut Vec<WeekActions<K>>, grown: &[(K, u32)], kind: Count, t: u64) {
let week = week_of(t);
weeks.retain(|w| w.week.saturating_add(MAX_WEEKS * WEEK) > week);
for (component, n) in grown {
let at = match weeks.iter().position(|w| w.week == week && w.component == *component) {
Some(at) => at,
None => {
weeks.push(WeekActions { week, component: component.clone(), insertions: 0, detachments: 0 });
weeks.len() - 1
}
};
if let Some(w) = weeks.get_mut(at) {
let slot = if kind == Count::Inserted { &mut w.insertions } else { &mut w.detachments };
*slot = slot.saturating_add(*n);
}
}
if weeks.len() > MAX_WEEK_ROWS {
weeks.sort_by_key(|w| std::cmp::Reverse(w.week));
weeks.truncate(MAX_WEEK_ROWS);
}
}
impl Store {
/// A file's team library for `ip`, unless its version is `have`; within what one address and
/// everyone may download (the size is known before anything is read).
pub(super) fn fetch_library(&mut self, ip: IpAddr, file: &str, have: Option<u64>) -> Result<LibraryReply, String> {
let (dir, max) = (self.dir(file), self.cfg.max_file_bytes);
let Some(lib) = self.entry(file)?.meta.library.clone() else { return Ok(LibraryReply::Empty) };
if have == Some(lib.version) {
return Ok(LibraryReply::Unchanged { version: lib.version });
}
let path = library_file(&dir, lib.version);
let size = fs::metadata(&path).map_or(0, |m| m.len());
let now = self.clock();
if !self.abuse.allow(ip, Action::Download(usize::try_from(size).unwrap_or(usize::MAX)), now) || !self.served.take_spread(now, size as f64) {
return Err(BUSY_DOWNLOADS.into());
}
Ok(LibraryReply::Library { version: lib.version, archive: disk::read_capped(&path, max)? })
}
/// Keep `archive` as a file's team library: it must read as a library (with the limits files
/// using it read it with), be the file's library if it has one, and be a newer version.
// shortcut: the archive is checked on the hub thread (other files' sessions wait meanwhile);
// move it to a worker if large libraries make that noticeable.
pub fn publish_library(&mut self, file: &str, archive: &[u8]) -> Result<u64, String> {
let max = self.cfg.max_file_bytes;
if self.cfg.hosting.open && self.entry(file)?.meta.owner.is_some() {
return Err(NO_PUBLIC_LIBRARY.into());
}
if archive.len() as u64 > max {
return Err(format!("the library is {} MiB; this server keeps files up to {} MiB", archive.len() >> 20, max >> 20));
}
// Everything it unpacks to counts against the same cap, so a small archive can't expand
// into a large library in memory.
let lib = match figcraft_format::read_library_within(Cursor::new(archive), max) {
Ok(Some(lib)) => lib,
Ok(None) => return Err("that isn't a published library".into()),
Err(e) => return Err(format!("the library can't be read: {e}")),
};
if lib.doc.node_count() > self.cfg.core.max_nodes {
return Err(format!("the library has more than {} layers", self.cfg.core.max_nodes));
}
let m = &lib.manifest;
let dir = self.dir(file);
let entry = self.entry(file)?;
if let Some(old) = &entry.meta.library {
if old.id != m.id {
return Err(format!("this file's library on the server is another one (\"{}\")", old.name));
}
if m.version <= old.version {
return Err(format!("version {} is on the server already: publish again to follow it", old.version));
}
// A jump would let a client use up the numbers and leave no newer version to publish.
if m.version > old.version.saturating_add(1) {
return Err(format!("version {} is too high: the next one after {} is {}", m.version, old.version, old.version.saturating_add(1)));
}
} else if m.version > MAX_FIRST_LIBRARY_VERSION {
return Err(format!("version {} is too high: a first publish goes up to {MAX_FIRST_LIBRARY_VERSION}", m.version));
}
// Each version has a file of its own, written before the description names it: a failed
// description write or a crash in between leaves the previous version served whole, never
// the new bytes under the old number.
let path = dir.join(library_name(m.version));
disk::write_atomic(&path, archive).map_err(|e| format!("couldn't keep the library: {e}"))?;
let before = entry.meta.library.replace(LibraryMeta { id: m.id.clone(), name: m.name.clone(), version: m.version });
if let Err(e) = write_json(&dir.join(META), &entry.meta) {
// Never served (the description doesn't name it); the next publish of it replaces it.
if let Err(r) = fs::remove_file(&path) {
log::warn!("file {file}: an unpublished library version couldn't be deleted: {r}");
}
entry.meta.library = before;
return Err(format!("couldn't keep the library: {e}"));
}
// The previous version isn't served any more.
if let Some(Err(e)) = before.map(|old| fs::remove_file(library_file(&dir, old.version))) {
log::warn!("file {file}: the previous library version couldn't be deleted: {e}");
}
log::info!("file {file}: published library version {} ({} bytes)", m.version, archive.len());
Ok(m.version)
}
/// A library's usage reports in memory (read from disk the first time; a damaged list starts
/// over: it is only statistics). `None` when the file has no library.
fn usage_of(&mut self, file: &str) -> Result<Option<&mut UsageCache>, String> {
if self.entry(file)?.meta.library.is_none() {
return Ok(None);
}
if !self.usage.contains_key(file) {
let path = self.dir(file).join(USAGE);
let dir = self.dir(file);
let (list, weeks, asset_weeks) = (read_kept(&path), read_kept(&dir.join(ACTIONS)), read_kept(&dir.join(ASSET_ACTIONS)));
self.usage.insert(file.to_string(), UsageCache { list, weeks, asset_weeks, dirty: false, written: now() });
}
Ok(self.usage.get_mut(file))
}
/// The usage reports kept for a file's library (none when it has no library).
pub fn library_usage(&mut self, file: &str) -> Result<Vec<FileUsage>, String> {
let list = self.usage_of(file)?.map(|u| u.list.as_slice()).unwrap_or_default();
Ok(list.iter().filter(|e| e.updated > 0).map(|e| FileUsage { usage: e.usage.clone(), updated: e.updated }).collect())
}
/// The detach reports kept for a file's library (none when it has no library).
pub fn library_detached(&mut self, file: &str) -> Result<Vec<FileDetached>, String> {
let list = self.usage_of(file)?.map(|u| u.list.as_slice()).unwrap_or_default();
Ok(list
.iter()
.filter(|e| !e.detached.is_empty())
.map(|e| FileDetached { detached: Detached { file: e.usage.file.clone(), components: e.detached.clone() }, updated: e.detached_updated })
.collect())
}
/// The style and variable reports kept for a file's library (none when it has no library).
pub fn library_assets(&mut self, file: &str) -> Result<Vec<AssetUsage>, String> {
let list = self.usage_of(file)?.map(|u| u.list.as_slice()).unwrap_or_default();
let used = list.iter().filter(|e| !e.styles.is_empty() || !e.variables.is_empty());
Ok(used.map(|e| AssetUsage { file: e.usage.file.clone(), styles: e.styles.clone(), variables: e.variables.clone() }).collect())
}
/// A file's library's components' insertions and detachments week by week, over the last
/// [`MAX_WEEKS`] weeks (none when it has no library).
pub fn library_actions(&mut self, file: &str) -> Result<Vec<WeekActions>, String> {
let since = week_of(now()).saturating_sub((MAX_WEEKS - 1) * WEEK);
let weeks = self.usage_of(file)?.map(|u| u.weeks.as_slice()).unwrap_or_default();
Ok(weeks.iter().filter(|w| w.week >= since).cloned().collect())
}
/// Keep how one file uses a file's library (in memory; written when it changed, at most every
/// [`USAGE_WRITE_EVERY`] seconds).
pub fn report_usage(&mut self, file: &str, usage: Usage) -> Result<(), String> {
self.report(file, &usage.file, usage.components, |e, components| {
if e.updated > 0 && e.usage.components == components {
return false;
}
e.usage.components = components;
e.updated = now();
true
})
}
/// A file's library's styles' and variables' insertions and detachments week by week, over
/// the last [`MAX_WEEKS`] weeks (none when it has no library).
pub fn library_asset_actions(&mut self, file: &str) -> Result<Vec<WeekActions<String>>, String> {
let since = week_of(now()).saturating_sub((MAX_WEEKS - 1) * WEEK);
let weeks = self.usage_of(file)?.map(|u| u.asset_weeks.as_slice()).unwrap_or_default();
Ok(weeks.iter().filter(|w| w.week >= since).cloned().collect())
}
/// Note how many times one file has inserted and detached a file's library's styles and
/// variables: what its totals grew past their highest counts in this week.
pub fn report_asset_actions(&mut self, file: &str, actions: AssetActions) -> Result<(), String> {
for (counts, kind) in [(actions.inserted, Count::Inserted), (actions.detached, Count::Detached)] {
let mut grown = Vec::new();
self.report(file, &actions.file, known_assets(counts), |e, counts| {
grown = grow(&mut e.asset_peaks, &counts, kind, |_| 0);
if grown.is_empty() {
return false;
}
// Heard from for retention: evicting the row would drop asset_peaks, the
// baseline grow uses so a later identical report is not counted twice.
e.asset_updated = now();
true
})?;
if let (false, Some(u)) = (grown.is_empty(), self.usage.get_mut(file)) {
add_to_week(&mut u.asset_weeks, &grown, kind, now());
u.dirty = true;
}
}
Ok(())
}
/// Keep how many layers of one file use a file's library's styles and variables (as
/// [`Self::report_usage`] keeps its usage, beside it).
pub fn report_assets(&mut self, file: &str, assets: AssetUsage) -> Result<(), String> {
let variables = cleaned(known_assets(assets.variables));
self.report(file, &assets.file, known_assets(assets.styles), |e, styles| {
if e.styles == styles && e.variables == variables {
return false;
}
(e.styles, e.variables) = (styles, variables);
true
})
}
/// Keep how many instances of a file's library components one file has detached (as
/// [`Self::report_usage`] keeps its usage, beside it); what it detached since its last report
/// counts in this week.
pub fn report_detached(&mut self, file: &str, detached: Detached) -> Result<(), String> {
self.report_counts(file, detached, Count::Detached)
}
/// Note how many instances of a file's library components one file has inserted: what it
/// inserted since its last report counts in this week (the totals themselves aren't shown).
pub fn report_inserted(&mut self, file: &str, inserted: Detached) -> Result<(), String> {
self.report_counts(file, inserted, Count::Inserted)
}
fn report_counts(&mut self, file: &str, d: Detached, kind: Count) -> Result<(), String> {
let mut grown = Vec::new();
self.report(file, &d.file, d.components, |e, components| {
grown = e.grow_peaks(&components, kind);
if kind == Count::Inserted || e.detached == components {
return !grown.is_empty();
}
e.detached = components;
e.detached_updated = now();
true
})?;
if let (false, Some(u)) = (grown.is_empty(), self.usage.get_mut(file)) {
add_to_week(&mut u.weeks, &grown, kind, now());
// Written with the reports, at most every USAGE_WRITE_EVERY seconds and at shutdown.
u.dirty = true;
}
Ok(())
}
/// Keep `counts` from the file called `name` ([`cleaned`]) in its entry with `set` (which says whether anything changed),
/// keeping at most [`MAX_USERS`] files, the ones heard from most recently.
fn report<K: Ord>(
&mut self,
file: &str,
name: &str,
counts: Vec<(K, u32)>,
set: impl FnOnce(&mut UsageEntry, Vec<(K, u32)>) -> bool,
) -> Result<(), String> {
let name = clean_name(name);
if name.is_empty() {
return Ok(());
}
let counts = cleaned(counts);
let Some(u) = self.usage_of(file)? else { return Ok(()) };
let at = match u.list.iter().position(|e| e.usage.file == name) {
Some(at) => at,
None => {
u.list.push(UsageEntry { usage: Usage { file: name.clone(), components: Vec::new() }, ..UsageEntry::default() });
u.list.len() - 1
}
};
let Some(entry) = u.list.get_mut(at) else { return Ok(()) };
if !set(entry, counts) {
return Ok(());
}
if u.list.len() > MAX_USERS {
// The file just heard from stays, even when several share a timestamp (now() is
// seconds). Asset-action reports use asset_updated so their peaks are not dropped
// in favour of a usage-only row from the same second.
u.list.sort_by_key(|e| {
let recent = e.usage.file == name;
std::cmp::Reverse((e.updated.max(e.detached_updated).max(e.asset_updated), recent))
});
u.list.truncate(MAX_USERS);
}
u.dirty = true;
if now() >= u.written.saturating_add(USAGE_WRITE_EVERY) {
self.write_usage(false);
}
Ok(())
}
/// Write the usage reports that changed (`all`: whenever they were last written).
pub(super) fn write_usage(&mut self, all: bool) {
let t = now();
let root = self.root.join(FILES);
for (file, u) in &mut self.usage {
if u.dirty && (all || t >= u.written.saturating_add(USAGE_WRITE_EVERY)) {
if let Err(e) = write_json(&root.join(file).join(USAGE), &u.list) {
log::warn!("file {file}: library usage couldn't be kept: {e}");
}
let actions = write_json(&root.join(file).join(ACTIONS), &u.weeks);
if let Err(e) = actions.and_then(|()| write_json(&root.join(file).join(ASSET_ACTIONS), &u.asset_weeks)) {
log::warn!("file {file}: library analytics couldn't be kept: {e}");
}
(u.dirty, u.written) = (false, t);
}
}
}
}
/// The name of the file holding version `version` of a file's team library.
pub fn library_name(version: u64) -> String {
format!("library-{version}.bin")
}
/// Where version `version` of the team library in `dir` is: its own file, or `library.bin` for a
/// version published before each version had one.
fn library_file(dir: &Path, version: u64) -> PathBuf {
let path = dir.join(library_name(version));
if path.exists() { path } else { dir.join(LIBRARY) }
}
#[cfg(test)]
mod grow_tests {
use super::*;
#[test]
fn a_new_key_is_dropped_once_peaks_are_full() {
let mut peaks: Vec<(u32, u32, u32)> = (0..MAX_REPORTED as u32).map(|i| (i, 1, 0)).collect();
let grown = grow(&mut peaks, &[(MAX_REPORTED as u32, 3)], Count::Inserted, |_| 0);
assert!(grown.is_empty(), "a key past the cap must not start a peak or count as growth");
assert_eq!(peaks.len(), MAX_REPORTED);
assert!(peaks.iter().all(|p| p.0 < MAX_REPORTED as u32));
}
}
No file is open. Choose one in the tree, or press Ctrl+O (⌘O on a Mac) to find one by name.
Syntax highlighting could not load. The unmodified read-only text remains available above.