Image caches that find an entry in O(log n), so drawing many small images no longer slows down as the caches fill

Prompt's window had come to spend 35 ms of CPU on every frame after some hours open, most of it in `GlesCanvas::draw_image_data`. Every cache hit looked its key up in the LRU order with `VecDeque::iter().position()` and moved it to the back with `remove`, both linear in the number of entries. A terminal draws each glyph as an image, one per colour it is drawn in, so the cache filled towards its 32 MiB with thousands of small textures, 7573 in the window measured, and every glyph of every frame walked all of them. The software canvas's premultiplied cache and the GLES surface cache did the same.
`render/lru.rs` is a least-recently-used cache bounded by a byte budget. Each entry keeps the tick it was last used at and a `BTreeMap` orders the keys by it, so a hit, an insertion and an eviction cost O(log n). `insert` returns the values it evicted, for the GLES caches to delete their textures, and never evicts the one just inserted, so a value larger than the whole budget is still kept; inserting a key again returns its old value. Tests cover the eviction order, a value over the budget and a replaced key.
`ImageCache` holds its images and its surfaces in one `Lru` each, with the same budgets of 32 and 16 MiB; `ImageCache::image` uploads a texture only when it is not cached, and `evict_overflow` and the two linear lookups are gone. The software canvas keeps its premultiplied images in an `Lru` in place of `PremulCache`. What is cached and evicted is unchanged.

src/gles_render/image.rs

@@ -28,11 +28,10 @@
 //! (`shared_image_cache`), so a surface that is torn down and recreated
 //! draws its images without uploading them again.
 
-use std::collections::hash_map::Entry;
-
 use glow::HasContext;
 
 use crate::render::helpers::fingerprint_rgba;
+use crate::render::lru::Lru;
 use crate::types::Rect;
 
 use super::helpers::{ ortho_rect, upload_rgba_texture };
@@ -49,81 +48,55 @@ type ImageKey = ( u32, u32, u64 );
 /// Content-keyed texture cache shared by every canvas on one GL context.
 pub( super ) struct ImageCache
 {
-	map:   std::collections::HashMap<ImageKey, ( glow::Texture, u32, u32 )>,
-	/// LRU order of keys, least-recent at the front.
-	lru:   std::collections::VecDeque<ImageKey>,
-	/// Estimated GPU bytes held (RGBA8: w × h × 4).
-	bytes: usize,
+	images:   Lru<ImageKey, glow::Texture>,
 	/// Pre-rendered themed surfaces (shadows + fill + insets), keyed by
 	/// size and style; see `surface_cache.rs`.
-	surfaces:       std::collections::HashMap<u64, ( glow::Texture, u32, u32 )>,
-	surfaces_lru:   std::collections::VecDeque<u64>,
-	surfaces_bytes: usize,
+	surfaces: Lru<u64, ( glow::Texture, u32, u32 )>,
 }
 
 /// Byte budget for the pre-rendered surface textures.
 pub const SURFACE_CACHE_MAX_BYTES: usize = 16 * 1024 * 1024;
 
+/// Estimated GPU bytes of an RGBA8 texture.
+fn texture_bytes( w: u32, h: u32 ) -> usize
+{
+	( w as usize ) * ( h as usize ) * 4
+}
+
 impl ImageCache
 {
 	fn new() -> Self
 	{
-		Self
-		{
-			map:            std::collections::HashMap::new(),
-			lru:            std::collections::VecDeque::new(),
-			bytes:          0,
-			surfaces:       std::collections::HashMap::new(),
-			surfaces_lru:   std::collections::VecDeque::new(),
-			surfaces_bytes: 0,
-		}
+		Self { images: Lru::new( IMAGE_CACHE_MAX_BYTES ), surfaces: Lru::new( SURFACE_CACHE_MAX_BYTES ) }
 	}
 
 	pub( super ) fn surface( &mut self, key: u64 ) -> Option<( glow::Texture, u32, u32 )>
 	{
-		let hit = *self.surfaces.get( &key )?;
-		if let Some( pos ) = self.surfaces_lru.iter().position( |k| *k == key )
-		{
-			self.surfaces_lru.remove( pos );
-			self.surfaces_lru.push_back( key );
-		}
-		Some( hit )
+		self.surfaces.get( &key ).copied()
 	}
 
 	pub( super ) fn insert_surface( &mut self, gl: &glow::Context, key: u64, tex: glow::Texture, w: u32, h: u32 )
 	{
-		self.surfaces.insert( key, ( tex, w, h ) );
-		self.surfaces_bytes += ( w as usize ) * ( h as usize ) * 4;
-		self.surfaces_lru.push_back( key );
-		while self.surfaces_bytes > SURFACE_CACHE_MAX_BYTES && self.surfaces_lru.len() > 1
+		for ( old, _, _ ) in self.surfaces.insert( key, ( tex, w, h ), texture_bytes( w, h ) )
 		{
-			let Some( old ) = self.surfaces_lru.pop_front() else { break };
-			if let Some( ( t, ow, oh ) ) = self.surfaces.remove( &old )
-			{
-				self.surfaces_bytes = self.surfaces_bytes.saturating_sub( ( ow as usize ) * ( oh as usize ) * 4 );
-				// SAFETY: `t` was created through `gl` in `end_offscreen` and
-				// just left the map, so it is deleted exactly once.
-				unsafe { gl.delete_texture( t ); }
-			}
+			// SAFETY: `old` was created through `gl` in `end_offscreen` and
+			// just left the cache, so it is deleted exactly once.
+			unsafe { gl.delete_texture( old ); }
 		}
 	}
 
-	/// Delete least-recently-drawn textures until the estimated byte total
-	/// fits [`IMAGE_CACHE_MAX_BYTES`]. The back of the LRU — the entry the
-	/// current draw is about to use — is never removed.
-	fn evict_overflow( &mut self, gl: &glow::Context )
+	/// The texture for `key`, uploaded by `upload` when it is not cached yet.
+	fn image( &mut self, gl: &glow::Context, key: ImageKey, upload: impl FnOnce() -> glow::Texture ) -> glow::Texture
 	{
-		while self.bytes > IMAGE_CACHE_MAX_BYTES && self.lru.len() > 1
+		if let Some( &tex ) = self.images.get( &key ) { return tex; }
+		let tex = upload();
+		for old in self.images.insert( key, tex, texture_bytes( key.0, key.1 ) )
 		{
-			let Some( key ) = self.lru.pop_front() else { break };
-			if let Some( ( tex, w, h ) ) = self.map.remove( &key )
-			{
-				self.bytes = self.bytes.saturating_sub( ( w as usize ) * ( h as usize ) * 4 );
-				// SAFETY: `tex` was created through `gl` in `draw_image_data`
-				// and just left the map, so it is deleted exactly once.
-				unsafe { gl.delete_texture( tex ); }
-			}
+			// SAFETY: `old` was created through `gl` in `draw_image_data`
+			// and just left the cache, so it is deleted exactly once.
+			unsafe { gl.delete_texture( old ); }
 		}
+		tex
 	}
 }
 
@@ -174,21 +147,10 @@ impl GlesCanvas
 		// depth).
 		let cache_key = ( img_w, img_h, fingerprint_rgba( rgba_data ) );
 
-		let cache = std::rc::Rc::clone( &self.image_cache );
-		let mut cache = cache.borrow_mut();
-		let cache     = &mut *cache;
-		if let Entry::Vacant( vacant ) = cache.map.entry( cache_key )
+		let tex = self.image_cache.borrow_mut().image( &self.gl, cache_key, ||
 		{
-			let tex = upload_rgba_texture( &self.gl, self.version, rgba_data, img_w as i32, img_h as i32 );
-			vacant.insert( ( tex, img_w, img_h ) );
-			cache.bytes += ( img_w as usize ) * ( img_h as usize ) * 4;
-			cache.lru.push_back( cache_key );
-			cache.evict_overflow( &self.gl );
-		} else if let Some( pos ) = cache.lru.iter().position( |k| *k == cache_key )
-		{
-			cache.lru.remove( pos );
-			cache.lru.push_back( cache_key );
-		}
+			upload_rgba_texture( &self.gl, self.version, rgba_data, img_w as i32, img_h as i32 )
+		} );
 
 		// Snap to integer pixels. With GL_LINEAR sampling, a
 		// fractional `dest.x` / `dest.y` makes every fragment sample
@@ -205,27 +167,23 @@ impl GlesCanvas
 			height: dest.height.round(),
 		};
 
-		if let Some( ( tex, _, _ ) ) = cache.map.get( &cache_key )
+		let mvp   = ortho_rect( self.width, self.height, dest );
+		let alpha = opacity * self.global_alpha;
+		// SAFETY: see `primitives.rs` module doc. `tex` is owned by the
+		// shared cache so it outlives the call: eviction already ran and
+		// never removes the most recent key, which is `cache_key`.
+		unsafe
 		{
-			let mvp   = ortho_rect( self.width, self.height, dest );
-			let alpha = opacity * self.global_alpha;
-			// SAFETY: see `primitives.rs` module doc. `*tex` is owned by
-			// the shared cache so it outlives the call. Eviction ran
-			// before this borrow and never removes the most-recent key,
-			// which is `cache_key`.
-			unsafe
-			{
-				self.gl.use_program( Some( self.tex_program ) );
-				self.gl.uniform_matrix_4_f32_slice( Some( &self.u_tex_mvp ), false, &mvp );
-				self.gl.uniform_1_f32( Some( &self.u_tex_opacity ), alpha );
-				self.gl.active_texture( glow::TEXTURE0 );
-				self.gl.bind_texture( glow::TEXTURE_2D, Some( *tex ) );
-				self.gl.uniform_1_i32( Some( &self.u_tex_sampler ), 0 );
-				self.gl.bind_vertex_array( Some( self.quad_vao ) );
-				self.gl.draw_arrays( glow::TRIANGLES, 0, 6 );
-				self.gl.bind_vertex_array( None );
-				self.gl.bind_texture( glow::TEXTURE_2D, None );
-			}
+			self.gl.use_program( Some( self.tex_program ) );
+			self.gl.uniform_matrix_4_f32_slice( Some( &self.u_tex_mvp ), false, &mvp );
+			self.gl.uniform_1_f32( Some( &self.u_tex_opacity ), alpha );
+			self.gl.active_texture( glow::TEXTURE0 );
+			self.gl.bind_texture( glow::TEXTURE_2D, Some( tex ) );
+			self.gl.uniform_1_i32( Some( &self.u_tex_sampler ), 0 );
+			self.gl.bind_vertex_array( Some( self.quad_vao ) );
+			self.gl.draw_arrays( glow::TRIANGLES, 0, 6 );
+			self.gl.bind_vertex_array( None );
+			self.gl.bind_texture( glow::TEXTURE_2D, None );
 		}
 	}
 

src/render/image.rs

@@ -14,13 +14,13 @@
 //! of four pixels (Argb8888 fallback).
 
 use std::cell::RefCell;
-use std::collections::{ HashMap, VecDeque };
 
 use tiny_skia::{ IntSize, Pixmap, PixmapPaint, Transform };
 
 use crate::types::Rect;
 
 use super::helpers::fingerprint_rgba;
+use super::lru::Lru;
 use super::SoftwareCanvas;
 
 /// Byte budget of the premultiplied-image cache, matching the GLES texture
@@ -37,47 +37,11 @@ struct Premul
 	opaque: bool,
 }
 
-/// Premultiplied copies of the images drawn recently, keyed by size,
-/// content fingerprint and alpha, so a backdrop or a logo is converted once
-/// rather than on every frame. Least-recently-drawn eviction; the entry
-/// being drawn is never evicted.
-#[ derive( Default ) ]
-struct PremulCache
-{
-	entries: HashMap<PremulKey, Premul>,
-	lru:     VecDeque<PremulKey>,
-	bytes:   usize,
-}
-
 thread_local! {
-	static PREMUL_CACHE: RefCell<PremulCache> = RefCell::new( PremulCache::default() );
-}
-
-impl PremulCache
-{
-	/// Mark `key` as just used. `false` when it is not cached.
-	fn touch( &mut self, key: &PremulKey ) -> bool
-	{
-		let Some( pos ) = self.lru.iter().position( |k| k == key ) else { return false };
-		self.lru.remove( pos );
-		self.lru.push_back( *key );
-		true
-	}
-
-	fn insert( &mut self, key: PremulKey, premul: Premul )
-	{
-		self.bytes += premul.pixmap.data().len();
-		self.entries.insert( key, premul );
-		self.lru.push_back( key );
-		while self.bytes > PREMUL_CACHE_MAX_BYTES && self.lru.len() > 1
-		{
-			let Some( old ) = self.lru.pop_front() else { break };
-			if let Some( p ) = self.entries.remove( &old )
-			{
-				self.bytes = self.bytes.saturating_sub( p.pixmap.data().len() );
-			}
-		}
-	}
+	/// Premultiplied copies of the images drawn recently, keyed by size,
+	/// content fingerprint and alpha, so a backdrop or a logo is converted
+	/// once rather than on every frame.
+	static PREMUL_CACHE: RefCell<Lru<PremulKey, Premul>> = RefCell::new( Lru::new( PREMUL_CACHE_MAX_BYTES ) );
 }
 
 fn premultiply( rgba: &[u8], size: IntSize, alpha: f32 ) -> Option<Premul>
@@ -144,12 +108,13 @@ impl SoftwareCanvas
 		PREMUL_CACHE.with( |cell|
 		{
 			let mut cache = cell.borrow_mut();
-			if !cache.touch( &key )
+			if cache.get( &key ).is_none()
 			{
 				let Some( premul ) = premultiply( rgba_data, int_size, alpha ) else { return };
-				cache.insert( key, premul );
+				let bytes = premul.pixmap.data().len();
+				cache.insert( key, premul, bytes );
 			}
-			let Some( src ) = cache.entries.get( &key ) else { return };
+			let Some( src ) = cache.get( &key ) else { return };
 
 			let unscaled = dest.width as u32 == img_w && dest.height as u32 == img_h;
 			if src.opaque && unscaled && self.clip_mask.is_none()

src/render/lru.rs

@@ -0,0 +1,110 @@
+// SPDX-License-Identifier: LGPL-2.1-only
+// Copyright (C) 2026 Liberux Labs, S. L. <info@liberux.net>
+
+//! A least-recently-used cache bounded by a byte budget, whose hits, insertions and evictions cost O(log n).
+
+use std::collections::{ BTreeMap, HashMap };
+use std::hash::Hash;
+
+struct Slot<V>
+{
+	value: V,
+	bytes: usize,
+	used:  u64,
+}
+
+pub( crate ) struct Lru<K, V>
+{
+	slots:  HashMap<K, Slot<V>>,
+	/// Keys by the tick they were last used at, least recent first.
+	order:  BTreeMap<u64, K>,
+	tick:   u64,
+	bytes:  usize,
+	budget: usize,
+}
+
+impl<K: Copy + Eq + Hash, V> Lru<K, V>
+{
+	pub( crate ) fn new( budget: usize ) -> Self
+	{
+		Self { slots: HashMap::new(), order: BTreeMap::new(), tick: 0, bytes: 0, budget }
+	}
+
+	/// The value under `key`, which becomes the most recently used.
+	pub( crate ) fn get( &mut self, key: &K ) -> Option<&V>
+	{
+		let slot = self.slots.get_mut( key )?;
+		self.order.remove( &slot.used );
+		self.tick += 1;
+		slot.used = self.tick;
+		self.order.insert( self.tick, *key );
+		Some( &slot.value )
+	}
+
+	/// Store `value` as the most recently used and return the values that left to fit the budget; `value` itself never does, so one larger than the whole budget is still kept.
+	pub( crate ) fn insert( &mut self, key: K, value: V, bytes: usize ) -> Vec<V>
+	{
+		let mut evicted: Vec<V> = self.remove( &key ).into_iter().collect();
+		self.tick += 1;
+		self.order.insert( self.tick, key );
+		self.slots.insert( key, Slot { value, bytes, used: self.tick } );
+		self.bytes += bytes;
+		while self.bytes > self.budget && self.slots.len() > 1
+		{
+			let Some( ( _, oldest ) ) = self.order.pop_first() else { break };
+			if let Some( slot ) = self.slots.remove( &oldest )
+			{
+				self.bytes -= slot.bytes;
+				evicted.push( slot.value );
+			}
+		}
+		evicted
+	}
+
+	fn remove( &mut self, key: &K ) -> Option<V>
+	{
+		let slot = self.slots.remove( key )?;
+		self.order.remove( &slot.used );
+		self.bytes -= slot.bytes;
+		Some( slot.value )
+	}
+}
+
+#[ cfg( test ) ]
+mod tests
+{
+	use super::Lru;
+
+	#[ test ]
+	fn evicts_the_least_recently_used()
+	{
+		let mut lru = Lru::new( 30 );
+		assert!( lru.insert( 1, 'a', 10 ).is_empty() );
+		assert!( lru.insert( 2, 'b', 10 ).is_empty() );
+		assert!( lru.insert( 3, 'c', 10 ).is_empty() );
+		assert_eq!( lru.get( &1 ), Some( &'a' ) );
+		assert_eq!( lru.insert( 4, 'd', 10 ), vec![ 'b' ] );
+		assert_eq!( lru.get( &2 ), None );
+		assert_eq!( lru.get( &1 ), Some( &'a' ) );
+		assert_eq!( lru.get( &3 ), Some( &'c' ) );
+	}
+
+	#[ test ]
+	fn keeps_a_value_larger_than_the_budget()
+	{
+		let mut lru = Lru::new( 10 );
+		lru.insert( 1, 'a', 5 );
+		assert_eq!( lru.insert( 2, 'b', 50 ), vec![ 'a' ] );
+		assert_eq!( lru.get( &2 ), Some( &'b' ) );
+	}
+
+	#[ test ]
+	fn replacing_a_key_returns_the_old_value()
+	{
+		let mut lru = Lru::new( 100 );
+		lru.insert( 1, 'a', 10 );
+		assert_eq!( lru.insert( 1, 'z', 10 ), vec![ 'a' ] );
+		assert_eq!( lru.get( &1 ), Some( &'z' ) );
+		assert_eq!( lru.bytes, 10 );
+	}
+}

src/render/mod.rs

@@ -47,6 +47,7 @@ pub( crate ) mod primitives;
 pub( crate ) mod text;
 pub( crate ) mod image;
 pub( crate ) mod helpers;
+pub( crate ) mod lru;
 
 // ─── Backend flag ────────────────────────────────────────────────────────────