ZJIT: Box the rarely-used profile side tables

IseqProfile holds two HashMaps -- super_cme for invokesuper receivers and
splat_lengths for splat call sites -- that only ISEQs containing those
instructions ever populate. An empty std HashMap allocates nothing but
still occupies 48 bytes inline, so every payload paid 96 bytes to hold
two empty tables. ZJIT allocates a payload for every ISEQ it merely
profiles, which on an RDoc-over-stdlib workload was 28,614 payloads.

Wrap both in Option<Box<_>>, taking IseqPayload from 152 to 72 bytes.
mem_iseq_payload_bytes drops from 2.36 MB to 1.12 MB and the payloads
still retained for freed ISEQs (mem_unaccounted_bytes) from 2.03 MB to
0.98 MB.

Adapted for zjit/min: this branch has six side tables, not two, so all six are
boxed -- `send_mid`, `forwarded_cis`, `block_handlers` and `block_fallbacks`
alongside `super_cme` and `splat_lengths`. That is 288 bytes of empty
`HashMap` headers per payload rather than 96. The mutable accessors this branch
already routes every write through (`super_cme_mut()` and friends) become the
`get_or_insert_with` sites, so no caller changes; the readers take `as_ref()?`
and the object walks iterate `iter().flat_map(...)`.

Author: k0kubun

zjit/src/profile.rs

@@ -973,28 +973,32 @@ pub struct IseqProfile {
     /// Only instructions that have actually been profiled have entries here.
     entries: Vec<ProfileEntry>,
 
-    /// Method entries for `super` calls (stored as VALUE to be GC-safe)
-    super_cme: HashMap<YarvInsnIdx, TypeDistribution>,
+    /// Method entries for `super` calls (stored as VALUE to be GC-safe).
+    /// Boxed because only ISEQs containing `invokesuper` ever use it, and an
+    /// inline `HashMap` costs every payload 48 bytes to hold nothing.
+    super_cme: Option<Box<HashMap<YarvInsnIdx, TypeDistribution>>>,
 
     /// Method-name symbols observed as the first argument of `send`/`__send__` call sites
     /// (stored as VALUE to be GC-safe)
-    send_mid: HashMap<YarvInsnIdx, TypeDistribution>,
+    send_mid: Option<Box<HashMap<YarvInsnIdx, TypeDistribution>>>,
 
-    /// Observed lengths of caller splat arrays for call instructions.
-    splat_lengths: HashMap<YarvInsnIdx, SplatLengthDistribution>,
+    /// Observed lengths of caller splat arrays for call instructions. Boxed for
+    /// the same reason as `super_cme`.
+    splat_lengths: Option<Box<HashMap<YarvInsnIdx, SplatLengthDistribution>>>,
+
+    /// Callinfos observed in the `...` local at `sendforward` sites. Unlike the tables above,
+    /// this one holds no objects: see `profile_forwarded_callinfo`.
+    forwarded_cis: Option<Box<HashMap<YarvInsnIdx, ForwardedCiDistribution>>>,
 
     /// Block handlers observed at `invokeblock` sites (stored as VALUE to be GC-safe).
     /// Kept out of `opnd_types` so that the entry's operand slots profile the yielded
     /// arguments instead: a `yield` to a Symbol block sends to the first argument, and
     /// that send only specializes if the argument's class was profiled.
-    block_handlers: HashMap<YarvInsnIdx, TypeDistribution>,
-    /// Callinfos observed in the `...` local at `sendforward` sites. Unlike the tables above,
-    /// this one holds no objects: see `profile_forwarded_callinfo`.
-    forwarded_cis: HashMap<YarvInsnIdx, ForwardedCiDistribution>,
+    block_handlers: Option<Box<HashMap<YarvInsnIdx, TypeDistribution>>>,
 
     /// Live-traffic re-profiling windows for `invokeblock` sites whose compiled dispatch keeps
     /// reaching `rb_vm_invokeblock()`.
-    block_fallbacks: HashMap<YarvInsnIdx, BlockFallbackEntry>,
+    block_fallbacks: Option<Box<HashMap<YarvInsnIdx, BlockFallbackEntry>>>,
 
     /// Dense copy of every object the distributions above reference, which is what
     /// GC marking actually walks. See [`IseqProfile::marked_objects`].
@@ -1017,12 +1021,12 @@ impl IseqProfile {
     pub fn new() -> Self {
         Self {
             entries: Vec::new(),
-            super_cme: HashMap::new(),
-            send_mid: HashMap::new(),
-            splat_lengths: HashMap::new(),
-            block_handlers: HashMap::new(),
-            forwarded_cis: HashMap::new(),
-            block_fallbacks: HashMap::new(),
+            super_cme: None,
+            send_mid: None,
+            splat_lengths: None,
+            forwarded_cis: None,
+            block_handlers: None,
+            block_fallbacks: None,
             marked_objects: Vec::new(),
             // Nothing recorded yet, so the (empty) dense copy is already accurate.
             marked_objects_stale: false,
@@ -1053,47 +1057,47 @@ impl IseqProfile {
         }
     }
 
-    /// Mutable access to the forwarded callinfo distributions. Needs no marking invalidation
-    /// because a packed callinfo is an immediate, not an object, and a heap one is recorded as
-    /// `None`.
-    fn forwarded_cis_mut(&mut self) -> &mut HashMap<YarvInsnIdx, ForwardedCiDistribution> {
-        &mut self.forwarded_cis
-    }
-
     /// Mutable access to the `invokesuper` method-entry distributions. Goes through
     /// here so that recording a CME cannot skip invalidating the dense copy GC
     /// marking walks.
     fn super_cme_mut(&mut self) -> &mut HashMap<YarvInsnIdx, TypeDistribution> {
         self.marked_objects_stale = true;
-        &mut self.super_cme
+        self.super_cme.get_or_insert_with(Default::default)
     }
 
     /// Mutable access to the `send`/`__send__` method-name distributions. Same reason
     /// as [`Self::super_cme_mut`].
     fn send_mid_mut(&mut self) -> &mut HashMap<YarvInsnIdx, TypeDistribution> {
         self.marked_objects_stale = true;
-        &mut self.send_mid
+        self.send_mid.get_or_insert_with(Default::default)
     }
 
     /// Mutable access to the splat length distributions. Unlike the two above this
     /// needs no invalidation: a `SplatLengthDistribution` holds array lengths, not
     /// objects, so GC marking never looks at it.
     fn splat_lengths_mut(&mut self) -> &mut HashMap<YarvInsnIdx, SplatLengthDistribution> {
-        &mut self.splat_lengths
+        self.splat_lengths.get_or_insert_with(Default::default)
+    }
+
+    /// Mutable access to the forwarded callinfo distributions. Needs no marking invalidation
+    /// for the same reason `splat_lengths_mut` does not: a packed callinfo is an immediate, not
+    /// an object, and a heap one is recorded as `None`.
+    fn forwarded_cis_mut(&mut self) -> &mut HashMap<YarvInsnIdx, ForwardedCiDistribution> {
+        self.forwarded_cis.get_or_insert_with(Default::default)
     }
 
     /// Mutable access to the `invokeblock` block-handler distributions. Same reason
     /// as [`Self::super_cme_mut`].
     fn block_handlers_mut(&mut self) -> &mut HashMap<YarvInsnIdx, TypeDistribution> {
         self.marked_objects_stale = true;
-        &mut self.block_handlers
+        self.block_handlers.get_or_insert_with(Default::default)
     }
 
     /// Mutable access to the `invokeblock` re-profiling windows. Same reason as
     /// [`Self::super_cme_mut`]: the windows hold block handlers, which are objects.
     fn block_fallbacks_mut(&mut self) -> &mut HashMap<YarvInsnIdx, BlockFallbackEntry> {
         self.marked_objects_stale = true;
-        &mut self.block_fallbacks
+        self.block_fallbacks.get_or_insert_with(Default::default)
     }
 
     /// Record one block handler that reached a compiled `yield`'s generic fallback, and say
@@ -1206,7 +1210,7 @@ impl IseqProfile {
     }
 
     pub fn get_splat_length_summary(&self, insn_idx: YarvInsnIdx) -> Option<SplatLengthDistributionSummary> {
-        self.splat_lengths.get(&insn_idx)
+        self.splat_lengths.as_ref()?.get(&insn_idx)
             .map(SplatLengthDistributionSummary::new)
     }
 
@@ -1220,29 +1224,29 @@ impl IseqProfile {
         }
     }
 
-    /// The distribution of block handlers observed at an `invokeblock` site.
-    pub fn get_block_handlers(&self, insn_idx: YarvInsnIdx) -> Option<TypeDistributionSummary> {
-        self.block_handlers.get(&insn_idx).map(TypeDistributionSummary::new)
-    }
-
-    /// The distribution of callinfos a `sendforward` site was seen forwarding. See
-    /// [`profile_forwarded_callinfo`].
-    pub fn get_forwarded_callinfos(&self, insn_idx: YarvInsnIdx) -> Option<ForwardedCiDistributionSummary> {
-        self.forwarded_cis.get(&insn_idx).map(ForwardedCiDistributionSummary::new)
-    }
-
     /// The whole distribution of frame method entries seen at an `invokesuper` site. A site with
     /// more than one is a `super` inside a method body that several classes run, most often a
     /// module method reached through more than one includer; each such method entry resolves
     /// `super` to a different target.
     pub fn get_super_method_entries(&self, insn_idx: YarvInsnIdx) -> Option<TypeDistributionSummary> {
-        let entry = self.super_cme.get(&insn_idx)?;
+        let entry = self.super_cme.as_ref()?.get(&insn_idx)?;
         Some(TypeDistributionSummary::new(entry))
     }
 
     /// Get the distribution of method-name symbols seen at a `send`/`__send__` call site.
     pub fn get_send_method_names(&self, insn_idx: YarvInsnIdx) -> Option<TypeDistributionSummary> {
-        self.send_mid.get(&insn_idx).map(TypeDistributionSummary::new)
+        self.send_mid.as_ref()?.get(&insn_idx).map(TypeDistributionSummary::new)
+    }
+
+    /// The distribution of callinfos a `sendforward` site was seen forwarding. See
+    /// [`profile_forwarded_callinfo`].
+    pub fn get_forwarded_callinfos(&self, insn_idx: YarvInsnIdx) -> Option<ForwardedCiDistributionSummary> {
+        self.forwarded_cis.as_ref()?.get(&insn_idx).map(ForwardedCiDistributionSummary::new)
+    }
+
+    /// The distribution of block handlers observed at an `invokeblock` site.
+    pub fn get_block_handlers(&self, insn_idx: YarvInsnIdx) -> Option<TypeDistributionSummary> {
+        self.block_handlers.as_ref()?.get(&insn_idx).map(TypeDistributionSummary::new)
     }
 
     /// Bytes this profile owns on the Rust heap, excluding the `IseqProfile`
@@ -1265,20 +1269,38 @@ impl IseqProfile {
                 }
             }
         }
-        out.bytes += hash_table_bytes::<(YarvInsnIdx, TypeDistribution)>(self.super_cme.capacity());
-        out.bytes += hash_table_bytes::<(YarvInsnIdx, TypeDistribution)>(self.send_mid.capacity());
-        out.bytes += hash_table_bytes::<(YarvInsnIdx, SplatLengthDistribution)>(self.splat_lengths.capacity());
-        out.bytes += hash_table_bytes::<(YarvInsnIdx, ForwardedCiDistribution)>(self.forwarded_cis.capacity());
-        out.bytes += hash_table_bytes::<(YarvInsnIdx, TypeDistribution)>(self.block_handlers.capacity());
-        out.bytes += hash_table_bytes::<(YarvInsnIdx, BlockFallbackEntry)>(self.block_fallbacks.capacity());
-        // Boxed distribution tails, wherever they live.
-        out.bytes += self.super_cme.values().map(TypeDistribution::heap_size).sum::<usize>();
-        out.bytes += self.send_mid.values().map(TypeDistribution::heap_size).sum::<usize>();
-        out.bytes += self.splat_lengths.values().map(SplatLengthDistribution::heap_size).sum::<usize>();
-        out.bytes += self.forwarded_cis.values().map(ForwardedCiDistribution::heap_size).sum::<usize>();
-        out.bytes += self.block_handlers.values().map(TypeDistribution::heap_size).sum::<usize>();
-        out.bytes += self.block_fallbacks.values()
-            .map(|entry| entry.dist.heap_size() + entry.symbol_recv.heap_size()).sum::<usize>();
+        // The boxed side tables: the `HashMap` header itself once boxed, the bucket
+        // array, and each distribution's own boxed tail.
+        if let Some(table) = self.super_cme.as_ref() {
+            out.bytes += size_of::<HashMap<YarvInsnIdx, TypeDistribution>>()
+                + hash_table_bytes::<(YarvInsnIdx, TypeDistribution)>(table.capacity())
+                + table.values().map(TypeDistribution::heap_size).sum::<usize>();
+        }
+        if let Some(table) = self.send_mid.as_ref() {
+            out.bytes += size_of::<HashMap<YarvInsnIdx, TypeDistribution>>()
+                + hash_table_bytes::<(YarvInsnIdx, TypeDistribution)>(table.capacity())
+                + table.values().map(TypeDistribution::heap_size).sum::<usize>();
+        }
+        if let Some(table) = self.splat_lengths.as_ref() {
+            out.bytes += size_of::<HashMap<YarvInsnIdx, SplatLengthDistribution>>()
+                + hash_table_bytes::<(YarvInsnIdx, SplatLengthDistribution)>(table.capacity())
+                + table.values().map(SplatLengthDistribution::heap_size).sum::<usize>();
+        }
+        if let Some(table) = self.forwarded_cis.as_ref() {
+            out.bytes += size_of::<HashMap<YarvInsnIdx, ForwardedCiDistribution>>()
+                + hash_table_bytes::<(YarvInsnIdx, ForwardedCiDistribution)>(table.capacity())
+                + table.values().map(ForwardedCiDistribution::heap_size).sum::<usize>();
+        }
+        if let Some(table) = self.block_handlers.as_ref() {
+            out.bytes += size_of::<HashMap<YarvInsnIdx, TypeDistribution>>()
+                + hash_table_bytes::<(YarvInsnIdx, TypeDistribution)>(table.capacity())
+                + table.values().map(TypeDistribution::heap_size).sum::<usize>();
+        }
+        if let Some(table) = self.block_fallbacks.as_ref() {
+            out.bytes += size_of::<HashMap<YarvInsnIdx, BlockFallbackEntry>>()
+                + hash_table_bytes::<(YarvInsnIdx, BlockFallbackEntry)>(table.capacity())
+                + table.values().map(|entry| entry.dist.heap_size() + entry.symbol_recv.heap_size()).sum::<usize>();
+        }
         out
     }
 
@@ -1294,25 +1316,25 @@ impl IseqProfile {
             }
         }
 
-        for super_cme_values in self.super_cme.values() {
+        for super_cme_values in self.super_cme.iter().flat_map(|map| map.values()) {
             for profiled_type in super_cme_values.each_item() {
                 callback(profiled_type.class)
             }
         }
 
-        for handler_values in self.block_handlers.values() {
-            for profiled_type in handler_values.each_item() {
+        for send_mid_values in self.send_mid.iter().flat_map(|map| map.values()) {
+            for profiled_type in send_mid_values.each_item() {
                 callback(profiled_type.class)
             }
         }
 
-        for send_mid_values in self.send_mid.values() {
-            for profiled_type in send_mid_values.each_item() {
+        for handler_values in self.block_handlers.iter().flat_map(|map| map.values()) {
+            for profiled_type in handler_values.each_item() {
                 callback(profiled_type.class)
             }
         }
 
-        for fallback in self.block_fallbacks.values() {
+        for fallback in self.block_fallbacks.iter().flat_map(|map| map.values()) {
             for profiled_type in fallback.dist.each_item().chain(fallback.symbol_recv.each_item()) {
                 callback(profiled_type.class)
             }
@@ -1383,25 +1405,25 @@ impl IseqProfile {
         }
 
         // Update CME references if they move during compaction.
-        for super_cme_values in self.super_cme.values_mut() {
+        for super_cme_values in self.super_cme.iter_mut().flat_map(|map| map.values_mut()) {
             for ref mut profiled_type in super_cme_values.each_item_mut() {
                 callback(&mut profiled_type.class)
             }
         }
 
-        for handler_values in self.block_handlers.values_mut() {
-            for ref mut profiled_type in handler_values.each_item_mut() {
+        for send_mid_values in self.send_mid.iter_mut().flat_map(|map| map.values_mut()) {
+            for ref mut profiled_type in send_mid_values.each_item_mut() {
                 callback(&mut profiled_type.class)
             }
         }
 
-        for send_mid_values in self.send_mid.values_mut() {
-            for ref mut profiled_type in send_mid_values.each_item_mut() {
+        for handler_values in self.block_handlers.iter_mut().flat_map(|map| map.values_mut()) {
+            for ref mut profiled_type in handler_values.each_item_mut() {
                 callback(&mut profiled_type.class)
             }
         }
 
-        for fallback in self.block_fallbacks.values_mut() {
+        for fallback in self.block_fallbacks.iter_mut().flat_map(|map| map.values_mut()) {
             for ref mut profiled_type in fallback.dist.each_item_mut() {
                 callback(&mut profiled_type.class)
             }