Increase the size of the bitmaps

The for loop can be unrolled/make use of intrinsics later.

Author: jvoisin

calculate-waste

@@ -97,13 +97,13 @@ for size, slots, fragmentation in zip(size_classes, size_class_slots, fragmentat
         end=" |\n",
     )
 
-max_bits = 256
+max_bits = 512
 max_page_span = 16
 
 print()
 
 print("maximum bitmap size is {}-bit".format(max_bits))
-print( "maximum page span size is {} ({})".format(max_page_span, max_page_span * page_size))
+print("maximum page span size is {} ({})".format(max_page_span, max_page_span * page_size))
 
 for size_class in size_classes:
     choices = []

h_malloc.c

@@ -115,7 +115,7 @@ static bool memory_map_fixed_tagged(void *ptr, size_t size) {
 #define SLAB_METADATA_COUNT
 
 struct slab_metadata {
-    u64 bitmap[4];
+    u64 bitmap[8];
     struct slab_metadata *next;
     struct slab_metadata *prev;
 #if SLAB_CANARY
@@ -125,19 +125,19 @@ struct slab_metadata {
     u16 count;
 #endif
 #if SLAB_QUARANTINE
-    u64 quarantine_bitmap[4];
+    u64 quarantine_bitmap[8];
 #endif
 #ifdef HAS_ARM_MTE
     // arm_mte_tags is used as a u4 array (MTE tags are 4-bit wide)
     //
     // Its size is calculated by the following formula:
     // (MAX_SLAB_SLOT_COUNT + 2) / 2
-    // MAX_SLAB_SLOT_COUNT is currently 256, 2 extra slots are needed for branchless handling of
+    // MAX_SLAB_SLOT_COUNT is currently 512, 2 extra slots are needed for branchless handling of
     // edge slots in tag_and_clear_slab_slot()
     //
     // It's intentionally placed at the end of struct to improve locality: for most size classes,
     // slot count is far lower than MAX_SLAB_SLOT_COUNT.
-    u8 arm_mte_tags[129];
+    u8 arm_mte_tags[257];
 #endif
 };
 
@@ -460,20 +460,14 @@ static bool has_free_slots(size_t slots, const struct slab_metadata *metadata) {
 #ifdef SLAB_METADATA_COUNT
     return metadata->count < slots;
 #else
-    if (slots <= U64_WIDTH) {
-        u64 masked = metadata->bitmap[0] | get_mask(slots);
-        return masked != ~0UL;
-    }
-    if (slots <= U64_WIDTH * 2) {
-        u64 masked = metadata->bitmap[1] | get_mask(slots - U64_WIDTH);
-        return metadata->bitmap[0] != ~0UL || masked != ~0UL;
-    }
-    if (slots <= U64_WIDTH * 3) {
-        u64 masked = metadata->bitmap[2] | get_mask(slots - U64_WIDTH * 2);
-        return metadata->bitmap[0] != ~0UL || metadata->bitmap[1] != ~0UL || masked != ~0UL;
+    size_t last = (slots - 1) / U64_WIDTH;
+    for (size_t i = 0; i < last; i++) {
+        if (metadata->bitmap[i] != ~0UL) {
+            return true;
+        }
     }
-    u64 masked = metadata->bitmap[3] | get_mask(slots - U64_WIDTH * 3);
-    return metadata->bitmap[0] != ~0UL || metadata->bitmap[1] != ~0UL || metadata->bitmap[2] != ~0UL || masked != ~0UL;
+    u64 masked = metadata->bitmap[last] | get_mask(slots - last * U64_WIDTH);
+    return masked != ~0UL;
 #endif
 }
 
@@ -482,7 +476,8 @@ static bool is_free_slab(const struct slab_metadata *metadata) {
     return !metadata->count;
 #else
     return !metadata->bitmap[0] && !metadata->bitmap[1] && !metadata->bitmap[2] &&
-        !metadata->bitmap[3];
+        !metadata->bitmap[3] && !metadata->bitmap[4] && !metadata->bitmap[5] &&
+        !metadata->bitmap[6] && !metadata->bitmap[7];
 #endif
 }