Narrow bounds on kernel malloc allocations.

sys/kern/kern_malloc.c

@@ -110,7 +110,7 @@
 #define	MALLOC_DEBUG	1
 #endif
 
-#if defined(KASAN) || defined(DEBUG_REDZONE)
+#if defined(KASAN) || defined(DEBUG_REDZONE) || defined(__CHERI_PURE_CAPABILITY__)
 #define	DEBUG_REDZONE_ARG_DEF	, unsigned long osize
 #define	DEBUG_REDZONE_ARG	, osize
 #else
@@ -633,14 +633,24 @@
 	size = roundup(size, PAGE_SIZE);
 	va = kmem_malloc_domainset(policy, size, flags);
 	if (va != NULL) {
-		/* Use low bits unused for slab pointers. */
-		vsetzoneslab((uintptr_t)va, NULL, MALLOC_LARGE_SLAB(size));
+		/*
+		 * Use low bits unused for slab pointers.
+		 * XXX-AM: Abuse the zone pointer to stash the original pointer.
+		 * On CHERI systems, this is necessary to recover the bounds of
+		 * the original allocation.
+		 */
+		vsetzoneslab((uintptr_t)va, va, MALLOC_LARGE_SLAB(size));
 		uma_total_inc(size);
 #ifdef __CHERI_PURE_CAPABILITY__
-		KASSERT(cheri_getlen(va) <= CHERI_REPRESENTABLE_LENGTH(size),
+		va = cheri_setbounds(va, osize);
+		KASSERT(cheri_getlen(va) <= CHERI_REPRESENTABLE_LENGTH(osize),
 		    ("Invalid bounds: expected %zx found %zx",
-		        (size_t)CHERI_REPRESENTABLE_LENGTH(size),
+		        (size_t)CHERI_REPRESENTABLE_LENGTH(osize),
 		        (size_t)cheri_getlen(va)));
+		if ((flags & M_ZERO) == 0 && osize < cheri_getlen(va)) {
+			bzero((void *)((uintptr_t)va + osize),
+			    cheri_getlen(va) - osize);
+		}
 #endif
 	}
 	malloc_type_allocated(mtp, va, va == NULL ? 0 : size);
@@ -659,10 +669,32 @@
 static void
 free_large(void *addr, size_t size)
 {
-
 	kmem_free(addr, size);
 	uma_total_dec(size);
 }
+
+#ifdef __CHERI_PURE_CAPABILITY__
+/*
+ * Recover original bounds for a malloc_large allocation.
+ *
+ * Error conditions:
+ * - Clear the capability tag if the given capability is not a subset of
+ * the saved object capability.
+ */
+static void *
+malloc_large_grow_bounds(void *saved_ptr, void *addr)
+{
+	KASSERT(cheri_is_subset(saved_ptr, addr),
+	    ("Unexpected malloc_large grow bounds: pointer %#p is "
+		"not derived from %#p", addr, saved_ptr));
+
+	addr = cheri_setaddress(saved_ptr, (vm_offset_t)addr);
+	addr = cheri_setboundsexact(addr, cheri_getlen(saved_ptr));
+	KASSERT(cheri_gettag(addr),
+	    ("Failed to recover malloc_large bounds for %#p", addr));
+	return (addr);
+}
+#endif
 #undef	IS_MALLOC_LARGE
 #undef	MALLOC_LARGE_SLAB
 
@@ -680,7 +712,7 @@
 	int indx;
 	caddr_t va;
 	uma_zone_t zone;
-#if defined(DEBUG_REDZONE) || defined(KASAN)
+#if defined(DEBUG_REDZONE) || defined(KASAN) || defined(__CHERI_PURE_CAPABILITY__)
 	unsigned long osize = size;
 #endif
 
@@ -722,10 +754,17 @@
 		kasan_mark((void *)va, osize, size, KASAN_MALLOC_REDZONE);
 #endif
 #ifdef __CHERI_PURE_CAPABILITY__
-	KASSERT(cheri_getlen(va) <= CHERI_REPRESENTABLE_LENGTH(size),
-	    ("Invalid bounds: expected %zx found %zx",
-	        (size_t)CHERI_REPRESENTABLE_LENGTH(size),
-	        (size_t)cheri_getlen(va)));
+	/* Intentionally inexect bounds allow for non-representable sizes */
+	va = cheri_setbounds(va, osize);
+	KASSERT(cheri_gettag(va),
+	    ("Invalid malloc: %#p requested size %zx", va, osize));
+	KASSERT(cheri_getlen(va) <= CHERI_REPRESENTABLE_LENGTH(osize),
+	    ("Invalid malloc: %#p expected length %zx", va,
+		(size_t)CHERI_REPRESENTABLE_LENGTH(osize)));
+	if (va != NULL && (flags & M_ZERO) == 0 && osize < cheri_getlen(va)) {
+		bzero((void *)((uintptr_t)va + osize),
+		    cheri_getlen(va) - osize);
+	}
 #endif
 	return ((void *) va);
 }
@@ -765,7 +804,7 @@
 	caddr_t va;
 	int domain;
 	int indx;
-#if defined(KASAN) || defined(DEBUG_REDZONE)
+#if defined(KASAN) || defined(DEBUG_REDZONE) || defined(__CHERI_PURE_CAPABILITY__)
 	unsigned long osize = size;
 #endif
 
@@ -781,6 +820,11 @@
 		return (malloc_large(size, mtp, DOMAINSET_RR(), flags
 		    DEBUG_REDZONE_ARG));
 
+	/* XXX-AM: see malloc() */
+	if (size != CHERI_REPRESENTABLE_LENGTH(size)) {
+		flags |= M_ZERO;
+	}
+
 	vm_domainset_iter_policy_init(&di, ds, &domain, &flags);
 	do {
 		va = malloc_domain(&size, &indx, mtp, domain, flags);
@@ -803,6 +847,15 @@
 		kmsan_mark(va, size, KMSAN_STATE_UNINIT);
 		kmsan_orig(va, size, KMSAN_TYPE_MALLOC, KMSAN_RET_ADDR);
 	}
+#endif
+#ifdef __CHERI_PURE_CAPABILITY__
+	/* Intentionally inexect bounds allow for non-representable sizes */
+	va = cheri_setbounds(va, osize);
+	KASSERT(cheri_gettag(va),
+	    ("Invalid malloc: %#p requested size %zx", va, osize));
+	KASSERT(cheri_getlen(va) == CHERI_REPRESENTABLE_LENGTH(osize),
+	    ("Invalid malloc: %#p expected length %zx", va,
+		(size_t)CHERI_REPRESENTABLE_LENGTH(osize)));
 #endif
 	return (va);
 }
@@ -821,7 +874,7 @@
 malloc_domainset_exec(size_t size, struct malloc_type *mtp, struct domainset *ds,
     int flags)
 {
-#if defined(DEBUG_REDZONE) || defined(KASAN)
+#if defined(DEBUG_REDZONE) || defined(KASAN) || defined(__CHERI_PURE_CAPABILITY__)
 	unsigned long osize = size;
 #endif
 #ifdef MALLOC_DEBUG
@@ -996,11 +1049,10 @@
 	case __predict_true(SLAB_COOKIE_SLAB_PTR):
 		size = zone->uz_size;
 #ifdef __CHERI_PURE_CAPABILITY__
-		if (__predict_false(cheri_getlen(addr) !=
-		    CHERI_REPRESENTABLE_LENGTH(size)))
-			panic("Invalid bounds: expected %zx found %zx",
-			    (size_t)CHERI_REPRESENTABLE_LENGTH(size),
-			    cheri_getlen(addr));
+		addr = uma_zgrow_bounds(zone, addr);
+		KASSERT(cheri_getlen(addr) == size,
+		    ("vtozoneslab disagrees with uma_zgrow_bounds: %zx != %zx",
+			cheri_getlen(addr), size));
 #endif
 #if defined(INVARIANTS) && !defined(KASAN)
 		free_save_type(addr, mtp, size);
@@ -1014,11 +1066,10 @@
 	case SLAB_COOKIE_MALLOC_LARGE:
 		size = malloc_large_size(slab);
 #ifdef __CHERI_PURE_CAPABILITY__
-		if (__predict_false(cheri_getlen(addr) !=
-		    CHERI_REPRESENTABLE_LENGTH(size)))
-			panic("Invalid bounds: expected %zx found %zx",
-			    (size_t)CHERI_REPRESENTABLE_LENGTH(size),
-			    cheri_getlen(addr));
+		addr = malloc_large_grow_bounds(zone, addr);
+		KASSERT(cheri_getlen(addr) == size,
+		    ("malloc_large object bounds mismatch: %zx != %zx",
+			cheri_getlen(addr), size));
 #endif
 		if (dozero) {
 			kasan_mark(addr, size, size, 0);
@@ -1074,6 +1125,9 @@
 #endif
 	unsigned long alloc;
 	void *newaddr;
+#ifdef __CHERI_PURE_CAPABILITY__
+	size_t olength;
+#endif
 
 	KASSERT(mtp->ks_version == M_VERSION,
 	    ("realloc: bad malloc type version"));
@@ -1088,6 +1142,7 @@
 		panic("Expect valid capability");
 	if (__predict_false(cheri_getsealed(addr)))
 		panic("Expect unsealed capability");
+	olength = cheri_getlen(addr);
 #endif
 
 	/*
@@ -1113,9 +1168,25 @@
 	switch (GET_SLAB_COOKIE(slab)) {
 	case __predict_true(SLAB_COOKIE_SLAB_PTR):
 		alloc = zone->uz_size;
+#ifdef __CHERI_PURE_CAPABILITY__
+		if (size > olength) {
+			addr = uma_zgrow_bounds(zone, addr);
+			KASSERT(cheri_getlen(addr) == alloc,
+			    ("realloc mismatch uma_zgrow_bounds: %zx != %zx",
+			        cheri_getlen(addr), alloc));
+		}
+#endif
 		break;
 	case SLAB_COOKIE_MALLOC_LARGE:
 		alloc = malloc_large_size(slab);
+#ifdef __CHERI_PURE_CAPABILITY__
+		if (size > olength) {
+			addr = malloc_large_grow_bounds(zone, addr);
+			KASSERT(cheri_getlen(addr) == size,
+			    ("realloc large object bounds mismatch: %zx != %zx",
+			        cheri_getlen(addr), size));
+		}
+#endif
 		break;
 	default:
 #ifdef INVARIANTS
@@ -1129,7 +1200,19 @@
 	if (size <= alloc &&
 	    (size > (alloc >> REALLOC_FRACTION) || alloc == MINALLOCSIZE)) {
 		kasan_mark((void *)addr, size, alloc, KASAN_MALLOC_REDZONE);
+#ifdef __CHERI_PURE_CAPABILITY__
+		addr = cheri_setbounds(addr, size);
+		/*
+		 * Zero only the non-representable portion of allocation
+		 * that was not reachable from the original capability.
+		 */
+		if (size > olength) {
+			bzero((void *)((uintptr_t)addr + olength),
+			    cheri_getlen(addr) - olength);
+		}
+#else
 		return (addr);
+#endif
 	}
 #endif /* !DEBUG_REDZONE */
 
@@ -1142,7 +1225,16 @@
 	 * valid before performing the copy.
 	 */
 	kasan_mark(addr, alloc, alloc, 0);
+#ifdef __CHERI_PURE_CAPABILITY__
+	/*
+	 * We have unbounded addr here, need to avoid copying
+	 * past the original length.
+	 * XXX-AM: is it worth it re-setting bounds on addr?
+	 */
+	bcopy(addr, newaddr, min(size, olength));
+#else
 	bcopy(addr, newaddr, min(size, alloc));
+#endif
 	free(addr, mtp);
 	return (newaddr);
 }

sys/vm/uma.h

@@ -414,6 +414,31 @@ uma_zfree_pcpu(uma_zone_t zone, void *item)
 	uma_zfree_pcpu_arg(zone, item, NULL);
 }
 
+#ifdef __CHERI_PURE_CAPABILITY__
+/*
+ * Reset bounds on an item to the full item size.
+ *
+ * This is intended for nested allocators to recover items capabilities before
+ * free-ing them.
+ * Note that UMA maintains the invariant that a full item capability must be
+ * passed to uma_zfree functions.
+ * XXX-AM: Should we instead relax that invariant (optionally via a zone flag),
+ * or add a different set of uma_zfree_narrow functions that allow for partial
+ * item capabilities?
+ * Given that this is a privileged operation, I would like to maintain the
+ * intentionality of the operation here.
+ *
+ * Error conditions:
+ * - Clear the item capability tag if the item does not belong
+ * to the given zone.
+ * - Clear the item capability tag if the item does not belong
+ * to the slab from vtoslab.
+ */
+void *uma_zgrow_bounds(uma_zone_t zone, void *item);
+#else
+#define	uma_zgrow_bounds(zone, item) (item)
+#endif
+
 /*
  * Wait until the specified zone can allocate an item.
  */

sys/vm/uma_core.c

@@ -5036,6 +5036,40 @@ zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip)
 		zone_free_limit(zone, 1);
 }
 
+#ifdef __CHERI_PURE_CAPABILITY__
+/* See uma.h */
+void *
+uma_zgrow_bounds(uma_zone_t zone, void *item)
+{
+	uma_keg_t keg = zone->uz_keg;
+	uma_slab_t slab;
+	int index;
+
+	KASSERT((zone->uz_flags & UMA_ZFLAG_VTOSLAB) != 0,
+	    ("Purecap kernel UMA zone missing UMA_ZFLAG_VTOSLAB"));
+	KASSERT((zone->uz_flags & (UMA_ZONE_PCPU | UMA_ZONE_SMR)) == 0,
+	    ("Can not grow bounds on PCPU and SMR zones"));
+
+	/*
+	 * XXX-AM: It should be safe to only check the index range
+	 * with INVARIANTS. If an item does not belong to the slab the computed
+	 * index will be garbage but it will fail setbounds with the slab_data()
+	 * capability.
+	 */
+	slab = vtoslab((vm_offset_t)item);
+	index = slab_item_index(slab, keg, item);
+	KASSERT(index >= 0 && index < keg->uk_ipers,
+	    ("Invalid item index %d for slab %#p from item %#p for zone %s",
+		index, slab, item, zone->uz_name));
+	item = cheri_setboundsexact(slab_item(slab, keg, index),
+	    keg->uk_size);
+	KASSERT(cheri_gettag(item),
+	    ("Failed to recover item %#p bounds for zone %s",
+		item, zone->uz_name));
+	return (item);
+}
+#endif
+
 /* See uma.h */
 int
 uma_zone_set_max(uma_zone_t zone, int nitems)

sys/vm/uma_int.h

@@ -403,6 +403,7 @@
 
 typedef struct uma_hash_slab * uma_hash_slab_t;
 
+#ifdef _KERNEL
 static inline uma_hash_slab_t
 slab_tohashslab(uma_slab_t slab)
 {
@@ -437,9 +438,10 @@
 	data = (uintptr_t)slab_data(slab, keg);
 	return (((ptraddr_t)item - (ptraddr_t)data) / keg->uk_rsize);
 }
+#endif /* _KERNEL */
 
 STAILQ_HEAD(uma_bucketlist, uma_bucket);
 
 struct uma_zone_domain {
 	struct uma_bucketlist uzd_buckets; /* full buckets */
 	uma_bucket_t	uzd_cross;	/* Fills from cross buckets. */