mm: Add support for MAP_FIXED_NOREPLACE flag in mmap

[wdcui]

Summary

Add support for the MAP_FIXED_NOREPLACE mmap flag, which allows creating a mapping at a specific address while failing with EEXIST if the range overlaps with existing mappings (unlike MAP_FIXED which silently replaces them).

This matches the Linux kernel behavior for MAP_FIXED_NOREPLACE (added in Linux 4.17).

Changes

• Add NOREPLACE flag to CreatePagesFlags (used in combination with FIXED_ADDR)
• MAP_FIXED_NOREPLACE now sets both FIXED_ADDR and NOREPLACE flags, matching Linux semantics where MAP_FIXED_NOREPLACE implies MAP_FIXED behavior for address selection
• Return EINVAL for MAP_FIXED_NOREPLACE with addr=0 (Linux requires a non-zero address)
• Add comprehensive syscall-level test covering:
  • Full overlap detection
  • Partial overlap at start/end
  • Adjacent (non-overlapping) mappings
  • Zero address validation

[github-actions]

🤖 SemverChecks 🤖 No breaking API changes detected

Note: this does not mean API is unchanged, or even that there are no breaking changes; simply, none of the detections triggered.

[CvvT]

I asked copilot to generate C test for it:

#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <sys/mman.h>
#include <unistd.h>

#define PAGE_SIZE 4096

int main(void) {
    void *addr;

    /* Case 1: mmap(NULL, ...) without MAP_FIXED
     * NULL is treated as a hint; kernel picks a suitable address.
     */
    printf("=== Case 1: mmap(NULL, ...) — no MAP_FIXED ===\n");
    addr = mmap(NULL, PAGE_SIZE, PROT_READ | PROT_WRITE,
                MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
    if (addr == MAP_FAILED) {
        printf("  FAILED: %s (errno=%d)\n", strerror(errno), errno);
    } else {
        printf("  OK: mapped at %p (kernel chose address)\n", addr);
        munmap(addr, PAGE_SIZE);
    }

    /* Case 2: mmap(0, ..., MAP_FIXED, ...)
     * MAP_FIXED forces the kernel to map exactly at address 0.
     * On modern Linux, vm.mmap_min_addr prevents this → EPERM.
     * If it somehow succeeded, it would also unmap any existing
     * mapping at that address.
     */
    printf("\n=== Case 2: mmap(0, ..., MAP_FIXED) ===\n");
    addr = mmap(0, PAGE_SIZE, PROT_READ | PROT_WRITE,
                MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
    if (addr == MAP_FAILED) {
        printf("  FAILED: %s (errno=%d)\n", strerror(errno), errno);
        if (errno == EPERM)
            printf("  → Blocked by vm.mmap_min_addr (cannot map the zero page)\n");
    } else {
        printf("  OK: mapped at %p (zero page mapped!)\n", addr);
        munmap(addr, PAGE_SIZE);
    }

    /* Case 3: mmap(0, ..., MAP_FIXED_NOREPLACE, ...)
     * MAP_FIXED_NOREPLACE forces address 0, but won't replace
     * existing mappings. Same vm.mmap_min_addr restriction applies.
     */
    printf("\n=== Case 3: mmap(0, ..., MAP_FIXED_NOREPLACE) ===\n");
    addr = mmap(0, PAGE_SIZE, PROT_READ | PROT_WRITE,
                MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0);
    if (addr == MAP_FAILED) {
        printf("  FAILED: %s (errno=%d)\n", strerror(errno), errno);
        if (errno == EPERM)
            printf("  → Blocked by vm.mmap_min_addr (cannot map the zero page)\n");
        else if (errno == EEXIST)
            printf("  → Address 0 already mapped (NOREPLACE refuses to overwrite)\n");
    } else {
        printf("  OK: mapped at %p (zero page mapped!)\n", addr);
        munmap(addr, PAGE_SIZE);
    }

    /* Case 4: Compare MAP_FIXED vs MAP_FIXED_NOREPLACE at a valid address.
     * This shows the key behavioral difference between the two flags.
     */
    printf("\n=== Case 4: MAP_FIXED vs MAP_FIXED_NOREPLACE at a valid address ===\n");

    /* First, get a valid mapping */
    void *base = mmap(NULL, PAGE_SIZE * 2, PROT_READ | PROT_WRITE,
                      MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
    if (base == MAP_FAILED) {
        perror("  initial mmap");
        return 1;
    }
    printf("  Initial mapping at %p (2 pages)\n", base);
    memset(base, 'A', PAGE_SIZE * 2);

    /* MAP_FIXED at the same address: silently replaces the existing mapping */
    addr = mmap(base, PAGE_SIZE, PROT_READ | PROT_WRITE,
                MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
    if (addr == MAP_FAILED) {
        printf("  MAP_FIXED replace FAILED: %s\n", strerror(errno));
    } else {
        printf("  MAP_FIXED at %p: OK (replaced existing mapping, data zeroed: first byte = 0x%02x)\n",
               addr, ((unsigned char *)addr)[0]);
    }

    /* MAP_FIXED_NOREPLACE at the same address: should fail with EEXIST */
    addr = mmap(base, PAGE_SIZE, PROT_READ | PROT_WRITE,
                MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0);
    if (addr == MAP_FAILED) {
        printf("  MAP_FIXED_NOREPLACE at %p: FAILED: %s (errno=%d)\n",
               base, strerror(errno), errno);
        if (errno == EEXIST)
            printf("  → Refuses to overwrite existing mapping (safe behavior)\n");
    } else {
        printf("  MAP_FIXED_NOREPLACE at %p: OK (unexpected)\n", addr);
    }

    munmap(base, PAGE_SIZE * 2);

    printf("\n=== Summary ===\n");
    printf("- mmap(NULL) without MAP_FIXED: kernel chooses address (NULL is just a hint)\n");
    printf("- mmap(0, MAP_FIXED): tries to map at address 0 → blocked by mmap_min_addr\n");
    printf("- mmap(0, MAP_FIXED_NOREPLACE): same → blocked by mmap_min_addr\n");
    printf("- MAP_FIXED: silently replaces existing mappings at the target address\n");
    printf("- MAP_FIXED_NOREPLACE: fails with EEXIST if address is already mapped\n");

    return 0;
}

The correct error code should be EPERM. We already have check at:

litebox/litebox/src/mm/linux.rs

Lines 416 to 418 in 90c7ed9

	let (start, end) = (suggested_range.start, suggested_range.end);
	if start < Platform::TASK_ADDR_MIN || end > Platform::TASK_ADDR_MAX {
	return Err(AllocationError::InvalidRange);

But get_unmmaped_area incorrectly returns an available address when suggested addr is zero and MAP_FIEXED flag is set.

litebox/litebox/src/mm/linux.rs

Lines 515 to 525 in 90c7ed9

	let new_addr = self
	.get_unmmaped_area(
	suggested_address,
	total_length,
	flags.contains(CreatePagesFlags::FIXED_ADDR),
	)
	.ok_or(AllocationError::OutOfMemory)?;
	// new_addr must be ALIGN aligned
	let new_range = PageRange::new(new_addr, new_addr + length.as_usize()).unwrap();
	unsafe {
	self.insert_mapping(

We should probably fix these functions and new error for AllocationError.