From 7d5567a8d7b6b347eea6bd3f51d094ceb855e452 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 3 Nov 2025 15:50:33 +0100 Subject: [PATCH 1/3] Fix macOS IPC cleanup using sysctl: shared memory segments In Linux, IPC objects are automatically cleaned up when the IPC namespace is destroyed. On Darwin, since there are no IPC namespaces, the IPC objects may sometimes persist after the build user's processes are killed. This patch modifies the cleanup logic to use sysctl calls to identify and remove left over shm segments associated with the build user. Fixes: #12548 --- .../unix/build/darwin-derivation-builder.cc | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/src/libstore/unix/build/darwin-derivation-builder.cc b/src/libstore/unix/build/darwin-derivation-builder.cc index 21b3c6cb9..2ec3ea845 100644 --- a/src/libstore/unix/build/darwin-derivation-builder.cc +++ b/src/libstore/unix/build/darwin-derivation-builder.cc @@ -3,11 +3,27 @@ # include # include # include +# include +# include /* This definition is undocumented but depended upon by all major browsers. */ extern "C" int sandbox_init_with_parameters(const char * profile, uint64_t flags, const char * const parameters[], char ** errorbuf); +/* Darwin IPC cleanup structures and constants */ +# define IPCS_MAGIC 0x00000001 +# define IPCS_SHM_ITER 0x00000002 +# define IPCS_SHM_SYSCTL "kern.sysv.ipcs.shm" + +struct IPCS_command +{ + uint32_t ipcs_magic; + uint32_t ipcs_op; + uint32_t ipcs_cursor; + uint32_t ipcs_datalen; + void * ipcs_data; +}; + namespace nix { struct DarwinDerivationBuilder : DerivationBuilderImpl @@ -204,6 +220,44 @@ struct DarwinDerivationBuilder : DerivationBuilderImpl posix_spawn( NULL, drv.builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); } + + void cleanupSysVIPCForUser(uid_t uid) + { + struct IPCS_command ic; + struct shmid_ds shm_ds; + size_t ic_size = sizeof(ic); + + ic.ipcs_magic = IPCS_MAGIC; + ic.ipcs_op = IPCS_SHM_ITER; + ic.ipcs_cursor = 0; + ic.ipcs_data = &shm_ds; + ic.ipcs_datalen = sizeof(shm_ds); + + while (true) { + memset(&shm_ds, 0, sizeof(shm_ds)); + + if (sysctlbyname(IPCS_SHM_SYSCTL, &ic, &ic_size, &ic, ic_size) != 0) { + break; + } + + if (shm_ds.shm_perm.uid == uid) { + int shmid = shmget(shm_ds.shm_perm._key, 0, 0); + if (shmid != -1) { + if (shmctl(shmid, IPC_RMID, NULL) == 0) + debug("removed shared memory segment with shmid %d (key: 0x%x)", shmid, shm_ds.shm_perm._key); + } + } + } + } + + void killSandbox(bool getStats) override + { + DerivationBuilderImpl::killSandbox(getStats); + if (buildUser) { + auto uid = buildUser->getUID(); + cleanupSysVIPCForUser(uid); + } + } }; } // namespace nix From 4f85cfe824a0cc01870992851bad43f89eabccf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 4 Nov 2025 14:59:41 +0000 Subject: [PATCH 2/3] fix(darwin): extend IPC cleanup to message queues and semaphores Previously, only shared memory segments were cleaned up. This could lead to leaked message queues and semaphore sets when builds use System V IPC, exhausting kernel IPC limits over time. This commit extends the cleanup to all three System V IPC types: 1. Shared memory segments 2. Message queues 3. Semaphores Additionally, we stop removing IPC objects during iteration, as it could corrupt the kernel's iterator state and cause some objects to be skipped. The new implementation uses a two-pass approach where we list first and then remove them in a separate pass. The IPC IDs are now extracted during iteration using actual system calls (shmget, msgget, semget) rather than being looked up later, ensuring the objects exist when we capture their IDs. --- .../unix/build/darwin-derivation-builder.cc | 110 +++++++++++++++--- 1 file changed, 91 insertions(+), 19 deletions(-) diff --git a/src/libstore/unix/build/darwin-derivation-builder.cc b/src/libstore/unix/build/darwin-derivation-builder.cc index 2ec3ea845..701da9513 100644 --- a/src/libstore/unix/build/darwin-derivation-builder.cc +++ b/src/libstore/unix/build/darwin-derivation-builder.cc @@ -5,17 +5,23 @@ # include # include # include +# include +# include /* This definition is undocumented but depended upon by all major browsers. */ extern "C" int sandbox_init_with_parameters(const char * profile, uint64_t flags, const char * const parameters[], char ** errorbuf); -/* Darwin IPC cleanup structures and constants */ +/* Darwin IPC structures and constants */ # define IPCS_MAGIC 0x00000001 # define IPCS_SHM_ITER 0x00000002 +# define IPCS_SEM_ITER 0x00000020 +# define IPCS_MSG_ITER 0x00000200 # define IPCS_SHM_SYSCTL "kern.sysv.ipcs.shm" +# define IPCS_MSG_SYSCTL "kern.sysv.ipcs.msg" +# define IPCS_SEM_SYSCTL "kern.sysv.ipcs.sem" -struct IPCS_command +struct IpcsCommand { uint32_t ipcs_magic; uint32_t ipcs_op; @@ -223,31 +229,97 @@ struct DarwinDerivationBuilder : DerivationBuilderImpl void cleanupSysVIPCForUser(uid_t uid) { - struct IPCS_command ic; - struct shmid_ds shm_ds; + struct IpcsCommand ic; size_t ic_size = sizeof(ic); + // IPC ids to cleanup + std::vector shm_ids, msg_ids, sem_ids; - ic.ipcs_magic = IPCS_MAGIC; - ic.ipcs_op = IPCS_SHM_ITER; - ic.ipcs_cursor = 0; - ic.ipcs_data = &shm_ds; - ic.ipcs_datalen = sizeof(shm_ds); + { + struct shmid_ds shm_ds; + ic.ipcs_magic = IPCS_MAGIC; + ic.ipcs_op = IPCS_SHM_ITER; + ic.ipcs_cursor = 0; + ic.ipcs_data = &shm_ds; + ic.ipcs_datalen = sizeof(shm_ds); - while (true) { - memset(&shm_ds, 0, sizeof(shm_ds)); + while (true) { + memset(&shm_ds, 0, sizeof(shm_ds)); - if (sysctlbyname(IPCS_SHM_SYSCTL, &ic, &ic_size, &ic, ic_size) != 0) { - break; - } + if (sysctlbyname(IPCS_SHM_SYSCTL, &ic, &ic_size, &ic, ic_size) != 0) { + break; + } - if (shm_ds.shm_perm.uid == uid) { - int shmid = shmget(shm_ds.shm_perm._key, 0, 0); - if (shmid != -1) { - if (shmctl(shmid, IPC_RMID, NULL) == 0) - debug("removed shared memory segment with shmid %d (key: 0x%x)", shmid, shm_ds.shm_perm._key); + if (shm_ds.shm_perm.uid == uid) { + int shmid = shmget(shm_ds.shm_perm._key, 0, 0); + if (shmid != -1) { + shm_ids.push_back(shmid); + } } } } + + for (auto id : shm_ids) { + if (shmctl(id, IPC_RMID, NULL) == 0) + debug("removed shared memory segment with shmid %d", id); + } + + { + struct msqid_ds msg_ds; + ic.ipcs_magic = IPCS_MAGIC; + ic.ipcs_op = IPCS_MSG_ITER; + ic.ipcs_cursor = 0; + ic.ipcs_data = &msg_ds; + ic.ipcs_datalen = sizeof(msg_ds); + + while (true) { + memset(&msg_ds, 0, sizeof(msg_ds)); + + if (sysctlbyname(IPCS_MSG_SYSCTL, &ic, &ic_size, &ic, ic_size) != 0) { + break; + } + + if (msg_ds.msg_perm.uid == uid) { + int msgid = msgget(msg_ds.msg_perm._key, 0); + if (msgid != -1) { + msg_ids.push_back(msgid); + } + } + } + } + + for (auto id : msg_ids) { + if (msgctl(id, IPC_RMID, NULL) == 0) + debug("removed message queue with msgid %d", id); + } + + { + struct semid_ds sem_ds; + ic.ipcs_magic = IPCS_MAGIC; + ic.ipcs_op = IPCS_SEM_ITER; + ic.ipcs_cursor = 0; + ic.ipcs_data = &sem_ds; + ic.ipcs_datalen = sizeof(sem_ds); + + while (true) { + memset(&sem_ds, 0, sizeof(sem_ds)); + + if (sysctlbyname(IPCS_SEM_SYSCTL, &ic, &ic_size, &ic, ic_size) != 0) { + break; + } + + if (sem_ds.sem_perm.uid == uid) { + int semid = semget(sem_ds.sem_perm._key, 0, 0); + if (semid != -1) { + sem_ids.push_back(semid); + } + } + } + } + + for (auto id : sem_ids) { + if (semctl(id, 0, IPC_RMID) == 0) + debug("removed semaphore with semid %d", id); + } } void killSandbox(bool getStats) override From 0507674a13a02a46c789b6e314deffb73302e135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Tue, 4 Nov 2025 20:56:15 +0000 Subject: [PATCH 3/3] Document the new cleanup function using a Doxygen-style comment --- src/libstore/unix/build/darwin-derivation-builder.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/libstore/unix/build/darwin-derivation-builder.cc b/src/libstore/unix/build/darwin-derivation-builder.cc index 701da9513..613ec6d54 100644 --- a/src/libstore/unix/build/darwin-derivation-builder.cc +++ b/src/libstore/unix/build/darwin-derivation-builder.cc @@ -227,6 +227,15 @@ struct DarwinDerivationBuilder : DerivationBuilderImpl NULL, drv.builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); } + /** + * Cleans up all System V IPC objects owned by the specified user. + * + * On Darwin, IPC objects (shared memory segments, message queues, and semaphore) + * can persist after the build user's processes are killed, since there are no IPC namespaces + * like on Linux. This can exhaust kernel IPC limits over time. + * + * Uses sysctl to enumerate and remove all IPC objects owned by the given UID. + */ void cleanupSysVIPCForUser(uid_t uid) { struct IpcsCommand ic;