diff --git a/src/libexpr/gc.cc b/src/libexpr/gc.cc index 902a4c712..96b2728bf 100644 --- a/src/libexpr/gc.cc +++ b/src/libexpr/gc.cc @@ -9,6 +9,8 @@ GC gc; GC::GC() { + nextSize = std::max((size_t) 2, parseSize(getEnv("GC_INITIAL_HEAP_SIZE", "131072")) / WORD_SIZE); + // FIXME: placement new frontSentinel = (Ptr *) malloc(sizeof(Ptr)); backSentinel = (Ptr *) malloc(sizeof(Ptr)); @@ -27,10 +29,23 @@ GC::GC() backRootSentinel->prev = frontRootSentinel; backRootSentinel->next = nullptr; + + freeLists[0].minSize = 2; + freeLists[1].minSize = 3; + freeLists[2].minSize = 4; + freeLists[3].minSize = 8; + freeLists[4].minSize = 16; + freeLists[5].minSize = 32; + freeLists[6].minSize = 64; + freeLists[7].minSize = 128; + + addArena(nextSize); } GC::~GC() { + debug("allocated %d bytes in total", totalSize * WORD_SIZE); + size_t n = 0; for (Ptr * p = frontSentinel->next; p != backSentinel; p = p->next) n++; @@ -49,6 +64,34 @@ GC::~GC() assert(!backRootSentinel->next); } +void GC::addArena(size_t arenaSize) +{ + debug("allocating arena of %d bytes", arenaSize * WORD_SIZE); + + auto arena = Arena(arenaSize); + + // Add this arena to a freelist as a single block. + addToFreeList(new (arena.start) Free(arenaSize)); + + arenas.emplace_back(std::move(arena)); + + totalSize += arenaSize; + + nextSize = arenaSize * 1.5; // FIXME: overflow, clamp +} + +void GC::addToFreeList(Free * obj) +{ + auto size = obj->words(); + for (auto i = freeLists.rbegin(); i != freeLists.rend(); ++i) + if (size >= i->minSize) { + obj->next = i->front; + i->front = obj; + return; + } + abort(); +} + void GC::gc() { size_t marked = 0; @@ -190,49 +233,45 @@ void GC::gc() processStack(); } + // Reset all the freelists. + for (auto & freeList : freeLists) + freeList.front = nullptr; + + // Go through all the arenas and add free objects to the + // appropriate freelists. Size totalObjectsFreed = 0; Size totalWordsFreed = 0; - for (auto & arenaList : arenaLists) { - - for (auto & arena : arenaList.arenas) { - auto [objectsFreed, wordsFreed] = arena.freeUnmarked(); - totalObjectsFreed += objectsFreed; - totalWordsFreed += wordsFreed; - } - - std::sort(arenaList.arenas.begin(), arenaList.arenas.end(), - [](const Arena & a, const Arena & b) { - return b.free < a.free; - }); + for (auto & arena : arenas) { + auto [objectsFreed, wordsFreed] = freeUnmarked(arena); + totalObjectsFreed += objectsFreed; + totalWordsFreed += wordsFreed; } debug("freed %d bytes in %d dead objects, keeping %d objects", totalWordsFreed * WORD_SIZE, totalObjectsFreed, marked); } -std::pair GC::Arena::freeUnmarked() +std::pair GC::freeUnmarked(Arena & arena) { Size objectsFreed = 0; Size wordsFreed = 0; - auto end = start + size; - auto pos = start; + auto end = arena.start + arena.size; + auto pos = arena.start; Free * curFree = nullptr; - Free * * freeLink = &firstFree; - free = 0; + auto linkCurFree = [&]() { + if (curFree && curFree->words() > 1) + addToFreeList(curFree); + curFree = nullptr; + }; while (pos < end) { auto obj = (Object *) pos; auto tag = obj->type; - auto linkFree = [&]() { - *freeLink = curFree; - freeLink = &curFree->next; - }; - Size objSize; if (tag >= tInt && tag <= tFloat) { objSize = ((Value *) obj)->words(); @@ -265,33 +304,24 @@ std::pair GC::Arena::freeUnmarked() auto mergeFree = [&]() { //printError("MERGE %x %x %d", curFree, obj, curFree->size() + objSize); assert(curFree->words() >= 1); - if (curFree->words() == 1) { - linkFree(); - free += 1; - } curFree->setSize(curFree->words() + objSize); - free += objSize; }; if (tag == tFree) { - //debug("FREE %x %d", obj, obj->getMisc()); + //debug("KEEP FREE %x %d", obj, obj->getMisc()); if (curFree) { // Merge this object into the previous free // object. mergeFree(); } else { curFree = (Free *) obj; - if (curFree->words() > 1) { - linkFree(); - free += curFree->words(); - } } } else { if (obj->isMarked()) { // Unmark to prepare for the next GC run. //debug("KEEP OBJECT %x %d %d", obj, obj->type, objSize); - curFree = nullptr; + linkCurFree(); obj->unmark(); } else { //debug("FREE OBJECT %x %d %d", obj, obj->type, objSize); @@ -308,8 +338,6 @@ std::pair GC::Arena::freeUnmarked() curFree = (Free *) obj; curFree->type = tFree; curFree->setSize(objSize); - linkFree(); - free += objSize; } } } @@ -317,28 +345,19 @@ std::pair GC::Arena::freeUnmarked() pos += objSize; } - assert(pos == end); + linkCurFree(); - *freeLink = nullptr; + assert(pos == end); return {objectsFreed, wordsFreed}; } bool GC::isObject(void * p) { - for (auto & arenaList : arenaLists) { - for (auto & arena : arenaList.arenas) { - if (p >= arena.start && p < arena.start + arena.size) - return true; - } - } + for (auto & arena : arenas) + if (p >= arena.start && p < arena.start + arena.size) + return true; return false; } -GC::ArenaList::ArenaList() -{ - static Size initialHeapSize = std::stol(getEnv("GC_INITIAL_HEAP_SIZE", "1000000")) / WORD_SIZE; - nextSize = initialHeapSize; -} - } diff --git a/src/libexpr/gc.hh b/src/libexpr/gc.hh index dfd8c1b08..0bc951923 100644 --- a/src/libexpr/gc.hh +++ b/src/libexpr/gc.hh @@ -6,6 +6,8 @@ #include #include +//#define GC_DEBUG 1 + namespace nix { typedef unsigned long Word; @@ -150,90 +152,100 @@ private: struct Arena { Size size; // in words - Size free; // words free - Free * firstFree; Word * start; Arena(Size size) : size(size) - , free(size) , start(new Word[size]) { assert(size >= 2); - firstFree = new (start) Free(size); } Arena(const Arena & arena) = delete; Arena(Arena && arena) - { - *this = std::move(arena); - } - - Arena & operator =(Arena && arena) { size = arena.size; - free = arena.free; - firstFree = arena.firstFree; start = arena.start; arena.start = nullptr; - return *this; } ~Arena() { delete[] start; } + }; - Object * alloc(Size size) - { - assert(size >= 2); + size_t totalSize = 0; + size_t nextSize; - Free * * prev = &firstFree; + std::vector arenas; - while (Free * freeObj = *prev) { - //printError("LOOK %x %d %x", freeObj, freeObj->words(), freeObj->next); - assert(freeObj->words() >= 2); - if (freeObj->words() == size) { - *prev = freeObj->next; - assert(free >= size); - free -= size; - return (Object *) freeObj; - } else if (freeObj->words() >= size + 2) { - // Split this free object. - auto newSize = freeObj->words() - size; - freeObj->setSize(newSize); - assert(free >= size); - free -= size; - return (Object *) (((Word *) freeObj) + newSize); - } else if (freeObj->words() == size + 1) { - // Return this free object and add a padding word. - *prev = freeObj->next; - freeObj->setSize(1); - assert(free >= size + 1); - free -= size + 1; - return (Object *) (((Word *) freeObj) + 1); - } else { - assert(freeObj->words() < size); - prev = &freeObj->next; + struct FreeList + { + Size minSize; + Free * front = nullptr; + }; + + std::array freeLists; + + Object * allocObject(Size size) + { + assert(size >= 2); + + for (int attempt = 0; attempt < 3; attempt++) { + + for (size_t i = 0; i < freeLists.size(); ++i) { + auto & freeList = freeLists[i]; + + if ((size <= freeList.minSize || i == freeLists.size() - 1) && freeList.front) { + //printError("TRY %d %d %d", size, i, freeList.minSize); + + Free * * prev = &freeList.front; + + while (Free * freeObj = *prev) { + //printError("LOOK %x %d %x", freeObj, freeObj->words(), freeObj->next); + assert(freeObj->words() >= freeList.minSize); + if (freeObj->words() == size) { + // Convert the free object. + *prev = freeObj->next; + return (Object *) freeObj; + } else if (freeObj->words() >= size + 2) { + // Split the free object. + auto newSize = freeObj->words() - size; + freeObj->setSize(newSize); + if (newSize < freeList.minSize) { + /* The free object is now smaller than + the minimum size for this freelist, + so move it to another one. */ + //printError("MOVE %x %d -> %d", freeObj, newSize + size, newSize); + *prev = freeObj->next; + addToFreeList(freeObj); + } + return (Object *) (((Word *) freeObj) + newSize); + } else if (freeObj->words() == size + 1) { + // Return the free object and add a padding word. + *prev = freeObj->next; + freeObj->setSize(1); + return (Object *) (((Word *) freeObj) + 1); + } else { + assert(freeObj->words() < size); + prev = &freeObj->next; + } + } } } - return nullptr; + if (attempt == 0) { + debug("allocation of %d bytes failed, GCing...", size * WORD_SIZE); + gc(); + } else if (attempt == 1) { + addArena(std::max(nextSize, size)); + } } - std::pair freeUnmarked(); - }; - - // Note: arenas are sorted by ascending amount of free space. - struct ArenaList - { - Size nextSize; - std::vector arenas; - ArenaList(); - }; - - std::array arenaLists; + throw Error("allocation of %d bytes failed", size); + } public: @@ -243,34 +255,8 @@ public: template Ptr alloc(Size size, const Args & ... args) { - ArenaList & arenaList = - size == 3 ? arenaLists[0] : - size == 4 ? arenaLists[1] : - arenaLists[2]; - - for (int i = 0; i < 3; i++) { - - for (auto j = arenaList.arenas.rbegin(); j != arenaList.arenas.rend(); ++j) { - auto & arena = *j; - auto raw = arena.alloc(size); - if (raw) { - auto obj = new (raw) T(args...); - return obj; - } - } - - if (i == 0) { - debug("allocation of %d bytes failed, GCing...", size * WORD_SIZE); - gc(); - } else { - Size arenaSize = std::max(arenaList.nextSize, size); - arenaList.nextSize = arenaSize * 1.5; // FIXME: overflow - debug("allocating arena of %d bytes", arenaSize * WORD_SIZE); - arenaList.arenas.emplace_back(arenaSize); - } - } - - throw Error("allocation of %d bytes failed", size); + auto raw = allocObject(size); + return new (raw) T(args...); } void gc(); @@ -286,6 +272,14 @@ public: } #endif } + +private: + + void addArena(size_t arenaSize); + + void addToFreeList(Free * obj); + + std::pair freeUnmarked(Arena & arena); }; extern GC gc;