From 84719794e06c13fd9a3fd0c547af5bf9d76f7c3b Mon Sep 17 00:00:00 2001 From: "Liam R. Howlett" Date: Thu, 6 Apr 2023 21:05:55 -0400 Subject: [PATCH] FROMGIT: maple_tree: introduce mas_next_slot() interface Sometimes, during a tree walk, the user needs the next slot regardless of if it is empty or not. Add an interface to get the next slot. Since there are no consecutive NULLs allowed in the tree, the mas_next() function can only advance two slots at most. So use the new mas_next_slot() interface to align both implementations. Use this method for mas_find() as well. Link: https://lkml.kernel.org/r/20230518145544.1722059-28-Liam.Howlett@oracle.com Signed-off-by: Liam R. Howlett Cc: David Binderman Cc: Peng Zhang Cc: Sergey Senozhatsky Cc: Vernon Yang Cc: Wei Yang Signed-off-by: Andrew Morton (cherry picked from commit d0e70747bdb8f9cfc9ff48474e83171587ad94c1 git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-unstable) Bug: 274059236 Change-Id: I9efc69267859c26aae1b776d103dff584f1961ea Signed-off-by: Suren Baghdasaryan --- lib/maple_tree.c | 256 +++++++++++++++++++++-------------------------- 1 file changed, 116 insertions(+), 140 deletions(-) diff --git a/lib/maple_tree.c b/lib/maple_tree.c index a51a9b93dc5f..9fcae5f1880e 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -4634,11 +4634,10 @@ no_entry: static inline int mas_next_node(struct ma_state *mas, struct maple_node *node, unsigned long max) { - unsigned long min, pivot; + unsigned long min; unsigned long *pivots; struct maple_enode *enode; int level = 0; - unsigned char offset; unsigned char node_end; enum maple_type mt; void __rcu **slots; @@ -4646,19 +4645,16 @@ static inline int mas_next_node(struct ma_state *mas, struct maple_node *node, if (mas->max >= max) goto no_entry; + min = mas->max + 1; level = 0; do { if (ma_is_root(node)) goto no_entry; - min = mas->max + 1; - if (min > max) - goto no_entry; - + /* Walk up. */ if (unlikely(mas_ascend(mas))) return 1; - offset = mas->offset; level++; node = mas_mn(mas); mt = mte_node_type(mas->node); @@ -4667,36 +4663,37 @@ static inline int mas_next_node(struct ma_state *mas, struct maple_node *node, if (unlikely(ma_dead_node(node))) return 1; - } while (unlikely(offset == node_end)); + } while (unlikely(mas->offset == node_end)); slots = ma_slots(node, mt); - pivot = mas_safe_pivot(mas, pivots, ++offset, mt); - while (unlikely(level > 1)) { - /* Descend, if necessary */ - enode = mas_slot(mas, slots, offset); - if (unlikely(ma_dead_node(node))) - return 1; + mas->offset++; + enode = mas_slot(mas, slots, mas->offset); + if (unlikely(ma_dead_node(node))) + return 1; - mas->node = enode; + if (level > 1) + mas->offset = 0; + + while (unlikely(level > 1)) { level--; + mas->node = enode; node = mas_mn(mas); mt = mte_node_type(mas->node); slots = ma_slots(node, mt); - pivots = ma_pivots(node, mt); + enode = mas_slot(mas, slots, 0); if (unlikely(ma_dead_node(node))) return 1; - - offset = 0; - pivot = pivots[0]; } - enode = mas_slot(mas, slots, offset); + if (!mas->offset) + pivots = ma_pivots(node, mt); + + mas->max = mas_safe_pivot(mas, pivots, mas->offset, mt); if (unlikely(ma_dead_node(node))) return 1; mas->node = enode; mas->min = min; - mas->max = pivot; return 0; no_entry: @@ -4707,76 +4704,6 @@ no_entry: return 0; } -/* - * mas_next_nentry() - Get the next node entry - * @mas: The maple state - * @max: The maximum value to check - * @*range_start: Pointer to store the start of the range. - * - * Sets @mas->offset to the offset of the next node entry, @mas->last to the - * pivot of the entry. - * - * Return: The next entry, %NULL otherwise - */ -static inline void *mas_next_nentry(struct ma_state *mas, - struct maple_node *node, unsigned long max, enum maple_type type) -{ - unsigned char count; - unsigned long pivot; - unsigned long *pivots; - void __rcu **slots; - void *entry; - - if (mas->last == mas->max) { - mas->index = mas->max; - return NULL; - } - - slots = ma_slots(node, type); - pivots = ma_pivots(node, type); - count = ma_data_end(node, type, pivots, mas->max); - if (unlikely(ma_dead_node(node))) - return NULL; - - mas->index = mas_safe_min(mas, pivots, mas->offset); - if (unlikely(ma_dead_node(node))) - return NULL; - - if (mas->index > max) - return NULL; - - if (mas->offset > count) - return NULL; - - while (mas->offset < count) { - pivot = pivots[mas->offset]; - entry = mas_slot(mas, slots, mas->offset); - if (ma_dead_node(node)) - return NULL; - - mas->last = pivot; - if (entry) - return entry; - - if (pivot >= max) - return NULL; - - if (pivot >= mas->max) - return NULL; - - mas->index = pivot + 1; - mas->offset++; - } - - pivot = mas_logical_pivot(mas, pivots, mas->offset, type); - entry = mas_slot(mas, slots, mas->offset); - if (ma_dead_node(node)) - return NULL; - - mas->last = pivot; - return entry; -} - static inline void mas_rewalk(struct ma_state *mas, unsigned long index) { @@ -4785,9 +4712,99 @@ retry: mas_state_walk(mas); if (mas_is_start(mas)) goto retry; +} - return; +static inline bool mas_rewalk_if_dead(struct ma_state *mas, + struct maple_node *node, const unsigned long index) +{ + if (unlikely(ma_dead_node(node))) { + mas_rewalk(mas, index); + return true; + } + return false; +} +/* + * mas_next_slot() - Get the entry in the next slot + * + * @mas: The maple state + * @max: The maximum starting range + * @empty: Can be empty + * + * Return: The entry in the next slot which is possibly NULL + */ +void *mas_next_slot(struct ma_state *mas, unsigned long max, bool empty) +{ + void __rcu **slots; + unsigned long *pivots; + unsigned long pivot; + enum maple_type type; + struct maple_node *node; + unsigned char data_end; + unsigned long save_point = mas->last; + void *entry; + +retry: + node = mas_mn(mas); + type = mte_node_type(mas->node); + pivots = ma_pivots(node, type); + data_end = ma_data_end(node, type, pivots, mas->max); + if (unlikely(mas_rewalk_if_dead(mas, node, save_point))) + goto retry; + +again: + if (mas->max >= max) { + if (likely(mas->offset < data_end)) + pivot = pivots[mas->offset]; + else + return NULL; /* must be mas->max */ + + if (unlikely(mas_rewalk_if_dead(mas, node, save_point))) + goto retry; + + if (pivot >= max) + return NULL; + } + + if (likely(mas->offset < data_end)) { + mas->index = pivots[mas->offset] + 1; + mas->offset++; + if (likely(mas->offset < data_end)) + mas->last = pivots[mas->offset]; + else + mas->last = mas->max; + } else { + if (mas_next_node(mas, node, max)) { + mas_rewalk(mas, save_point); + goto retry; + } + + if (mas_is_none(mas)) + return NULL; + + mas->offset = 0; + mas->index = mas->min; + node = mas_mn(mas); + type = mte_node_type(mas->node); + pivots = ma_pivots(node, type); + mas->last = pivots[0]; + } + + slots = ma_slots(node, type); + entry = mt_slot(mas->tree, slots, mas->offset); + if (unlikely(mas_rewalk_if_dead(mas, node, save_point))) + goto retry; + + if (entry) + return entry; + + if (!empty) { + if (!mas->offset) + data_end = 2; + goto again; + } + + return entry; } /* @@ -4805,47 +4822,12 @@ retry: static inline void *mas_next_entry(struct ma_state *mas, unsigned long limit) { void *entry = NULL; - struct maple_node *node; - unsigned long last; - enum maple_type mt; if (mas->last >= limit) return NULL; - last = mas->last; -retry: - node = mas_mn(mas); - mt = mte_node_type(mas->node); - mas->offset++; - if (unlikely(mas->offset >= mt_slots[mt])) { - mas->offset = mt_slots[mt] - 1; - goto next_node; - } - - while (!mas_is_none(mas)) { - entry = mas_next_nentry(mas, node, limit, mt); - if (unlikely(ma_dead_node(node))) { - mas_rewalk(mas, last); - goto retry; - } - - if (likely(entry)) - return entry; - - if (unlikely((mas->last >= limit))) - return NULL; - -next_node: - if (unlikely(mas_next_node(mas, node, limit))) { - mas_rewalk(mas, last); - goto retry; - } - mas->offset = 0; - node = mas_mn(mas); - mt = mte_node_type(mas->node); - } - - return NULL; + entry = mas_next_slot(mas, limit, false); + return entry; } /* @@ -4876,10 +4858,8 @@ retry: slots = ma_slots(mn, mt); pivots = ma_pivots(mn, mt); count = ma_data_end(mn, mt, pivots, mas->max); - if (unlikely(ma_dead_node(mn))) { - mas_rewalk(mas, index); + if (unlikely(mas_rewalk_if_dead(mas, mn, index))) goto retry; - } offset = mas->offset - 1; if (offset >= mt_slots[mt]) @@ -4892,10 +4872,8 @@ retry: pivot = pivots[offset]; } - if (unlikely(ma_dead_node(mn))) { - mas_rewalk(mas, index); + if (unlikely(mas_rewalk_if_dead(mas, mn, index))) goto retry; - } while (offset && !mas_slot(mas, slots, offset)) { pivot = pivots[--offset]; @@ -4912,10 +4890,8 @@ retry: min = mas_safe_min(mas, pivots, offset); entry = mas_slot(mas, slots, offset); - if (unlikely(ma_dead_node(mn))) { - mas_rewalk(mas, index); + if (unlikely(mas_rewalk_if_dead(mas, mn, index))) goto retry; - } mas->offset = offset; mas->last = pivot; @@ -6181,8 +6157,8 @@ void *mas_find(struct ma_state *mas, unsigned long max) if (mas->index == max) return NULL; - /* Retries on dead nodes handled by mas_next_entry */ - return mas_next_entry(mas, max); + /* Retries on dead nodes handled by mas_next_slot */ + return mas_next_slot(mas, max, false); ptr_out_of_range: mas->node = MAS_NONE;