Skip to content

Commit

Permalink
radix_tree: tag all internal tree nodes as indirect pointers
Browse files Browse the repository at this point in the history 
Set the 'indirect_ptr' bit on all the pointers to internal nodes, not
just on the root node.  This enables the following patches to support
multi-order entries in the radix tree.  This patch is split out for ease
of bisection.

Signed-off-by: Matthew Wilcox <willy@linux.intel.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Matthew Wilcox <willy@linux.intel.com>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Ross Zwisler <ross.zwisler@linux.intel.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
  • Loading branch information
Matthew Wilcox authored and Linus Torvalds committed Mar 17, 2016
1 parent 1366c37 commit 339e635
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 8 deletions.
24 changes: 18 additions & 6 deletions lib/radix-tree.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -368,9 +368,10 @@ static int radix_tree_extend(struct radix_tree_root *root, unsigned long index)
node->count = 1;
node->parent = NULL;
slot = root->rnode;
if (newheight > 1) {
if (radix_tree_is_indirect_ptr(slot) && newheight > 1) {
slot = indirect_to_ptr(slot);
slot->parent = node;
slot = ptr_to_indirect(slot);
}
node->slots[0] = slot;
node = ptr_to_indirect(node);
Expand Down Expand Up @@ -425,17 +426,20 @@ int __radix_tree_create(struct radix_tree_root *root, unsigned long index,
slot->path = height;
slot->parent = node;
if (node) {
rcu_assign_pointer(node->slots[offset], slot);
rcu_assign_pointer(node->slots[offset],
ptr_to_indirect(slot));
node->count++;
slot->path |= offset << RADIX_TREE_HEIGHT_SHIFT;
} else
rcu_assign_pointer(root->rnode, ptr_to_indirect(slot));
rcu_assign_pointer(root->rnode,
ptr_to_indirect(slot));
}

/* Go a level down */
offset = (index >> shift) & RADIX_TREE_MAP_MASK;
node = slot;
slot = node->slots[offset];
slot = indirect_to_ptr(slot);
shift -= RADIX_TREE_MAP_SHIFT;
height--;
}
Expand Down Expand Up @@ -533,6 +537,7 @@ void *__radix_tree_lookup(struct radix_tree_root *root, unsigned long index,
node = rcu_dereference_raw(*slot);
if (node == NULL)
return NULL;
node = indirect_to_ptr(node);

shift -= RADIX_TREE_MAP_SHIFT;
height--;
Expand Down Expand Up @@ -619,6 +624,7 @@ void *radix_tree_tag_set(struct radix_tree_root *root,
tag_set(slot, tag, offset);
slot = slot->slots[offset];
BUG_ON(slot == NULL);
slot = indirect_to_ptr(slot);
shift -= RADIX_TREE_MAP_SHIFT;
height--;
}
Expand Down Expand Up @@ -658,11 +664,12 @@ void *radix_tree_tag_clear(struct radix_tree_root *root,
goto out;

shift = height * RADIX_TREE_MAP_SHIFT;
slot = indirect_to_ptr(root->rnode);
slot = root->rnode;

while (shift) {
if (slot == NULL)
goto out;
slot = indirect_to_ptr(slot);

shift -= RADIX_TREE_MAP_SHIFT;
offset = (index >> shift) & RADIX_TREE_MAP_MASK;
Expand Down Expand Up @@ -738,6 +745,7 @@ int radix_tree_tag_get(struct radix_tree_root *root,

if (node == NULL)
return 0;
node = indirect_to_ptr(node);

offset = (index >> shift) & RADIX_TREE_MAP_MASK;
if (!tag_get(node, tag, offset))
Expand Down Expand Up @@ -838,6 +846,7 @@ void **radix_tree_next_chunk(struct radix_tree_root *root,
node = rcu_dereference_raw(node->slots[offset]);
if (node == NULL)
goto restart;
node = indirect_to_ptr(node);
shift -= RADIX_TREE_MAP_SHIFT;
offset = (index >> shift) & RADIX_TREE_MAP_MASK;
}
Expand Down Expand Up @@ -939,6 +948,7 @@ unsigned long radix_tree_range_tag_if_tagged(struct radix_tree_root *root,
shift -= RADIX_TREE_MAP_SHIFT;
node = slot;
slot = slot->slots[offset];
slot = indirect_to_ptr(slot);
continue;
}

Expand Down Expand Up @@ -1195,6 +1205,7 @@ static unsigned long __locate(struct radix_tree_node *slot, void *item,
slot = rcu_dereference_raw(slot->slots[i]);
if (slot == NULL)
goto out;
slot = indirect_to_ptr(slot);
}

/* Bottom level: check items */
Expand Down Expand Up @@ -1278,7 +1289,8 @@ static inline void radix_tree_shrink(struct radix_tree_root *root)
*/
if (to_free->count != 1)
break;
if (!to_free->slots[0])
slot = to_free->slots[0];
if (!slot)
break;

/*
Expand All @@ -1288,8 +1300,8 @@ static inline void radix_tree_shrink(struct radix_tree_root *root)
* (to_free->slots[0]), it will be safe to dereference the new
* one (root->rnode) as far as dependent read barriers go.
*/
slot = to_free->slots[0];
if (root->height > 1) {
slot = indirect_to_ptr(slot);
slot->parent = NULL;
slot = ptr_to_indirect(slot);
}
Expand Down
5 changes: 3 additions & 2 deletions tools/testing/radix-tree/test.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,8 @@ static int verify_node(struct radix_tree_node *slot, unsigned int tag,
int i;
int j;

slot = indirect_to_ptr(slot);

/* Verify consistency at this level */
for (i = 0; i < RADIX_TREE_TAG_LONGS; i++) {
if (slot->tags[tag][i]) {
Expand Down Expand Up @@ -184,8 +186,7 @@ void verify_tag_consistency(struct radix_tree_root *root, unsigned int tag)
{
if (!root->height)
return;
verify_node(indirect_to_ptr(root->rnode),
tag, root->height, !!root_tag_get(root, tag));
verify_node(root->rnode, tag, root->height, !!root_tag_get(root, tag));
}

void item_kill_tree(struct radix_tree_root *root)
Expand Down

0 comments on commit 339e635

Please sign in to comment.