Skip to content

Commit

Permalink
btrfs: subpage: pack all subpage bitmaps into a larger bitmap
Browse files Browse the repository at this point in the history 
Currently we use u16 bitmap to make 4k sectorsize work for 64K page
size.

But this u16 bitmap is not large enough to contain larger page size like
128K, nor is space efficient for 16K page size.

To handle both cases, here we pack all subpage bitmaps into a larger
bitmap, now btrfs_subpage::bitmaps[] will be the ultimate bitmap for
subpage usage.

Each sub-bitmap will has its start bit number recorded in
btrfs_subpage_info::*_start, and its bitmap length will be recorded in
btrfs_subpage_info::bitmap_nr_bits.

All subpage bitmap operations will be converted from using direct u16
operations to bitmap operations, with above *_start calculated.

For 64K page size with 4K sectorsize, this should not cause much
difference.

While for 16K page size, we will only need 1 unsigned long (u32) to
store all the bitmaps, which saves quite some space.

Furthermore, this allows us to support larger page size like 128K and
258K.

Signed-off-by: Qu Wenruo <wqu@suse.com>
Reviewed-by: David Sterba <dsterba@suse.com>
Signed-off-by: David Sterba <dsterba@suse.com>
  • Loading branch information
Qu Wenruo authored and David Sterba committed Oct 26, 2021
1 parent 8481dd8 commit 72a69cd
Show file tree
Hide file tree
Showing 3 changed files with 121 additions and 85 deletions.
61 changes: 37 additions & 24 deletions fs/btrfs/extent_io.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3854,12 +3854,11 @@ static void find_next_dirty_byte(struct btrfs_fs_info *fs_info,
struct page *page, u64 *start, u64 *end)
{
struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
struct btrfs_subpage_info *spi = fs_info->subpage_info;
u64 orig_start = *start;
/* Declare as unsigned long so we can use bitmap ops */
unsigned long dirty_bitmap;
unsigned long flags;
int nbits = (orig_start - page_offset(page)) >> fs_info->sectorsize_bits;
int range_start_bit = nbits;
int range_start_bit;
int range_end_bit;

/*
Expand All @@ -3872,13 +3871,18 @@ static void find_next_dirty_byte(struct btrfs_fs_info *fs_info,
return;
}

range_start_bit = spi->dirty_offset +
(offset_in_page(orig_start) >> fs_info->sectorsize_bits);

/* We should have the page locked, but just in case */
spin_lock_irqsave(&subpage->lock, flags);
dirty_bitmap = subpage->dirty_bitmap;
bitmap_next_set_region(subpage->bitmaps, &range_start_bit, &range_end_bit,
spi->dirty_offset + spi->bitmap_nr_bits);
spin_unlock_irqrestore(&subpage->lock, flags);

bitmap_next_set_region(&dirty_bitmap, &range_start_bit, &range_end_bit,
BTRFS_SUBPAGE_BITMAP_SIZE);
range_start_bit -= spi->dirty_offset;
range_end_bit -= spi->dirty_offset;

*start = page_offset(page) + range_start_bit * fs_info->sectorsize;
*end = page_offset(page) + range_end_bit * fs_info->sectorsize;
}
Expand Down Expand Up @@ -4602,12 +4606,11 @@ static int submit_eb_subpage(struct page *page,
int submitted = 0;
u64 page_start = page_offset(page);
int bit_start = 0;
const int nbits = BTRFS_SUBPAGE_BITMAP_SIZE;
int sectors_per_node = fs_info->nodesize >> fs_info->sectorsize_bits;
int ret;

/* Lock and write each dirty extent buffers in the range */
while (bit_start < nbits) {
while (bit_start < fs_info->subpage_info->bitmap_nr_bits) {
struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
struct extent_buffer *eb;
unsigned long flags;
Expand All @@ -4623,7 +4626,8 @@ static int submit_eb_subpage(struct page *page,
break;
}
spin_lock_irqsave(&subpage->lock, flags);
if (!((1 << bit_start) & subpage->dirty_bitmap)) {
if (!test_bit(bit_start + fs_info->subpage_info->dirty_offset,
subpage->bitmaps)) {
spin_unlock_irqrestore(&subpage->lock, flags);
spin_unlock(&page->mapping->private_lock);
bit_start++;
Expand Down Expand Up @@ -7169,32 +7173,41 @@ void memmove_extent_buffer(const struct extent_buffer *dst,
}
}

#define GANG_LOOKUP_SIZE 16
static struct extent_buffer *get_next_extent_buffer(
struct btrfs_fs_info *fs_info, struct page *page, u64 bytenr)
{
struct extent_buffer *gang[BTRFS_SUBPAGE_BITMAP_SIZE];
struct extent_buffer *gang[GANG_LOOKUP_SIZE];
struct extent_buffer *found = NULL;
u64 page_start = page_offset(page);
int ret;
int i;
u64 cur = page_start;

ASSERT(in_range(bytenr, page_start, PAGE_SIZE));
ASSERT(PAGE_SIZE / fs_info->nodesize <= BTRFS_SUBPAGE_BITMAP_SIZE);
lockdep_assert_held(&fs_info->buffer_lock);

ret = radix_tree_gang_lookup(&fs_info->buffer_radix, (void **)gang,
bytenr >> fs_info->sectorsize_bits,
PAGE_SIZE / fs_info->nodesize);
for (i = 0; i < ret; i++) {
/* Already beyond page end */
if (gang[i]->start >= page_start + PAGE_SIZE)
break;
/* Found one */
if (gang[i]->start >= bytenr) {
found = gang[i];
break;
while (cur < page_start + PAGE_SIZE) {
int ret;
int i;

ret = radix_tree_gang_lookup(&fs_info->buffer_radix,
(void **)gang, cur >> fs_info->sectorsize_bits,
min_t(unsigned int, GANG_LOOKUP_SIZE,
PAGE_SIZE / fs_info->nodesize));
if (ret == 0)
goto out;
for (i = 0; i < ret; i++) {
/* Already beyond page end */
if (gang[i]->start >= page_start + PAGE_SIZE)
goto out;
/* Found one */
if (gang[i]->start >= bytenr) {
found = gang[i];
goto out;
}
}
cur = gang[ret - 1]->start + gang[ret - 1]->len;
}
out:
return found;
}

Expand Down
126 changes: 81 additions & 45 deletions fs/btrfs/subpage.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,10 +133,13 @@ struct btrfs_subpage *btrfs_alloc_subpage(const struct btrfs_fs_info *fs_info,
enum btrfs_subpage_type type)
{
struct btrfs_subpage *ret;
unsigned int real_size;

ASSERT(fs_info->sectorsize < PAGE_SIZE);

ret = kzalloc(sizeof(struct btrfs_subpage), GFP_NOFS);
real_size = struct_size(ret, bitmaps,
BITS_TO_LONGS(fs_info->subpage_info->total_nr_bits));
ret = kzalloc(real_size, GFP_NOFS);
if (!ret)
return ERR_PTR(-ENOMEM);

Expand Down Expand Up @@ -319,37 +322,59 @@ void btrfs_page_end_writer_lock(const struct btrfs_fs_info *fs_info,
unlock_page(page);
}

/*
* Convert the [start, start + len) range into a u16 bitmap
*
* For example: if start == page_offset() + 16K, len = 16K, we get 0x00f0.
*/
static u16 btrfs_subpage_calc_bitmap(const struct btrfs_fs_info *fs_info,
struct page *page, u64 start, u32 len)
static bool bitmap_test_range_all_set(unsigned long *addr, unsigned int start,
unsigned int nbits)
{
const int bit_start = offset_in_page(start) >> fs_info->sectorsize_bits;
const int nbits = len >> fs_info->sectorsize_bits;
unsigned int found_zero;

btrfs_subpage_assert(fs_info, page, start, len);
found_zero = find_next_zero_bit(addr, start + nbits, start);
if (found_zero == start + nbits)
return true;
return false;
}

/*
* Here nbits can be 16, thus can go beyond u16 range. We make the
* first left shift to be calculate in unsigned long (at least u32),
* then truncate the result to u16.
*/
return (u16)(((1UL << nbits) - 1) << bit_start);
static bool bitmap_test_range_all_zero(unsigned long *addr, unsigned int start,
unsigned int nbits)
{
unsigned int found_set;

found_set = find_next_bit(addr, start + nbits, start);
if (found_set == start + nbits)
return true;
return false;
}

#define subpage_calc_start_bit(fs_info, page, name, start, len) \
({ \
unsigned int start_bit; \
\
btrfs_subpage_assert(fs_info, page, start, len); \
start_bit = offset_in_page(start) >> fs_info->sectorsize_bits; \
start_bit += fs_info->subpage_info->name##_offset; \
start_bit; \
})

#define subpage_test_bitmap_all_set(fs_info, subpage, name) \
bitmap_test_range_all_set(subpage->bitmaps, \
fs_info->subpage_info->name##_offset, \
fs_info->subpage_info->bitmap_nr_bits)

#define subpage_test_bitmap_all_zero(fs_info, subpage, name) \
bitmap_test_range_all_zero(subpage->bitmaps, \
fs_info->subpage_info->name##_offset, \
fs_info->subpage_info->bitmap_nr_bits)

void btrfs_subpage_set_uptodate(const struct btrfs_fs_info *fs_info,
struct page *page, u64 start, u32 len)
{
struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len);
unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
uptodate, start, len);
unsigned long flags;

spin_lock_irqsave(&subpage->lock, flags);
subpage->uptodate_bitmap |= tmp;
if (subpage->uptodate_bitmap == U16_MAX)
bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
if (subpage_test_bitmap_all_set(fs_info, subpage, uptodate))
SetPageUptodate(page);
spin_unlock_irqrestore(&subpage->lock, flags);
}
Expand All @@ -358,11 +383,12 @@ void btrfs_subpage_clear_uptodate(const struct btrfs_fs_info *fs_info,
struct page *page, u64 start, u32 len)
{
struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len);
unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
uptodate, start, len);
unsigned long flags;

spin_lock_irqsave(&subpage->lock, flags);
subpage->uptodate_bitmap &= ~tmp;
bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
ClearPageUptodate(page);
spin_unlock_irqrestore(&subpage->lock, flags);
}
Expand All @@ -371,11 +397,12 @@ void btrfs_subpage_set_error(const struct btrfs_fs_info *fs_info,
struct page *page, u64 start, u32 len)
{
struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len);
unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
error, start, len);
unsigned long flags;

spin_lock_irqsave(&subpage->lock, flags);
subpage->error_bitmap |= tmp;
bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
SetPageError(page);
spin_unlock_irqrestore(&subpage->lock, flags);
}
Expand All @@ -384,12 +411,13 @@ void btrfs_subpage_clear_error(const struct btrfs_fs_info *fs_info,
struct page *page, u64 start, u32 len)
{
struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len);
unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
error, start, len);
unsigned long flags;

spin_lock_irqsave(&subpage->lock, flags);
subpage->error_bitmap &= ~tmp;
if (subpage->error_bitmap == 0)
bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
if (subpage_test_bitmap_all_zero(fs_info, subpage, error))
ClearPageError(page);
spin_unlock_irqrestore(&subpage->lock, flags);
}
Expand All @@ -398,11 +426,12 @@ void btrfs_subpage_set_dirty(const struct btrfs_fs_info *fs_info,
struct page *page, u64 start, u32 len)
{
struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len);
unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
dirty, start, len);
unsigned long flags;

spin_lock_irqsave(&subpage->lock, flags);
subpage->dirty_bitmap |= tmp;
bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
spin_unlock_irqrestore(&subpage->lock, flags);
set_page_dirty(page);
}
Expand All @@ -421,13 +450,14 @@ bool btrfs_subpage_clear_and_test_dirty(const struct btrfs_fs_info *fs_info,
struct page *page, u64 start, u32 len)
{
struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len);
unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
dirty, start, len);
unsigned long flags;
bool last = false;

spin_lock_irqsave(&subpage->lock, flags);
subpage->dirty_bitmap &= ~tmp;
if (subpage->dirty_bitmap == 0)
bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
if (subpage_test_bitmap_all_zero(fs_info, subpage, dirty))
last = true;
spin_unlock_irqrestore(&subpage->lock, flags);
return last;
Expand All @@ -447,11 +477,12 @@ void btrfs_subpage_set_writeback(const struct btrfs_fs_info *fs_info,
struct page *page, u64 start, u32 len)
{
struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len);
unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
writeback, start, len);
unsigned long flags;

spin_lock_irqsave(&subpage->lock, flags);
subpage->writeback_bitmap |= tmp;
bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
set_page_writeback(page);
spin_unlock_irqrestore(&subpage->lock, flags);
}
Expand All @@ -460,12 +491,13 @@ void btrfs_subpage_clear_writeback(const struct btrfs_fs_info *fs_info,
struct page *page, u64 start, u32 len)
{
struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len);
unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
writeback, start, len);
unsigned long flags;

spin_lock_irqsave(&subpage->lock, flags);
subpage->writeback_bitmap &= ~tmp;
if (subpage->writeback_bitmap == 0) {
bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
if (subpage_test_bitmap_all_zero(fs_info, subpage, writeback)) {
ASSERT(PageWriteback(page));
end_page_writeback(page);
}
Expand All @@ -476,11 +508,12 @@ void btrfs_subpage_set_ordered(const struct btrfs_fs_info *fs_info,
struct page *page, u64 start, u32 len)
{
struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len);
unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
ordered, start, len);
unsigned long flags;

spin_lock_irqsave(&subpage->lock, flags);
subpage->ordered_bitmap |= tmp;
bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
SetPageOrdered(page);
spin_unlock_irqrestore(&subpage->lock, flags);
}
Expand All @@ -489,12 +522,13 @@ void btrfs_subpage_clear_ordered(const struct btrfs_fs_info *fs_info,
struct page *page, u64 start, u32 len)
{
struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len);
unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
ordered, start, len);
unsigned long flags;

spin_lock_irqsave(&subpage->lock, flags);
subpage->ordered_bitmap &= ~tmp;
if (subpage->ordered_bitmap == 0)
bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
if (subpage_test_bitmap_all_zero(fs_info, subpage, ordered))
ClearPageOrdered(page);
spin_unlock_irqrestore(&subpage->lock, flags);
}
Expand All @@ -507,12 +541,14 @@ bool btrfs_subpage_test_##name(const struct btrfs_fs_info *fs_info, \
struct page *page, u64 start, u32 len) \
{ \
struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; \
const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); \
unsigned int start_bit = subpage_calc_start_bit(fs_info, page, \
name, start, len); \
unsigned long flags; \
bool ret; \
\
spin_lock_irqsave(&subpage->lock, flags); \
ret = ((subpage->name##_bitmap & tmp) == tmp); \
ret = bitmap_test_range_all_set(subpage->bitmaps, start_bit, \
len >> fs_info->sectorsize_bits); \
spin_unlock_irqrestore(&subpage->lock, flags); \
return ret; \
}
Expand Down Expand Up @@ -609,5 +645,5 @@ void btrfs_page_assert_not_dirty(const struct btrfs_fs_info *fs_info,
return;

ASSERT(PagePrivate(page) && page->private);
ASSERT(subpage->dirty_bitmap == 0);
ASSERT(subpage_test_bitmap_all_zero(fs_info, subpage, dirty));
}
Loading

0 comments on commit 72a69cd

Please sign in to comment.