Skip to content

Commit

Permalink
GFS2: Reduce file fragmentation
Browse files Browse the repository at this point in the history 
This patch reduces GFS2 file fragmentation by pre-reserving blocks. The
resulting improved on disk layout greatly speeds up operations in cases
which would have resulted in interlaced allocation of blocks previously.
A typical example of this is 10 parallel dd processes, each writing to a
file in a common dirctory.

The implementation uses an rbtree of reservations attached to each
resource group (and each inode).

Signed-off-by: Bob Peterson <rpeterso@redhat.com>
Signed-off-by: Steven Whitehouse <swhiteho@redhat.com>
  • Loading branch information
Bob Peterson authored and Steven Whitehouse committed Jul 19, 2012
1 parent 294f2ad commit 8e2e004
Show file tree
Hide file tree
Showing 9 changed files with 708 additions and 92 deletions.
3 changes: 3 additions & 0 deletions fs/gfs2/bmap.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -785,6 +785,9 @@ static int do_strip(struct gfs2_inode *ip, struct buffer_head *dibh,
if (error)
goto out_rlist;

if (gfs2_rs_active(ip->i_res)) /* needs to be done with the rgrp glock held */
gfs2_rs_deltree(ip->i_res);

error = gfs2_trans_begin(sdp, rg_blocks + RES_DINODE +
RES_INDIRECT + RES_STATFS + RES_QUOTA,
revokes);
Expand Down
24 changes: 11 additions & 13 deletions fs/gfs2/file.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -383,6 +383,9 @@ static int gfs2_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf)
if (ret)
return ret;

atomic_set(&ip->i_res->rs_sizehint,
PAGE_CACHE_SIZE / sdp->sd_sb.sb_bsize);

gfs2_holder_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, &gh);
ret = gfs2_glock_nq(&gh);
if (ret)
Expand Down Expand Up @@ -571,22 +574,15 @@ static int gfs2_open(struct inode *inode, struct file *file)

static int gfs2_release(struct inode *inode, struct file *file)
{
struct gfs2_sbd *sdp = inode->i_sb->s_fs_info;
struct gfs2_file *fp;
struct gfs2_inode *ip = GFS2_I(inode);

fp = file->private_data;
kfree(file->private_data);
file->private_data = NULL;

if ((file->f_mode & FMODE_WRITE) && ip->i_res &&
if ((file->f_mode & FMODE_WRITE) &&
(atomic_read(&inode->i_writecount) == 1))
gfs2_rs_delete(ip);

if (gfs2_assert_warn(sdp, fp))
return -EIO;

kfree(fp);

return 0;
}

Expand Down Expand Up @@ -662,14 +658,18 @@ static ssize_t gfs2_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
unsigned long nr_segs, loff_t pos)
{
struct file *file = iocb->ki_filp;
size_t writesize = iov_length(iov, nr_segs);
struct dentry *dentry = file->f_dentry;
struct gfs2_inode *ip = GFS2_I(dentry->d_inode);
struct gfs2_sbd *sdp;
int ret;

sdp = GFS2_SB(file->f_mapping->host);
ret = gfs2_rs_alloc(ip);
if (ret)
return ret;

atomic_set(&ip->i_res->rs_sizehint, writesize / sdp->sd_sb.sb_bsize);
if (file->f_flags & O_APPEND) {
struct gfs2_holder gh;

Expand Down Expand Up @@ -795,6 +795,8 @@ static long gfs2_fallocate(struct file *file, int mode, loff_t offset,
if (unlikely(error))
goto out_uninit;

atomic_set(&ip->i_res->rs_sizehint, len / sdp->sd_sb.sb_bsize);

while (len > 0) {
if (len < bytes)
bytes = len;
Expand All @@ -803,10 +805,6 @@ static long gfs2_fallocate(struct file *file, int mode, loff_t offset,
offset += bytes;
continue;
}
error = gfs2_rindex_update(sdp);
if (error)
goto out_unlock;

error = gfs2_quota_lock_check(ip);
if (error)
goto out_unlock;
Expand Down
49 changes: 37 additions & 12 deletions fs/gfs2/incore.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ struct gfs2_rgrpd {
u32 rd_data; /* num of data blocks in rgrp */
u32 rd_bitbytes; /* number of bytes in data bitmaps */
u32 rd_free;
u32 rd_reserved; /* number of blocks reserved */
u32 rd_free_clone;
u32 rd_dinodes;
u64 rd_igeneration;
Expand All @@ -96,6 +97,9 @@ struct gfs2_rgrpd {
#define GFS2_RDF_UPTODATE 0x20000000 /* rg is up to date */
#define GFS2_RDF_ERROR 0x40000000 /* error in rg */
#define GFS2_RDF_MASK 0xf0000000 /* mask for internal flags */
spinlock_t rd_rsspin; /* protects reservation related vars */
struct rb_root rd_rstree; /* multi-block reservation tree */
u32 rd_rs_cnt; /* count of current reservations */
};

enum gfs2_state_bits {
Expand Down Expand Up @@ -233,6 +237,38 @@ struct gfs2_holder {
unsigned long gh_ip;
};

/* Resource group multi-block reservation, in order of appearance:
Step 1. Function prepares to write, allocates a mb, sets the size hint.
Step 2. User calls inplace_reserve to target an rgrp, sets the rgrp info
Step 3. Function get_local_rgrp locks the rgrp, determines which bits to use
Step 4. Bits are assigned from the rgrp based on either the reservation
or wherever it can.
*/

struct gfs2_blkreserv {
/* components used during write (step 1): */
atomic_t rs_sizehint; /* hint of the write size */

/* components used during inplace_reserve (step 2): */
u32 rs_requested; /* Filled in by caller of gfs2_inplace_reserve() */

/* components used during get_local_rgrp (step 3): */
struct gfs2_rgrpd *rs_rgd; /* pointer to the gfs2_rgrpd */
struct gfs2_holder rs_rgd_gh; /* Filled in by get_local_rgrp */
struct rb_node rs_node; /* link to other block reservations */

/* components used during block searches and assignments (step 4): */
struct gfs2_bitmap *rs_bi; /* bitmap for the current allocation */
u32 rs_biblk; /* start block relative to the bi */
u32 rs_free; /* how many blocks are still free */

/* ancillary quota stuff */
struct gfs2_quota_data *rs_qa_qd[2 * MAXQUOTAS];
struct gfs2_holder rs_qa_qd_ghs[2 * MAXQUOTAS];
unsigned int rs_qa_qd_num;
};

enum {
GLF_LOCK = 1,
GLF_DEMOTE = 3,
Expand Down Expand Up @@ -290,24 +326,13 @@ struct gfs2_glock {

#define GFS2_MIN_LVB_SIZE 32 /* Min size of LVB that gfs2 supports */

struct gfs2_blkreserv {
u32 rs_requested; /* Filled in by caller of gfs2_inplace_reserve() */
struct gfs2_holder rs_rgd_gh; /* Filled in by gfs2_inplace_reserve() */

/* ancillary quota stuff */
struct gfs2_quota_data *rs_qa_qd[2 * MAXQUOTAS];
struct gfs2_holder rs_qa_qd_ghs[2 * MAXQUOTAS];
unsigned int rs_qa_qd_num;
};

enum {
GIF_INVALID = 0,
GIF_QD_LOCKED = 1,
GIF_ALLOC_FAILED = 2,
GIF_SW_PAGED = 3,
};


struct gfs2_inode {
struct inode i_inode;
u64 i_no_addr;
Expand All @@ -318,7 +343,7 @@ struct gfs2_inode {
struct gfs2_glock *i_gl; /* Move into i_gh? */
struct gfs2_holder i_iopen_gh;
struct gfs2_holder i_gh; /* for prepare/commit_write only */
struct gfs2_blkreserv *i_res; /* resource group block reservation */
struct gfs2_blkreserv *i_res; /* rgrp multi-block reservation */
struct gfs2_rgrpd *i_rgd;
u64 i_goal; /* goal block for allocations */
struct rw_semaphore i_rw_mutex;
Expand Down
37 changes: 28 additions & 9 deletions fs/gfs2/inode.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -521,6 +521,9 @@ static int make_dinode(struct gfs2_inode *dip, struct gfs2_glock *gl,
int error;

munge_mode_uid_gid(dip, &mode, &uid, &gid);
error = gfs2_rindex_update(sdp);
if (error)
return error;

error = gfs2_quota_lock(dip, uid, gid);
if (error)
Expand Down Expand Up @@ -551,6 +554,10 @@ static int link_dinode(struct gfs2_inode *dip, const struct qstr *name,
struct buffer_head *dibh;
int error;

error = gfs2_rindex_update(sdp);
if (error)
return error;

error = gfs2_quota_lock(dip, NO_QUOTA_CHANGE, NO_QUOTA_CHANGE);
if (error)
goto fail;
Expand Down Expand Up @@ -596,7 +603,8 @@ static int link_dinode(struct gfs2_inode *dip, const struct qstr *name,
gfs2_trans_end(sdp);

fail_ipreserv:
gfs2_inplace_release(dip);
if (alloc_required)
gfs2_inplace_release(dip);

fail_quota_locks:
gfs2_quota_unlock(dip);
Expand Down Expand Up @@ -647,7 +655,7 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry,
const struct qstr *name = &dentry->d_name;
struct gfs2_holder ghs[2];
struct inode *inode = NULL;
struct gfs2_inode *dip = GFS2_I(dir);
struct gfs2_inode *dip = GFS2_I(dir), *ip;
struct gfs2_sbd *sdp = GFS2_SB(&dip->i_inode);
struct gfs2_inum_host inum = { .no_addr = 0, .no_formal_ino = 0 };
int error;
Expand All @@ -657,6 +665,11 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry,
if (!name->len || name->len > GFS2_FNAMESIZE)
return -ENAMETOOLONG;

/* We need a reservation to allocate the new dinode block. The
directory ip temporarily points to the reservation, but this is
being done to get a set of contiguous blocks for the new dinode.
Since this is a create, we don't have a sizehint yet, so it will
have to use the minimum reservation size. */
error = gfs2_rs_alloc(dip);
if (error)
return error;
Expand Down Expand Up @@ -694,24 +707,29 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry,
if (IS_ERR(inode))
goto fail_gunlock2;

error = gfs2_inode_refresh(GFS2_I(inode));
ip = GFS2_I(inode);
error = gfs2_inode_refresh(ip);
if (error)
goto fail_gunlock2;

/* the new inode needs a reservation so it can allocate xattrs. */
error = gfs2_rs_alloc(GFS2_I(inode));
if (error)
goto fail_gunlock2;
/* The newly created inode needs a reservation so it can allocate
xattrs. At the same time, we want new blocks allocated to the new
dinode to be as contiguous as possible. Since we allocated the
dinode block under the directory's reservation, we transfer
ownership of that reservation to the new inode. The directory
doesn't need a reservation unless it needs a new allocation. */
ip->i_res = dip->i_res;
dip->i_res = NULL;

error = gfs2_acl_create(dip, inode);
if (error)
goto fail_gunlock2;

error = gfs2_security_init(dip, GFS2_I(inode), name);
error = gfs2_security_init(dip, ip, name);
if (error)
goto fail_gunlock2;

error = link_dinode(dip, name, GFS2_I(inode));
error = link_dinode(dip, name, ip);
if (error)
goto fail_gunlock2;

Expand All @@ -738,6 +756,7 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry,
iput(inode);
}
fail:
gfs2_rs_delete(dip);
if (bh)
brelse(bh);
return error;
Expand Down
Loading

0 comments on commit 8e2e004

Please sign in to comment.