Skip to content

Commit

Permalink
ext4: use io_end for multiple bios
Browse files Browse the repository at this point in the history 
Change writeback path to create just one io_end structure for the
extent to which we submit IO and share it among bios writing that
extent. This prevents needless splitting and joining of unwritten
extents when they cannot be submitted as a single bio.

Bugs in ENOMEM handling found by Linux File System Verification project
(linuxtesting.org) and fixed by Alexey Khoroshilov
<khoroshilov@ispras.ru>.

CC: Alexey Khoroshilov <khoroshilov@ispras.ru>
Signed-off-by: Jan Kara <jack@suse.cz>
Signed-off-by: "Theodore Ts'o" <tytso@mit.edu>
  • Loading branch information
Jan Kara authored and Theodore Ts'o committed Jun 4, 2013
1 parent 8af8eec commit 97a851e
Show file tree
Hide file tree
Showing 3 changed files with 142 additions and 85 deletions.
8 changes: 6 additions & 2 deletions fs/ext4/ext4.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,7 @@ typedef struct ext4_io_end {
ssize_t size; /* size of the extent */
struct kiocb *iocb; /* iocb struct for AIO */
int result; /* error value for AIO */
atomic_t count; /* reference counter */
} ext4_io_end_t;

struct ext4_io_submit {
Expand Down Expand Up @@ -2648,11 +2649,14 @@ extern int ext4_move_extents(struct file *o_filp, struct file *d_filp,

/* page-io.c */
extern int __init ext4_init_pageio(void);
extern void ext4_add_complete_io(ext4_io_end_t *io_end);
extern void ext4_exit_pageio(void);
extern void ext4_ioend_shutdown(struct inode *);
extern void ext4_free_io_end(ext4_io_end_t *io);
extern ext4_io_end_t *ext4_init_io_end(struct inode *inode, gfp_t flags);
extern ext4_io_end_t *ext4_get_io_end(ext4_io_end_t *io_end);
extern int ext4_put_io_end(ext4_io_end_t *io_end);
extern void ext4_put_io_end_defer(ext4_io_end_t *io_end);
extern void ext4_io_submit_init(struct ext4_io_submit *io,
struct writeback_control *wbc);
extern void ext4_end_io_work(struct work_struct *work);
extern void ext4_io_submit(struct ext4_io_submit *io);
extern int ext4_bio_write_page(struct ext4_io_submit *io,
Expand Down
98 changes: 60 additions & 38 deletions fs/ext4/inode.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1465,6 +1465,8 @@ static void ext4_da_page_release_reservation(struct page *page,
* Delayed allocation stuff
*/

static void ext4_da_block_invalidatepages(struct mpage_da_data *mpd);

/*
* mpage_da_submit_io - walks through extent of pages and try to write
* them with writepage() call back
Expand Down Expand Up @@ -1493,7 +1495,12 @@ static int mpage_da_submit_io(struct mpage_da_data *mpd,
struct ext4_io_submit io_submit;

BUG_ON(mpd->next_page <= mpd->first_page);
memset(&io_submit, 0, sizeof(io_submit));
ext4_io_submit_init(&io_submit, mpd->wbc);
io_submit.io_end = ext4_init_io_end(inode, GFP_NOFS);
if (!io_submit.io_end) {
ext4_da_block_invalidatepages(mpd);
return -ENOMEM;
}
/*
* We need to start from the first_page to the next_page - 1
* to make sure we also write the mapped dirty buffer_heads.
Expand Down Expand Up @@ -1581,6 +1588,8 @@ static int mpage_da_submit_io(struct mpage_da_data *mpd,
pagevec_release(&pvec);
}
ext4_io_submit(&io_submit);
/* Drop io_end reference we got from init */
ext4_put_io_end_defer(io_submit.io_end);
return ret;
}

Expand Down Expand Up @@ -2239,9 +2248,17 @@ static int ext4_writepage(struct page *page,
*/
return __ext4_journalled_writepage(page, len);

memset(&io_submit, 0, sizeof(io_submit));
ext4_io_submit_init(&io_submit, wbc);
io_submit.io_end = ext4_init_io_end(inode, GFP_NOFS);
if (!io_submit.io_end) {
redirty_page_for_writepage(wbc, page);
unlock_page(page);
return -ENOMEM;
}
ret = ext4_bio_write_page(&io_submit, page, len, wbc);
ext4_io_submit(&io_submit);
/* Drop io_end reference we got from init */
ext4_put_io_end_defer(io_submit.io_end);
return ret;
}

Expand Down Expand Up @@ -3076,35 +3093,27 @@ static void ext4_end_io_dio(struct kiocb *iocb, loff_t offset,
struct inode *inode = file_inode(iocb->ki_filp);
ext4_io_end_t *io_end = iocb->private;

/* if not async direct IO or dio with 0 bytes write, just return */
if (!io_end || !size)
goto out;
/* if not async direct IO just return */
if (!io_end) {
inode_dio_done(inode);
if (is_async)
aio_complete(iocb, ret, 0);
return;
}

ext_debug("ext4_end_io_dio(): io_end 0x%p "
"for inode %lu, iocb 0x%p, offset %llu, size %zd\n",
iocb->private, io_end->inode->i_ino, iocb, offset,
size);

iocb->private = NULL;

/* if not aio dio with unwritten extents, just free io and return */
if (!(io_end->flag & EXT4_IO_END_UNWRITTEN)) {
ext4_free_io_end(io_end);
out:
inode_dio_done(inode);
if (is_async)
aio_complete(iocb, ret, 0);
return;
}

io_end->offset = offset;
io_end->size = size;
if (is_async) {
io_end->iocb = iocb;
io_end->result = ret;
}

ext4_add_complete_io(io_end);
ext4_put_io_end_defer(io_end);
}

/*
Expand Down Expand Up @@ -3138,6 +3147,7 @@ static ssize_t ext4_ext_direct_IO(int rw, struct kiocb *iocb,
get_block_t *get_block_func = NULL;
int dio_flags = 0;
loff_t final_size = offset + count;
ext4_io_end_t *io_end = NULL;

/* Use the old path for reads and writes beyond i_size. */
if (rw != WRITE || final_size > inode->i_size)
Expand Down Expand Up @@ -3176,13 +3186,16 @@ static ssize_t ext4_ext_direct_IO(int rw, struct kiocb *iocb,
iocb->private = NULL;
ext4_inode_aio_set(inode, NULL);
if (!is_sync_kiocb(iocb)) {
ext4_io_end_t *io_end = ext4_init_io_end(inode, GFP_NOFS);
io_end = ext4_init_io_end(inode, GFP_NOFS);
if (!io_end) {
ret = -ENOMEM;
goto retake_lock;
}
io_end->flag |= EXT4_IO_END_DIRECT;
iocb->private = io_end;
/*
* Grab reference for DIO. Will be dropped in ext4_end_io_dio()
*/
iocb->private = ext4_get_io_end(io_end);
/*
* we save the io structure for current async direct
* IO, so that later ext4_map_blocks() could flag the
Expand All @@ -3206,26 +3219,35 @@ static ssize_t ext4_ext_direct_IO(int rw, struct kiocb *iocb,
NULL,
dio_flags);

if (iocb->private)
ext4_inode_aio_set(inode, NULL);
/*
* The io_end structure takes a reference to the inode, that
* structure needs to be destroyed and the reference to the
* inode need to be dropped, when IO is complete, even with 0
* byte write, or failed.
*
* In the successful AIO DIO case, the io_end structure will
* be destroyed and the reference to the inode will be dropped
* after the end_io call back function is called.
*
* In the case there is 0 byte write, or error case, since VFS
* direct IO won't invoke the end_io call back function, we
* need to free the end_io structure here.
* Put our reference to io_end. This can free the io_end structure e.g.
* in sync IO case or in case of error. It can even perform extent
* conversion if all bios we submitted finished before we got here.
* Note that in that case iocb->private can be already set to NULL
* here.
*/
if (ret != -EIOCBQUEUED && ret <= 0 && iocb->private) {
ext4_free_io_end(iocb->private);
iocb->private = NULL;
} else if (ret > 0 && !overwrite && ext4_test_inode_state(inode,
if (io_end) {
ext4_inode_aio_set(inode, NULL);
ext4_put_io_end(io_end);
/*
* When no IO was submitted ext4_end_io_dio() was not
* called so we have to put iocb's reference.
*/
if (ret <= 0 && ret != -EIOCBQUEUED && iocb->private) {
WARN_ON(iocb->private != io_end);
WARN_ON(io_end->flag & EXT4_IO_END_UNWRITTEN);
WARN_ON(io_end->iocb);
/*
* Generic code already did inode_dio_done() so we
* have to clear EXT4_IO_END_DIRECT to not do it for
* the second time.
*/
io_end->flag = 0;
ext4_put_io_end(io_end);
iocb->private = NULL;
}
}
if (ret > 0 && !overwrite && ext4_test_inode_state(inode,
EXT4_STATE_DIO_UNWRITTEN)) {
int err;
/*
Expand Down
Loading

0 comments on commit 97a851e

Please sign in to comment.