Skip to content

Commit

Permalink
drbd: Add struct drbd_resource
Browse files Browse the repository at this point in the history 
In a first step, each resource has exactly one connection, and both objects are
allocated at the same time.  The final result will be one resource and zero or
more connections.

Only allow to delete a resource if all its connections are C_STANDALONE.
Stop the worker threads of all connections early enough.

Signed-off-by: Andreas Gruenbacher <agruen@linbit.com>
Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
  • Loading branch information
Andreas Gruenbacher authored and Philipp Reisner committed Feb 17, 2014
1 parent 05a10ec commit 77c556f
Show file tree
Hide file tree
Showing 3 changed files with 179 additions and 78 deletions.
45 changes: 40 additions & 5 deletions drivers/block/drbd/drbd_int.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ struct drbd_connection;
#define DEV (disk_to_dev(device->vdisk))

#define conn_printk(LEVEL, TCONN, FMT, ARGS...) \
printk(LEVEL "d-con %s: " FMT, TCONN->name , ## ARGS)
printk(LEVEL "d-con %s: " FMT, TCONN->resource->name , ## ARGS)
#define conn_alert(TCONN, FMT, ARGS...) conn_printk(KERN_ALERT, TCONN, FMT, ## ARGS)
#define conn_crit(TCONN, FMT, ARGS...) conn_printk(KERN_CRIT, TCONN, FMT, ## ARGS)
#define conn_err(TCONN, FMT, ARGS...) conn_printk(KERN_ERR, TCONN, FMT, ## ARGS)
Expand Down Expand Up @@ -167,7 +167,7 @@ drbd_insert_fault(struct drbd_device *device, unsigned int type) {

extern struct ratelimit_state drbd_ratelimit_state;
extern struct idr drbd_devices; /* RCU, updates: genl_lock() */
extern struct list_head drbd_connections; /* RCU, updates: genl_lock() */
extern struct list_head drbd_resources; /* RCU, updates: genl_lock() */

extern const char *cmdname(enum drbd_packet cmd);

Expand Down Expand Up @@ -536,9 +536,16 @@ enum {
DISCONNECT_SENT,
};

struct drbd_connection { /* is a resource from the config file */
char *name; /* Resource name */
struct list_head connections; /* linked on global drbd_connections */
struct drbd_resource {
char *name;
struct kref kref;
struct list_head connections;
struct list_head resources;
};

struct drbd_connection {
struct list_head connections;
struct drbd_resource *resource;
struct kref kref;
struct idr volumes; /* <connection, vnr> to device mapping */
enum drbd_conns cstate; /* Only C_STANDALONE to C_WF_REPORT_PARAMS */
Expand Down Expand Up @@ -779,6 +786,24 @@ static inline struct drbd_peer_device *first_peer_device(struct drbd_device *dev
return list_first_entry(&device->peer_devices, struct drbd_peer_device, peer_devices);
}

#define for_each_resource(resource, _resources) \
list_for_each_entry(resource, _resources, resources)

#define for_each_resource_rcu(resource, _resources) \
list_for_each_entry_rcu(resource, _resources, resources)

#define for_each_resource_safe(resource, tmp, _resources) \
list_for_each_entry_safe(resource, tmp, _resources, resources)

#define for_each_connection(connection, resource) \
list_for_each_entry(connection, &resource->connections, connections)

#define for_each_connection_rcu(connection, resource) \
list_for_each_entry_rcu(connection, &resource->connections, connections)

#define for_each_connection_safe(connection, tmp, resource) \
list_for_each_entry_safe(connection, tmp, &resource->connections, connections)

#define for_each_peer_device(peer_device, device) \
list_for_each_entry(peer_device, &device->peer_devices, peer_devices)

Expand Down Expand Up @@ -1177,12 +1202,16 @@ extern int conn_lowest_minor(struct drbd_connection *connection);
enum drbd_ret_code drbd_create_minor(struct drbd_connection *connection, unsigned int minor, int vnr);
extern void drbd_destroy_device(struct kref *kref);

extern struct drbd_resource *drbd_create_resource(const char *name);
extern void drbd_free_resource(struct drbd_resource *resource);

extern int set_resource_options(struct drbd_connection *connection, struct res_opts *res_opts);
extern struct drbd_connection *conn_create(const char *name, struct res_opts *res_opts);
extern void drbd_destroy_connection(struct kref *kref);
struct drbd_connection *conn_get_by_name(const char *name);
extern struct drbd_connection *conn_get_by_addrs(void *my_addr, int my_addr_len,
void *peer_addr, int peer_addr_len);
extern void drbd_destroy_resource(struct kref *kref);
extern void conn_free_crypto(struct drbd_connection *connection);

extern int proc_details;
Expand Down Expand Up @@ -2082,4 +2111,10 @@ static inline void drbd_md_flush(struct drbd_device *device)
}
}

static inline struct drbd_connection *first_connection(struct drbd_resource *resource)
{
return list_first_entry(&resource->connections,
struct drbd_connection, connections);
}

#endif
96 changes: 71 additions & 25 deletions drivers/block/drbd/drbd_main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ module_param_string(usermode_helper, usermode_helper, sizeof(usermode_helper), 0
* as member "struct gendisk *vdisk;"
*/
struct idr drbd_devices;
struct list_head drbd_connections; /* list of struct drbd_connection */
struct list_head drbd_resources;

struct kmem_cache *drbd_request_cache;
struct kmem_cache *drbd_ee_cache; /* peer requests */
Expand Down Expand Up @@ -330,7 +330,8 @@ static int drbd_thread_setup(void *arg)
int retval;

snprintf(current->comm, sizeof(current->comm), "drbd_%c_%s",
thi->name[0], thi->connection->name);
thi->name[0],
thi->connection->resource->name);

restart:
retval = thi->function(thi);
Expand Down Expand Up @@ -411,7 +412,7 @@ int drbd_thread_start(struct drbd_thread *thi)
flush_signals(current); /* otherw. may get -ERESTARTNOINTR */

nt = kthread_create(drbd_thread_setup, (void *) thi,
"drbd_%c_%s", thi->name[0], thi->connection->name);
"drbd_%c_%s", thi->name[0], thi->connection->resource->name);

if (IS_ERR(nt)) {
conn_err(connection, "Couldn't start thread\n");
Expand Down Expand Up @@ -2276,12 +2277,31 @@ void drbd_restart_request(struct drbd_request *req)
queue_work(retry.wq, &retry.worker);
}

void drbd_destroy_resource(struct kref *kref)
{
struct drbd_resource *resource =
container_of(kref, struct drbd_resource, kref);

kfree(resource->name);
kfree(resource);
}

void drbd_free_resource(struct drbd_resource *resource)
{
struct drbd_connection *connection, *tmp;

for_each_connection_safe(connection, tmp, resource) {
list_del(&connection->connections);
kref_put(&connection->kref, drbd_destroy_connection);
}
kref_put(&resource->kref, drbd_destroy_resource);
}

static void drbd_cleanup(void)
{
unsigned int i;
struct drbd_device *device;
struct drbd_connection *connection, *tmp;
struct drbd_resource *resource, *tmp;

unregister_reboot_notifier(&drbd_notifier);

Expand Down Expand Up @@ -2311,10 +2331,9 @@ static void drbd_cleanup(void)
}

/* not _rcu since, no other updater anymore. Genl already unregistered */
list_for_each_entry_safe(connection, tmp, &drbd_connections, connections) {
list_del(&connection->connections); /* not _rcu no proc, not other threads */
/* synchronize_rcu(); */
kref_put(&connection->kref, drbd_destroy_connection);
for_each_resource_safe(resource, tmp, &drbd_resources) {
list_del(&resource->resources);
drbd_free_resource(resource);
}

drbd_destroy_mempools();
Expand Down Expand Up @@ -2391,13 +2410,15 @@ static void drbd_init_workqueue(struct drbd_work_queue* wq)
struct drbd_connection *conn_get_by_name(const char *name)
{
struct drbd_connection *connection;
struct drbd_resource *resource;

if (!name || !name[0])
return NULL;

rcu_read_lock();
list_for_each_entry_rcu(connection, &drbd_connections, connections) {
if (!strcmp(connection->name, name)) {
for_each_resource_rcu(resource, &drbd_resources) {
if (!strcmp(resource->name, name)) {
connection = first_connection(resource);
kref_get(&connection->kref);
goto found;
}
Expand All @@ -2411,16 +2432,19 @@ struct drbd_connection *conn_get_by_name(const char *name)
struct drbd_connection *conn_get_by_addrs(void *my_addr, int my_addr_len,
void *peer_addr, int peer_addr_len)
{
struct drbd_resource *resource;
struct drbd_connection *connection;

rcu_read_lock();
list_for_each_entry_rcu(connection, &drbd_connections, connections) {
if (connection->my_addr_len == my_addr_len &&
connection->peer_addr_len == peer_addr_len &&
!memcmp(&connection->my_addr, my_addr, my_addr_len) &&
!memcmp(&connection->peer_addr, peer_addr, peer_addr_len)) {
kref_get(&connection->kref);
goto found;
for_each_resource_rcu(resource, &drbd_resources) {
for_each_connection_rcu(connection, resource) {
if (connection->my_addr_len == my_addr_len &&
connection->peer_addr_len == peer_addr_len &&
!memcmp(&connection->my_addr, my_addr, my_addr_len) &&
!memcmp(&connection->peer_addr, peer_addr, peer_addr_len)) {
kref_get(&connection->kref);
goto found;
}
}
}
connection = NULL;
Expand Down Expand Up @@ -2506,19 +2530,34 @@ int set_resource_options(struct drbd_connection *connection, struct res_opts *re

}

struct drbd_resource *drbd_create_resource(const char *name)
{
struct drbd_resource *resource;

resource = kmalloc(sizeof(struct drbd_resource), GFP_KERNEL);
if (!resource)
return NULL;
resource->name = kstrdup(name, GFP_KERNEL);
if (!resource->name) {
kfree(resource);
return NULL;
}
kref_init(&resource->kref);
INIT_LIST_HEAD(&resource->connections);
list_add_tail_rcu(&resource->resources, &drbd_resources);
return resource;
}

/* caller must be under genl_lock() */
struct drbd_connection *conn_create(const char *name, struct res_opts *res_opts)
{
struct drbd_resource *resource;
struct drbd_connection *connection;

connection = kzalloc(sizeof(struct drbd_connection), GFP_KERNEL);
if (!connection)
return NULL;

connection->name = kstrdup(name, GFP_KERNEL);
if (!connection->name)
goto fail;

if (drbd_alloc_socket(&connection->data))
goto fail;
if (drbd_alloc_socket(&connection->meta))
Expand All @@ -2545,6 +2584,10 @@ struct drbd_connection *conn_create(const char *name, struct res_opts *res_opts)
connection->send.current_epoch_nr = 0;
connection->send.current_epoch_writes = 0;

resource = drbd_create_resource(name);
if (!resource)
goto fail;

connection->cstate = C_STANDALONE;
mutex_init(&connection->cstate_mutex);
spin_lock_init(&connection->req_lock);
Expand All @@ -2561,7 +2604,10 @@ struct drbd_connection *conn_create(const char *name, struct res_opts *res_opts)
drbd_thread_init(connection, &connection->asender, drbd_asender, "asender");

kref_init(&connection->kref);
list_add_tail_rcu(&connection->connections, &drbd_connections);

kref_get(&resource->kref);
connection->resource = resource;
list_add_tail_rcu(&connection->connections, &resource->connections);

return connection;

Expand All @@ -2570,7 +2616,6 @@ struct drbd_connection *conn_create(const char *name, struct res_opts *res_opts)
free_cpumask_var(connection->cpu_mask);
drbd_free_socket(&connection->meta);
drbd_free_socket(&connection->data);
kfree(connection->name);
kfree(connection);

return NULL;
Expand All @@ -2579,6 +2624,7 @@ struct drbd_connection *conn_create(const char *name, struct res_opts *res_opts)
void drbd_destroy_connection(struct kref *kref)
{
struct drbd_connection *connection = container_of(kref, struct drbd_connection, kref);
struct drbd_resource *resource = connection->resource;

if (atomic_read(&connection->current_epoch->epoch_size) != 0)
conn_err(connection, "epoch_size:%d\n", atomic_read(&connection->current_epoch->epoch_size));
Expand All @@ -2589,10 +2635,10 @@ void drbd_destroy_connection(struct kref *kref)
free_cpumask_var(connection->cpu_mask);
drbd_free_socket(&connection->meta);
drbd_free_socket(&connection->data);
kfree(connection->name);
kfree(connection->int_dig_in);
kfree(connection->int_dig_vv);
kfree(connection);
kref_put(&resource->kref, drbd_destroy_resource);
}

static int init_submitter(struct drbd_device *device)
Expand Down Expand Up @@ -2775,7 +2821,7 @@ int __init drbd_init(void)
idr_init(&drbd_devices);

rwlock_init(&global_state_lock);
INIT_LIST_HEAD(&drbd_connections);
INIT_LIST_HEAD(&drbd_resources);

err = drbd_genl_register();
if (err) {
Expand Down
Loading

0 comments on commit 77c556f

Please sign in to comment.