Skip to content

Commit

Permalink
net/smc: llc_conf_mutex refactor, replace it with rw_semaphore
Browse files Browse the repository at this point in the history 
llc_conf_mutex was used to protect links and link related configurations
in the same link group, for example, add or delete links. However,
in most cases, the protected critical area has only read semantics and
with no write semantics at all, such as obtaining a usable link or an
available rmb_desc.

This patch do simply code refactoring, replace mutex with rw_semaphore,
replace mutex_lock with down_write and replace mutex_unlock with
up_write.

Theoretically, this replacement is equivalent, but after this patch,
we can distinguish lock granularity according to different semantics
of critical areas.

Signed-off-by: D. Wythe <alibuda@linux.alibaba.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
  • Loading branch information
D. Wythe authored and David S. Miller committed Feb 4, 2023
1 parent 88c940c commit b5dd4d6
Show file tree
Hide file tree
Showing 4 changed files with 24 additions and 24 deletions.
8 changes: 4 additions & 4 deletions net/smc/af_smc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -502,15 +502,15 @@ static int smcr_lgr_reg_sndbufs(struct smc_link *link,
return -EINVAL;

/* protect against parallel smcr_link_reg_buf() */
mutex_lock(&lgr->llc_conf_mutex);
down_write(&lgr->llc_conf_mutex);
for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
if (!smc_link_active(&lgr->lnk[i]))
continue;
rc = smcr_link_reg_buf(&lgr->lnk[i], snd_desc);
if (rc)
break;
}
mutex_unlock(&lgr->llc_conf_mutex);
up_write(&lgr->llc_conf_mutex);
return rc;
}

Expand All @@ -527,7 +527,7 @@ static int smcr_lgr_reg_rmbs(struct smc_link *link,
/* protect against parallel smc_llc_cli_rkey_exchange() and
* parallel smcr_link_reg_buf()
*/
mutex_lock(&lgr->llc_conf_mutex);
down_write(&lgr->llc_conf_mutex);
for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
if (!smc_link_active(&lgr->lnk[i]))
continue;
Expand All @@ -544,7 +544,7 @@ static int smcr_lgr_reg_rmbs(struct smc_link *link,
}
rmb_desc->is_conf_rkey = true;
out:
mutex_unlock(&lgr->llc_conf_mutex);
up_write(&lgr->llc_conf_mutex);
smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
return rc;
}
Expand Down
20 changes: 10 additions & 10 deletions net/smc/smc_core.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1106,10 +1106,10 @@ static void smcr_buf_unuse(struct smc_buf_desc *buf_desc, bool is_rmb,
rc = smc_llc_flow_initiate(lgr, SMC_LLC_FLOW_RKEY);
if (!rc) {
/* protect against smc_llc_cli_rkey_exchange() */
mutex_lock(&lgr->llc_conf_mutex);
down_write(&lgr->llc_conf_mutex);
smc_llc_do_delete_rkey(lgr, buf_desc);
buf_desc->is_conf_rkey = false;
mutex_unlock(&lgr->llc_conf_mutex);
up_write(&lgr->llc_conf_mutex);
smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
}
}
Expand Down Expand Up @@ -1377,12 +1377,12 @@ static void smc_lgr_free(struct smc_link_group *lgr)
int i;

if (!lgr->is_smcd) {
mutex_lock(&lgr->llc_conf_mutex);
down_write(&lgr->llc_conf_mutex);
for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
if (lgr->lnk[i].state != SMC_LNK_UNUSED)
smcr_link_clear(&lgr->lnk[i], false);
}
mutex_unlock(&lgr->llc_conf_mutex);
up_write(&lgr->llc_conf_mutex);
smc_llc_lgr_clear(lgr);
}

Expand Down Expand Up @@ -1696,12 +1696,12 @@ static void smcr_link_down(struct smc_link *lnk)
} else {
if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) {
/* another llc task is ongoing */
mutex_unlock(&lgr->llc_conf_mutex);
up_write(&lgr->llc_conf_mutex);
wait_event_timeout(lgr->llc_flow_waiter,
(list_empty(&lgr->list) ||
lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE),
SMC_LLC_WAIT_TIME);
mutex_lock(&lgr->llc_conf_mutex);
down_write(&lgr->llc_conf_mutex);
}
if (!list_empty(&lgr->list)) {
smc_llc_send_delete_link(to_lnk, del_link_id,
Expand Down Expand Up @@ -1761,9 +1761,9 @@ static void smc_link_down_work(struct work_struct *work)
if (list_empty(&lgr->list))
return;
wake_up_all(&lgr->llc_msg_waiter);
mutex_lock(&lgr->llc_conf_mutex);
down_write(&lgr->llc_conf_mutex);
smcr_link_down(link);
mutex_unlock(&lgr->llc_conf_mutex);
up_write(&lgr->llc_conf_mutex);
}

static int smc_vlan_by_tcpsk_walk(struct net_device *lower_dev,
Expand Down Expand Up @@ -2247,7 +2247,7 @@ static int smcr_buf_map_usable_links(struct smc_link_group *lgr,
int i, rc = 0, cnt = 0;

/* protect against parallel link reconfiguration */
mutex_lock(&lgr->llc_conf_mutex);
down_write(&lgr->llc_conf_mutex);
for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
struct smc_link *lnk = &lgr->lnk[i];

Expand All @@ -2260,7 +2260,7 @@ static int smcr_buf_map_usable_links(struct smc_link_group *lgr,
cnt++;
}
out:
mutex_unlock(&lgr->llc_conf_mutex);
up_write(&lgr->llc_conf_mutex);
if (!rc && !cnt)
rc = -EINVAL;
return rc;
Expand Down
2 changes: 1 addition & 1 deletion net/smc/smc_core.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -298,7 +298,7 @@ struct smc_link_group {
/* queue for llc events */
spinlock_t llc_event_q_lock;
/* protects llc_event_q */
struct mutex llc_conf_mutex;
struct rw_semaphore llc_conf_mutex;
/* protects lgr reconfig. */
struct work_struct llc_add_link_work;
struct work_struct llc_del_link_work;
Expand Down
18 changes: 9 additions & 9 deletions net/smc/smc_llc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1202,12 +1202,12 @@ static void smc_llc_process_cli_add_link(struct smc_link_group *lgr)

qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);

mutex_lock(&lgr->llc_conf_mutex);
down_write(&lgr->llc_conf_mutex);
if (smc_llc_is_local_add_link(&qentry->msg))
smc_llc_cli_add_link_invite(qentry->link, qentry);
else
smc_llc_cli_add_link(qentry->link, qentry);
mutex_unlock(&lgr->llc_conf_mutex);
up_write(&lgr->llc_conf_mutex);
}

static int smc_llc_active_link_count(struct smc_link_group *lgr)
Expand Down Expand Up @@ -1509,13 +1509,13 @@ static void smc_llc_process_srv_add_link(struct smc_link_group *lgr)

qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);

mutex_lock(&lgr->llc_conf_mutex);
down_write(&lgr->llc_conf_mutex);
rc = smc_llc_srv_add_link(link, qentry);
if (!rc && lgr->type == SMC_LGR_SYMMETRIC) {
/* delete any asymmetric link */
smc_llc_delete_asym_link(lgr);
}
mutex_unlock(&lgr->llc_conf_mutex);
up_write(&lgr->llc_conf_mutex);
kfree(qentry);
}

Expand Down Expand Up @@ -1582,7 +1582,7 @@ static void smc_llc_process_cli_delete_link(struct smc_link_group *lgr)
smc_lgr_terminate_sched(lgr);
goto out;
}
mutex_lock(&lgr->llc_conf_mutex);
down_write(&lgr->llc_conf_mutex);
/* delete single link */
for (lnk_idx = 0; lnk_idx < SMC_LINKS_PER_LGR_MAX; lnk_idx++) {
if (lgr->lnk[lnk_idx].link_id != del_llc->link_num)
Expand Down Expand Up @@ -1616,7 +1616,7 @@ static void smc_llc_process_cli_delete_link(struct smc_link_group *lgr)
smc_lgr_terminate_sched(lgr);
}
out_unlock:
mutex_unlock(&lgr->llc_conf_mutex);
up_write(&lgr->llc_conf_mutex);
out:
kfree(qentry);
}
Expand Down Expand Up @@ -1652,7 +1652,7 @@ static void smc_llc_process_srv_delete_link(struct smc_link_group *lgr)
int active_links;
int i;

mutex_lock(&lgr->llc_conf_mutex);
down_write(&lgr->llc_conf_mutex);
qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
lnk = qentry->link;
del_llc = &qentry->msg.delete_link;
Expand Down Expand Up @@ -1708,7 +1708,7 @@ static void smc_llc_process_srv_delete_link(struct smc_link_group *lgr)
smc_llc_add_link_local(lnk);
}
out:
mutex_unlock(&lgr->llc_conf_mutex);
up_write(&lgr->llc_conf_mutex);
kfree(qentry);
}

Expand Down Expand Up @@ -2126,7 +2126,7 @@ void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc)
spin_lock_init(&lgr->llc_flow_lock);
init_waitqueue_head(&lgr->llc_flow_waiter);
init_waitqueue_head(&lgr->llc_msg_waiter);
mutex_init(&lgr->llc_conf_mutex);
init_rwsem(&lgr->llc_conf_mutex);
lgr->llc_testlink_time = READ_ONCE(net->smc.sysctl_smcr_testlink_time);
}

Expand Down

0 comments on commit b5dd4d6

Please sign in to comment.