Skip to content

Commit

Permalink
scsi: target: iscsi: switch to using the crc32c library
Browse files Browse the repository at this point in the history 
Now that the crc32c() library function directly takes advantage of
architecture-specific optimizations, it is unnecessary to go through the
crypto API.  Just use crc32c().  This is much simpler, and it improves
performance due to eliminating the crypto API overhead.

Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Reviewed-by: Martin K. Petersen <martin.petersen@oracle.com>
Link: https://lore.kernel.org/r/20241202010844.144356-20-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@google.com>
  • Loading branch information
Eric Biggers committed Dec 2, 2024
1 parent 3ca4bec commit 31e4cdd
Show file tree
Hide file tree
Showing 6 changed files with 49 additions and 182 deletions.
3 changes: 2 additions & 1 deletion drivers/target/iscsi/Kconfig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@
config ISCSI_TARGET
tristate "SCSI Target Mode Stack"
depends on INET
select CRC32
select CRYPTO
select CRYPTO_CRC32C
select CRYPTO_HASH
help
Say M to enable the SCSI target mode stack. A SCSI target mode stack
is software that makes local storage available over a storage network
Expand Down
153 changes: 45 additions & 108 deletions drivers/target/iscsi/iscsi_target.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
*
******************************************************************************/

#include <crypto/hash.h>
#include <linux/crc32c.h>
#include <linux/string.h>
#include <linux/kthread.h>
#include <linux/completion.h>
Expand Down Expand Up @@ -490,8 +490,8 @@ void iscsit_aborted_task(struct iscsit_conn *conn, struct iscsit_cmd *cmd)
}
EXPORT_SYMBOL(iscsit_aborted_task);

static void iscsit_do_crypto_hash_buf(struct ahash_request *, const void *,
u32, u32, const void *, void *);
static u32 iscsit_crc_buf(const void *buf, u32 payload_length,
u32 padding, const void *pad_bytes);
static void iscsit_tx_thread_wait_for_tcp(struct iscsit_conn *);

static int
Expand All @@ -510,9 +510,7 @@ iscsit_xmit_nondatain_pdu(struct iscsit_conn *conn, struct iscsit_cmd *cmd,
if (conn->conn_ops->HeaderDigest) {
u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];

iscsit_do_crypto_hash_buf(conn->conn_tx_hash, hdr,
ISCSI_HDR_LEN, 0, NULL,
header_digest);
*header_digest = iscsit_crc_buf(hdr, ISCSI_HDR_LEN, 0, NULL);

iov[0].iov_len += ISCSI_CRC_LEN;
tx_size += ISCSI_CRC_LEN;
Expand All @@ -537,11 +535,9 @@ iscsit_xmit_nondatain_pdu(struct iscsit_conn *conn, struct iscsit_cmd *cmd,
}

if (conn->conn_ops->DataDigest) {
iscsit_do_crypto_hash_buf(conn->conn_tx_hash,
data_buf, data_buf_len,
padding, &cmd->pad_bytes,
&cmd->data_crc);

cmd->data_crc = iscsit_crc_buf(data_buf, data_buf_len,
padding,
&cmd->pad_bytes);
iov[niov].iov_base = &cmd->data_crc;
iov[niov++].iov_len = ISCSI_CRC_LEN;
tx_size += ISCSI_CRC_LEN;
Expand All @@ -566,8 +562,8 @@ iscsit_xmit_nondatain_pdu(struct iscsit_conn *conn, struct iscsit_cmd *cmd,
static int iscsit_map_iovec(struct iscsit_cmd *cmd, struct kvec *iov, int nvec,
u32 data_offset, u32 data_length);
static void iscsit_unmap_iovec(struct iscsit_cmd *);
static u32 iscsit_do_crypto_hash_sg(struct ahash_request *, struct iscsit_cmd *,
u32, u32, u32, u8 *);
static u32 iscsit_crc_sglist(const struct iscsit_cmd *cmd, u32 data_length,
u32 padding, const u8 *pad_bytes);
static int
iscsit_xmit_datain_pdu(struct iscsit_conn *conn, struct iscsit_cmd *cmd,
const struct iscsi_datain *datain)
Expand All @@ -584,10 +580,8 @@ iscsit_xmit_datain_pdu(struct iscsit_conn *conn, struct iscsit_cmd *cmd,
if (conn->conn_ops->HeaderDigest) {
u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];

iscsit_do_crypto_hash_buf(conn->conn_tx_hash, cmd->pdu,
ISCSI_HDR_LEN, 0, NULL,
header_digest);

*header_digest = iscsit_crc_buf(cmd->pdu, ISCSI_HDR_LEN, 0,
NULL);
iov[0].iov_len += ISCSI_CRC_LEN;
tx_size += ISCSI_CRC_LEN;

Expand All @@ -614,12 +608,8 @@ iscsit_xmit_datain_pdu(struct iscsit_conn *conn, struct iscsit_cmd *cmd,
}

if (conn->conn_ops->DataDigest) {
cmd->data_crc = iscsit_do_crypto_hash_sg(conn->conn_tx_hash,
cmd, datain->offset,
datain->length,
cmd->padding,
cmd->pad_bytes);

cmd->data_crc = iscsit_crc_sglist(cmd, datain->length,
cmd->padding, cmd->pad_bytes);
iov[iov_count].iov_base = &cmd->data_crc;
iov[iov_count++].iov_len = ISCSI_CRC_LEN;
tx_size += ISCSI_CRC_LEN;
Expand Down Expand Up @@ -1404,77 +1394,45 @@ iscsit_handle_scsi_cmd(struct iscsit_conn *conn, struct iscsit_cmd *cmd,
return iscsit_get_immediate_data(cmd, hdr, dump_payload);
}

static u32 iscsit_do_crypto_hash_sg(
struct ahash_request *hash,
struct iscsit_cmd *cmd,
u32 data_offset,
u32 data_length,
u32 padding,
u8 *pad_bytes)
static u32 iscsit_crc_sglist(const struct iscsit_cmd *cmd, u32 data_length,
u32 padding, const u8 *pad_bytes)
{
u32 data_crc;
struct scatterlist *sg;
unsigned int page_off;

crypto_ahash_init(hash);

sg = cmd->first_data_sg;
page_off = cmd->first_data_sg_off;

if (data_length && page_off) {
struct scatterlist first_sg;
u32 len = min_t(u32, data_length, sg->length - page_off);

sg_init_table(&first_sg, 1);
sg_set_page(&first_sg, sg_page(sg), len, sg->offset + page_off);

ahash_request_set_crypt(hash, &first_sg, NULL, len);
crypto_ahash_update(hash);

data_length -= len;
sg = sg_next(sg);
}
struct scatterlist *sg = cmd->first_data_sg;
unsigned int page_off = cmd->first_data_sg_off;
u32 crc = ~0;

while (data_length) {
u32 cur_len = min_t(u32, data_length, sg->length);
u32 cur_len = min_t(u32, data_length, sg->length - page_off);
const void *virt;

ahash_request_set_crypt(hash, sg, NULL, cur_len);
crypto_ahash_update(hash);
virt = kmap_local_page(sg_page(sg)) + sg->offset + page_off;
crc = crc32c(crc, virt, cur_len);
kunmap_local(virt);

data_length -= cur_len;
/* iscsit_map_iovec has already checked for invalid sg pointers */
sg = sg_next(sg);
}

if (padding) {
struct scatterlist pad_sg;

sg_init_one(&pad_sg, pad_bytes, padding);
ahash_request_set_crypt(hash, &pad_sg, (u8 *)&data_crc,
padding);
crypto_ahash_finup(hash);
} else {
ahash_request_set_crypt(hash, NULL, (u8 *)&data_crc, 0);
crypto_ahash_final(hash);
page_off = 0;
data_length -= cur_len;
}

return data_crc;
if (padding)
crc = crc32c(crc, pad_bytes, padding);

return ~crc;
}

static void iscsit_do_crypto_hash_buf(struct ahash_request *hash,
const void *buf, u32 payload_length, u32 padding,
const void *pad_bytes, void *data_crc)
static u32 iscsit_crc_buf(const void *buf, u32 payload_length,
u32 padding, const void *pad_bytes)
{
struct scatterlist sg[2];
u32 crc = ~0;

sg_init_table(sg, ARRAY_SIZE(sg));
sg_set_buf(sg, buf, payload_length);
if (padding)
sg_set_buf(sg + 1, pad_bytes, padding);
crc = crc32c(crc, buf, payload_length);

ahash_request_set_crypt(hash, sg, data_crc, payload_length + padding);
if (padding)
crc = crc32c(crc, pad_bytes, padding);

crypto_ahash_digest(hash);
return ~crc;
}

int
Expand Down Expand Up @@ -1662,11 +1620,8 @@ iscsit_get_dataout(struct iscsit_conn *conn, struct iscsit_cmd *cmd,
if (conn->conn_ops->DataDigest) {
u32 data_crc;

data_crc = iscsit_do_crypto_hash_sg(conn->conn_rx_hash, cmd,
be32_to_cpu(hdr->offset),
payload_length, padding,
cmd->pad_bytes);

data_crc = iscsit_crc_sglist(cmd, payload_length, padding,
cmd->pad_bytes);
if (checksum != data_crc) {
pr_err("ITT: 0x%08x, Offset: %u, Length: %u,"
" DataSN: 0x%08x, CRC32C DataDigest 0x%08x"
Expand Down Expand Up @@ -1925,10 +1880,8 @@ static int iscsit_handle_nop_out(struct iscsit_conn *conn, struct iscsit_cmd *cm
}

if (conn->conn_ops->DataDigest) {
iscsit_do_crypto_hash_buf(conn->conn_rx_hash, ping_data,
payload_length, padding,
cmd->pad_bytes, &data_crc);

data_crc = iscsit_crc_buf(ping_data, payload_length,
padding, cmd->pad_bytes);
if (checksum != data_crc) {
pr_err("Ping data CRC32C DataDigest"
" 0x%08x does not match computed 0x%08x\n",
Expand Down Expand Up @@ -2328,10 +2281,7 @@ iscsit_handle_text_cmd(struct iscsit_conn *conn, struct iscsit_cmd *cmd,
goto reject;

if (conn->conn_ops->DataDigest) {
iscsit_do_crypto_hash_buf(conn->conn_rx_hash,
text_in, rx_size, 0, NULL,
&data_crc);

data_crc = iscsit_crc_buf(text_in, rx_size, 0, NULL);
if (checksum != data_crc) {
pr_err("Text data CRC32C DataDigest"
" 0x%08x does not match computed"
Expand Down Expand Up @@ -2688,10 +2638,8 @@ static int iscsit_handle_immediate_data(
if (conn->conn_ops->DataDigest) {
u32 data_crc;

data_crc = iscsit_do_crypto_hash_sg(conn->conn_rx_hash, cmd,
cmd->write_data_done, length, padding,
cmd->pad_bytes);

data_crc = iscsit_crc_sglist(cmd, length, padding,
cmd->pad_bytes);
if (checksum != data_crc) {
pr_err("ImmediateData CRC32C DataDigest 0x%08x"
" does not match computed 0x%08x\n", checksum,
Expand Down Expand Up @@ -4116,10 +4064,8 @@ static void iscsit_get_rx_pdu(struct iscsit_conn *conn)
break;
}

iscsit_do_crypto_hash_buf(conn->conn_rx_hash, buffer,
ISCSI_HDR_LEN, 0, NULL,
&checksum);

checksum = iscsit_crc_buf(buffer, ISCSI_HDR_LEN, 0,
NULL);
if (digest != checksum) {
pr_err("HeaderDigest CRC32C failed,"
" received 0x%08x, computed 0x%08x\n",
Expand Down Expand Up @@ -4406,15 +4352,6 @@ int iscsit_close_connection(
*/
iscsit_check_conn_usage_count(conn);

ahash_request_free(conn->conn_tx_hash);
if (conn->conn_rx_hash) {
struct crypto_ahash *tfm;

tfm = crypto_ahash_reqtfm(conn->conn_rx_hash);
ahash_request_free(conn->conn_rx_hash);
crypto_free_ahash(tfm);
}

if (conn->sock)
sock_release(conn->sock);

Expand Down
50 changes: 0 additions & 50 deletions drivers/target/iscsi/iscsi_target_login.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
*
******************************************************************************/

#include <crypto/hash.h>
#include <linux/module.h>
#include <linux/string.h>
#include <linux/kthread.h>
Expand Down Expand Up @@ -71,46 +70,6 @@ static struct iscsi_login *iscsi_login_init_conn(struct iscsit_conn *conn)
return NULL;
}

/*
* Used by iscsi_target_nego.c:iscsi_target_locate_portal() to setup
* per struct iscsit_conn libcrypto contexts for crc32c and crc32-intel
*/
int iscsi_login_setup_crypto(struct iscsit_conn *conn)
{
struct crypto_ahash *tfm;

/*
* Setup slicing by CRC32C algorithm for RX and TX libcrypto contexts
* which will default to crc32c_intel.ko for cpu_has_xmm4_2, or fallback
* to software 1x8 byte slicing from crc32c.ko
*/
tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC);
if (IS_ERR(tfm)) {
pr_err("crypto_alloc_ahash() failed\n");
return -ENOMEM;
}

conn->conn_rx_hash = ahash_request_alloc(tfm, GFP_KERNEL);
if (!conn->conn_rx_hash) {
pr_err("ahash_request_alloc() failed for conn_rx_hash\n");
crypto_free_ahash(tfm);
return -ENOMEM;
}
ahash_request_set_callback(conn->conn_rx_hash, 0, NULL, NULL);

conn->conn_tx_hash = ahash_request_alloc(tfm, GFP_KERNEL);
if (!conn->conn_tx_hash) {
pr_err("ahash_request_alloc() failed for conn_tx_hash\n");
ahash_request_free(conn->conn_rx_hash);
conn->conn_rx_hash = NULL;
crypto_free_ahash(tfm);
return -ENOMEM;
}
ahash_request_set_callback(conn->conn_tx_hash, 0, NULL, NULL);

return 0;
}

static int iscsi_login_check_initiator_version(
struct iscsit_conn *conn,
u8 version_max,
Expand Down Expand Up @@ -1165,15 +1124,6 @@ void iscsi_target_login_sess_out(struct iscsit_conn *conn,
iscsit_dec_session_usage_count(conn->sess);
}

ahash_request_free(conn->conn_tx_hash);
if (conn->conn_rx_hash) {
struct crypto_ahash *tfm;

tfm = crypto_ahash_reqtfm(conn->conn_rx_hash);
ahash_request_free(conn->conn_rx_hash);
crypto_free_ahash(tfm);
}

if (conn->param_list) {
iscsi_release_param_list(conn->param_list);
conn->param_list = NULL;
Expand Down
1 change: 0 additions & 1 deletion drivers/target/iscsi/iscsi_target_login.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ struct iscsi_login;
struct iscsi_np;
struct sockaddr_storage;

extern int iscsi_login_setup_crypto(struct iscsit_conn *);
extern int iscsi_check_for_session_reinstatement(struct iscsit_conn *);
extern int iscsi_login_post_auth_non_zero_tsih(struct iscsit_conn *, u16, u32);
extern int iscsit_setup_np(struct iscsi_np *,
Expand Down
21 changes: 2 additions & 19 deletions drivers/target/iscsi/iscsi_target_nego.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1194,14 +1194,7 @@ int iscsi_target_locate_portal(
goto get_target;

sess->sess_ops->SessionType = 1;
/*
* Setup crc32c modules from libcrypto
*/
if (iscsi_login_setup_crypto(conn) < 0) {
pr_err("iscsi_login_setup_crypto() failed\n");
ret = -1;
goto out;
}

/*
* Serialize access across the discovery struct iscsi_portal_group to
* process login attempt.
Expand Down Expand Up @@ -1258,17 +1251,7 @@ int iscsi_target_locate_portal(
}
conn->tpg_np = tpg_np;
pr_debug("Located Portal Group Object: %hu\n", conn->tpg->tpgt);
/*
* Setup crc32c modules from libcrypto
*/
if (iscsi_login_setup_crypto(conn) < 0) {
pr_err("iscsi_login_setup_crypto() failed\n");
kref_put(&tpg_np->tpg_np_kref, iscsit_login_kref_put);
iscsit_put_tiqn_for_login(tiqn);
conn->tpg = NULL;
ret = -1;
goto out;
}

/*
* Serialize access across the struct iscsi_portal_group to
* process login attempt.
Expand Down
Loading

0 comments on commit 31e4cdd

Please sign in to comment.