Skip to content

Commit

Permalink
crypto: sa2ul - Fix DMA mapping API usage
Browse files Browse the repository at this point in the history 
Make sure that we call the dma_unmap_sg on the correct scatterlist on
completion with the correct sg_nents.

Use sg_table to managed the DMA mapping and at the same time add the needed
dma_sync calls for the sg_table.

Signed-off-by: Peter Ujfalusi <peter.ujfalusi@ti.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
  • Loading branch information
Peter Ujfalusi authored and Herbert Xu committed Oct 2, 2020
1 parent 4fc9832 commit 00c9211
Showing 1 changed file with 117 additions and 98 deletions.
215 changes: 117 additions & 98 deletions drivers/crypto/sa2ul.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,34 +142,39 @@ struct sa_alg_tmpl {
bool registered;
};

/**
* struct sa_mapped_sg: scatterlist information for tx and rx
* @mapped: Set to true if the @sgt is mapped
* @dir: mapping direction used for @sgt
* @split_sg: Set if the sg is split and needs to be freed up
* @static_sg: Static scatterlist entry for overriding data
* @sgt: scatterlist table for DMA API use
*/
struct sa_mapped_sg {
bool mapped;
enum dma_data_direction dir;
struct scatterlist static_sg;
struct scatterlist *split_sg;
struct sg_table sgt;
};
/**
* struct sa_rx_data: RX Packet miscellaneous data place holder
* @req: crypto request data pointer
* @ddev: pointer to the DMA device
* @tx_in: dma_async_tx_descriptor pointer for rx channel
* @split_src_sg: Set if the src sg is split and needs to be freed up
* @split_dst_sg: Set if the dst sg is split and needs to be freed up
* @mapped_sg: Information on tx (0) and rx (1) scatterlist DMA mapping
* @enc: Flag indicating either encryption or decryption
* @enc_iv_size: Initialisation vector size
* @iv_idx: Initialisation vector index
* @rx_sg: Static scatterlist entry for overriding RX data
* @tx_sg: Static scatterlist entry for overriding TX data
* @src: Source data pointer
* @dst: Destination data pointer
*/
struct sa_rx_data {
void *req;
struct device *ddev;
struct dma_async_tx_descriptor *tx_in;
struct scatterlist *split_src_sg;
struct scatterlist *split_dst_sg;
struct sa_mapped_sg mapped_sg[2];
u8 enc;
u8 enc_iv_size;
u8 iv_idx;
struct scatterlist rx_sg;
struct scatterlist tx_sg;
struct scatterlist *src;
struct scatterlist *dst;
};

/**
Expand Down Expand Up @@ -976,23 +981,46 @@ static int sa_3des_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key,
return sa_cipher_setkey(tfm, key, keylen, &ad);
}

static void sa_sync_from_device(struct sa_rx_data *rxd)
{
struct sg_table *sgt;

if (rxd->mapped_sg[0].dir == DMA_BIDIRECTIONAL)
sgt = &rxd->mapped_sg[0].sgt;
else
sgt = &rxd->mapped_sg[1].sgt;

dma_sync_sgtable_for_cpu(rxd->ddev, sgt, DMA_FROM_DEVICE);
}

static void sa_free_sa_rx_data(struct sa_rx_data *rxd)
{
int i;

for (i = 0; i < ARRAY_SIZE(rxd->mapped_sg); i++) {
struct sa_mapped_sg *mapped_sg = &rxd->mapped_sg[i];

if (mapped_sg->mapped) {
dma_unmap_sgtable(rxd->ddev, &mapped_sg->sgt,
mapped_sg->dir, 0);
kfree(mapped_sg->split_sg);
}
}

kfree(rxd);
}

static void sa_aes_dma_in_callback(void *data)
{
struct sa_rx_data *rxd = (struct sa_rx_data *)data;
struct skcipher_request *req;
int sglen;
u32 *result;
__be32 *mdptr;
size_t ml, pl;
int i;
enum dma_data_direction dir_src;
bool diff_dst;

sa_sync_from_device(rxd);
req = container_of(rxd->req, struct skcipher_request, base);
sglen = sg_nents_for_len(req->src, req->cryptlen);

diff_dst = (req->src != req->dst) ? true : false;
dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL;

if (req->iv) {
mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl,
Expand All @@ -1003,18 +1031,7 @@ static void sa_aes_dma_in_callback(void *data)
result[i] = be32_to_cpu(mdptr[i + rxd->iv_idx]);
}

dma_unmap_sg(rxd->ddev, req->src, sglen, dir_src);
kfree(rxd->split_src_sg);

if (diff_dst) {
sglen = sg_nents_for_len(req->dst, req->cryptlen);

dma_unmap_sg(rxd->ddev, req->dst, sglen,
DMA_FROM_DEVICE);
kfree(rxd->split_dst_sg);
}

kfree(rxd);
sa_free_sa_rx_data(rxd);

skcipher_request_complete(req, 0);
}
Expand Down Expand Up @@ -1043,7 +1060,6 @@ static int sa_run(struct sa_req *req)
struct device *ddev;
struct dma_chan *dma_rx;
int sg_nents, src_nents, dst_nents;
int mapped_src_nents, mapped_dst_nents;
struct scatterlist *src, *dst;
size_t pl, ml, split_size;
struct sa_ctx_info *sa_ctx = req->enc ? &req->ctx->enc : &req->ctx->dec;
Expand All @@ -1052,6 +1068,7 @@ static int sa_run(struct sa_req *req)
u32 *mdptr;
bool diff_dst;
enum dma_data_direction dir_src;
struct sa_mapped_sg *mapped_sg;

gfp_flags = req->base->flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
GFP_KERNEL : GFP_ATOMIC;
Expand Down Expand Up @@ -1082,6 +1099,7 @@ static int sa_run(struct sa_req *req)
dma_rx = pdata->dma_rx1;

ddev = dma_rx->device->dev;
rxd->ddev = ddev;

memcpy(cmdl, sa_ctx->cmdl, sa_ctx->cmdl_size);

Expand Down Expand Up @@ -1109,56 +1127,88 @@ static int sa_run(struct sa_req *req)

split_size = req->size;

mapped_sg = &rxd->mapped_sg[0];
if (sg_nents == 1 && split_size <= req->src->length) {
src = &rxd->rx_sg;
src = &mapped_sg->static_sg;
src_nents = 1;
sg_init_table(src, 1);
sg_set_page(src, sg_page(req->src), split_size,
req->src->offset);
src_nents = 1;
dma_map_sg(ddev, src, sg_nents, dir_src);

mapped_sg->sgt.sgl = src;
mapped_sg->sgt.orig_nents = src_nents;
ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0);
if (ret)
return ret;

mapped_sg->dir = dir_src;
mapped_sg->mapped = true;
} else {
mapped_src_nents = dma_map_sg(ddev, req->src, sg_nents,
dir_src);
ret = sg_split(req->src, mapped_src_nents, 0, 1, &split_size,
&src, &src_nents, gfp_flags);
mapped_sg->sgt.sgl = req->src;
mapped_sg->sgt.orig_nents = sg_nents;
ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0);
if (ret)
return ret;

mapped_sg->dir = dir_src;
mapped_sg->mapped = true;

ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents, 0, 1,
&split_size, &src, &src_nents, gfp_flags);
if (ret) {
src_nents = sg_nents;
src = req->src;
src_nents = mapped_sg->sgt.nents;
src = mapped_sg->sgt.sgl;
} else {
rxd->split_src_sg = src;
mapped_sg->split_sg = src;
}
}

dma_sync_sgtable_for_device(ddev, &mapped_sg->sgt, DMA_TO_DEVICE);

if (!diff_dst) {
dst_nents = src_nents;
dst = src;
} else {
dst_nents = sg_nents_for_len(req->dst, req->size);
mapped_sg = &rxd->mapped_sg[1];

if (dst_nents == 1 && split_size <= req->dst->length) {
dst = &rxd->tx_sg;
dst = &mapped_sg->static_sg;
dst_nents = 1;
sg_init_table(dst, 1);
sg_set_page(dst, sg_page(req->dst), split_size,
req->dst->offset);
dst_nents = 1;
dma_map_sg(ddev, dst, dst_nents, DMA_FROM_DEVICE);

mapped_sg->sgt.sgl = dst;
mapped_sg->sgt.orig_nents = dst_nents;
ret = dma_map_sgtable(ddev, &mapped_sg->sgt,
DMA_FROM_DEVICE, 0);
if (ret)
goto err_cleanup;

mapped_sg->dir = DMA_FROM_DEVICE;
mapped_sg->mapped = true;
} else {
mapped_dst_nents = dma_map_sg(ddev, req->dst, dst_nents,
DMA_FROM_DEVICE);
ret = sg_split(req->dst, mapped_dst_nents, 0, 1,
&split_size, &dst, &dst_nents,
gfp_flags);
mapped_sg->sgt.sgl = req->dst;
mapped_sg->sgt.orig_nents = dst_nents;
ret = dma_map_sgtable(ddev, &mapped_sg->sgt,
DMA_FROM_DEVICE, 0);
if (ret)
dst = req->dst;
else
rxd->split_dst_sg = dst;
}
}
goto err_cleanup;

if (unlikely(src_nents != sg_nents)) {
dev_warn_ratelimited(sa_k3_dev, "failed to map tx pkt\n");
ret = -EIO;
goto err_cleanup;
mapped_sg->dir = DMA_FROM_DEVICE;
mapped_sg->mapped = true;

ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents,
0, 1, &split_size, &dst, &dst_nents,
gfp_flags);
if (ret) {
dst_nents = mapped_sg->sgt.nents;
dst = mapped_sg->sgt.sgl;
} else {
mapped_sg->split_sg = dst;
}
}
}

rxd->tx_in = dmaengine_prep_slave_sg(dma_rx, dst, dst_nents,
Expand All @@ -1172,9 +1222,6 @@ static int sa_run(struct sa_req *req)

rxd->req = (void *)req->base;
rxd->enc = req->enc;
rxd->ddev = ddev;
rxd->src = src;
rxd->dst = dst;
rxd->iv_idx = req->ctx->iv_idx;
rxd->enc_iv_size = sa_ctx->cmdl_upd_info.enc_iv.size;
rxd->tx_in->callback = req->callback;
Expand Down Expand Up @@ -1212,16 +1259,7 @@ static int sa_run(struct sa_req *req)
return -EINPROGRESS;

err_cleanup:
dma_unmap_sg(ddev, req->src, sg_nents, DMA_TO_DEVICE);
kfree(rxd->split_src_sg);

if (req->src != req->dst) {
dst_nents = sg_nents_for_len(req->dst, req->size);
dma_unmap_sg(ddev, req->dst, dst_nents, DMA_FROM_DEVICE);
kfree(rxd->split_dst_sg);
}

kfree(rxd);
sa_free_sa_rx_data(rxd);

return ret;
}
Expand Down Expand Up @@ -1291,11 +1329,12 @@ static void sa_sha_dma_in_callback(void *data)
struct ahash_request *req;
struct crypto_ahash *tfm;
unsigned int authsize;
int i, sg_nents;
int i;
size_t ml, pl;
u32 *result;
__be32 *mdptr;

sa_sync_from_device(rxd);
req = container_of(rxd->req, struct ahash_request, base);
tfm = crypto_ahash_reqtfm(req);
authsize = crypto_ahash_digestsize(tfm);
Expand All @@ -1306,12 +1345,7 @@ static void sa_sha_dma_in_callback(void *data)
for (i = 0; i < (authsize / 4); i++)
result[i] = be32_to_cpu(mdptr[i + 4]);

sg_nents = sg_nents_for_len(req->src, req->nbytes);
dma_unmap_sg(rxd->ddev, req->src, sg_nents, DMA_FROM_DEVICE);

kfree(rxd->split_src_sg);

kfree(rxd);
sa_free_sa_rx_data(rxd);

ahash_request_complete(req, 0);
}
Expand Down Expand Up @@ -1635,51 +1669,36 @@ static void sa_aead_dma_in_callback(void *data)
unsigned int authsize;
u8 auth_tag[SA_MAX_AUTH_TAG_SZ];
size_t pl, ml;
int i, sglen;
int i;
int err = 0;
u16 auth_len;
u32 *mdptr;
bool diff_dst;
enum dma_data_direction dir_src;

sa_sync_from_device(rxd);
req = container_of(rxd->req, struct aead_request, base);
tfm = crypto_aead_reqtfm(req);
start = req->assoclen + req->cryptlen;
authsize = crypto_aead_authsize(tfm);

diff_dst = (req->src != req->dst) ? true : false;
dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL;

mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml);
for (i = 0; i < (authsize / 4); i++)
mdptr[i + 4] = swab32(mdptr[i + 4]);

auth_len = req->assoclen + req->cryptlen;
if (!rxd->enc)
auth_len -= authsize;

sglen = sg_nents_for_len(rxd->src, auth_len);
dma_unmap_sg(rxd->ddev, rxd->src, sglen, dir_src);
kfree(rxd->split_src_sg);

if (diff_dst) {
sglen = sg_nents_for_len(rxd->dst, auth_len);
dma_unmap_sg(rxd->ddev, rxd->dst, sglen, DMA_FROM_DEVICE);
kfree(rxd->split_dst_sg);
}

if (rxd->enc) {
scatterwalk_map_and_copy(&mdptr[4], req->dst, start, authsize,
1);
} else {
auth_len -= authsize;
start -= authsize;
scatterwalk_map_and_copy(auth_tag, req->src, start, authsize,
0);

err = memcmp(&mdptr[4], auth_tag, authsize) ? -EBADMSG : 0;
}

kfree(rxd);
sa_free_sa_rx_data(rxd);

aead_request_complete(req, err);
}
Expand Down

0 comments on commit 00c9211

Please sign in to comment.