[][openwrt][mt7988][crypto][Add multiple ring for DDK look-aside]
[Description]
Add multiple ring for DDK look-aside.
[Release-log]
N/A
Change-Id: I4b27ec566ba3704567224b65c650f0bfb9cb89b5
Reviewed-on: https://gerrit.mediatek.inc/c/openwrt/feeds/mtk_openwrt_feeds/+/9463834
diff --git a/feed/kernel/crypto-eip/src/ddk-wrapper.c b/feed/kernel/crypto-eip/src/ddk-wrapper.c
index 107e25d..0c04798 100644
--- a/feed/kernel/crypto-eip/src/ddk-wrapper.c
+++ b/feed/kernel/crypto-eip/src/ddk-wrapper.c
@@ -11,24 +11,25 @@
#include <crypto/hmac.h>
#include <crypto/md5.h>
#include <linux/delay.h>
+#include <crypto/internal/hash.h>
#include <crypto-eip/ddk/slad/api_pcl.h>
#include <crypto-eip/ddk/slad/api_pcl_dtl.h>
#include <crypto-eip/ddk/slad/api_pec.h>
-#include <crypto-eip/ddk/slad/api_driver197_init.h>
#include "crypto-eip/crypto-eip.h"
#include "crypto-eip/ddk-wrapper.h"
#include "crypto-eip/internal.h"
-LIST_HEAD(result_list);
-
-void crypto_free_sa(void *sa_pointer)
+void crypto_free_sa(void *sa_pointer, int ring)
{
DMABuf_Handle_t SAHandle = {0};
+ if (ring < 0)
+ return;
+
SAHandle.p = sa_pointer;
- PEC_SA_UnRegister(PEC_INTERFACE_ID, SAHandle, DMABuf_NULLHandle,
+ PEC_SA_UnRegister(ring, SAHandle, DMABuf_NULLHandle,
DMABuf_NULLHandle);
DMABuf_Release(SAHandle);
}
@@ -98,7 +99,7 @@
unsigned int crypto_pe_busy_get_one(IOToken_Output_Dscr_t *const OutTokenDscr_p,
u32 *OutTokenData_p,
- PEC_ResultDescriptor_t *RD_p)
+ PEC_ResultDescriptor_t *RD_p, int ring)
{
int LoopCounter = MTK_EIP197_INLINE_NOF_TRIES;
int IOToken_Rc;
@@ -114,7 +115,7 @@
/* Try to get the processed packet from the driver */
unsigned int Counter = 0;
- pecres = PEC_Packet_Get(PEC_INTERFACE_ID, RD_p, 1, &Counter);
+ pecres = PEC_Packet_Get(ring, RD_p, 1, &Counter);
if (pecres != PEC_STATUS_OK) {
/* IO error */
CRYPTO_ERR("PEC_Packet_Get error %d\n", pecres);
@@ -151,7 +152,7 @@
unsigned int crypto_pe_get_one(IOToken_Output_Dscr_t *const OutTokenDscr_p,
u32 *OutTokenData_p,
- PEC_ResultDescriptor_t *RD_p)
+ PEC_ResultDescriptor_t *RD_p, int ring)
{
int IOToken_Rc;
unsigned int Counter = 0;
@@ -163,10 +164,10 @@
RD_p->OutputToken_p = OutTokenData_p;
/* Try to get the processed packet from the driver */
- pecres = PEC_Packet_Get(PEC_INTERFACE_ID, RD_p, 1, &Counter);
+ pecres = PEC_Packet_Get(ring, RD_p, 1, &Counter);
if (pecres != PEC_STATUS_OK) {
/* IO error */
- CRYPTO_ERR("PEC_Packet_Get error %d\n", pecres);
+ CRYPTO_ERR("PEC_Packet_Get error %d, ring id %d\n", pecres, ring);
return 0;
}
@@ -255,7 +256,7 @@
}
}
-void mtk_crypto_interrupt_handler(void)
+void mtk_crypto_ring3_handler(void)
{
struct mtk_crypto_result *rd;
struct mtk_crypto_context *ctx;
@@ -265,492 +266,201 @@
int ret = 0;
while (true) {
- spin_lock_bh(&add_lock);
- if (list_empty(&result_list)) {
- spin_unlock_bh(&add_lock);
+ spin_lock_bh(&priv->mtk_eip_ring[3].ring_lock);
+ if (list_empty(&priv->mtk_eip_ring[3].list)) {
+ spin_unlock_bh(&priv->mtk_eip_ring[3].ring_lock);
return;
}
- rd = list_first_entry(&result_list, struct mtk_crypto_result, list);
- spin_unlock_bh(&add_lock);
+ rd = list_first_entry(&priv->mtk_eip_ring[3].list, struct mtk_crypto_result, list);
+ spin_unlock_bh(&priv->mtk_eip_ring[3].ring_lock);
- if (crypto_pe_get_one(&OutTokenDscr, OutputToken, &Res) < 1) {
+ ctx = crypto_tfm_ctx(rd->async->tfm);
+ if (crypto_pe_get_one(&OutTokenDscr, OutputToken, &Res, ctx->ring) < 1) {
PEC_NotifyFunction_t CBFunc;
- CBFunc = mtk_crypto_interrupt_handler;
- if (OutTokenDscr.ErrorCode == 0) {
- PEC_ResultNotify_Request(PEC_INTERFACE_ID, CBFunc, 1);
- return;
- } else if (OutTokenDscr.ErrorCode & BIT(9)) {
+ CBFunc = mtk_crypto_ring3_handler;
+ if (OutTokenDscr.ErrorCode == 0)
+ continue;
+ else if (OutTokenDscr.ErrorCode & BIT(9))
ret = -EBADMSG;
- } else if (OutTokenDscr.ErrorCode == 0x4003) {
+ else if (OutTokenDscr.ErrorCode == 0x4003)
ret = 0;
- } else
+ else
ret = 1;
CRYPTO_ERR("error from crypto_pe_get_one: %d\n", ret);
}
- ctx = crypto_tfm_ctx(rd->async->tfm);
ret = ctx->handle_result(rd, ret);
- spin_lock_bh(&add_lock);
+ spin_lock_bh(&priv->mtk_eip_ring[3].ring_lock);
list_del(&rd->list);
- spin_unlock_bh(&add_lock);
+ spin_unlock_bh(&priv->mtk_eip_ring[3].ring_lock);
kfree(rd);
}
}
-int crypto_aead_cipher(struct crypto_async_request *async, struct mtk_crypto_cipher_req *mtk_req,
- struct scatterlist *src, struct scatterlist *dst, unsigned int cryptlen,
- unsigned int assoclen, unsigned int digestsize, u8 *iv, unsigned int ivsize)
+void mtk_crypto_ring2_handler(void)
{
- struct mtk_crypto_cipher_ctx *ctx = crypto_tfm_ctx(async->tfm);
- struct mtk_crypto_result *result;
- struct scatterlist *sg;
- unsigned int totlen_src;
- unsigned int totlen_dst;
- unsigned int src_pkt = cryptlen + assoclen;
- unsigned int pass_assoc = 0;
- int pass_id;
- int rc;
- int i;
- SABuilder_Params_t params;
- SABuilder_Params_Basic_t ProtocolParams;
- unsigned int SAWords = 0;
-
- DMABuf_Status_t DMAStatus;
- DMABuf_Properties_t DMAProperties = {0, 0, 0, 0};
- DMABuf_HostAddress_t SAHostAddress;
- DMABuf_HostAddress_t TokenHostAddress;
- DMABuf_HostAddress_t PktHostAddress;
-
- DMABuf_Handle_t SAHandle = {0};
- DMABuf_Handle_t TokenHandle = {0};
- DMABuf_Handle_t SrcSGListHandle = {0};
- DMABuf_Handle_t DstSGListHandle = {0};
-
- unsigned int TCRWords = 0;
- void *TCRData = 0;
- unsigned int TokenWords = 0;
- unsigned int TokenHeaderWord;
- unsigned int TokenMaxWords = 0;
-
- TokenBuilder_Params_t TokenParams;
- PEC_CommandDescriptor_t Cmd;
- PEC_NotifyFunction_t CBFunc;
- unsigned int count;
-
- IOToken_Input_Dscr_t InTokenDscr;
+ struct mtk_crypto_result *rd;
+ struct mtk_crypto_context *ctx;
IOToken_Output_Dscr_t OutTokenDscr;
- uint32_t InputToken[IOTOKEN_IN_WORD_COUNT];
- void *InTokenDscrExt_p = NULL;
- uint8_t gcm_iv[16] = {0};
- uint8_t *aad = NULL;
-
-#ifdef CRYPTO_IOTOKEN_EXT
- IOToken_Input_Dscr_Ext_t InTokenDscrExt;
-
- ZEROINIT(InTokenDscrExt);
- InTokenDscrExt_p = &InTokenDscrExt;
-#endif
- ZEROINIT(InTokenDscr);
- ZEROINIT(OutTokenDscr);
-
- /* Init SA */
- if (mtk_req->direction == MTK_CRYPTO_ENCRYPT) {
- totlen_src = cryptlen + assoclen;
- totlen_dst = totlen_src + digestsize;
- rc = SABuilder_Init_Basic(¶ms, &ProtocolParams, SAB_DIRECTION_OUTBOUND);
- } else {
- totlen_src = cryptlen + assoclen;
- totlen_dst = totlen_src - digestsize;
- rc = SABuilder_Init_Basic(¶ms, &ProtocolParams, SAB_DIRECTION_INBOUND);
- }
- if (rc) {
- CRYPTO_ERR("SABuilder_Init_Basic failed: %d\n", rc);
- goto error_exit;
- }
-
- /* Build SA */
- params.CryptoAlgo = lookaside_match_alg_name(ctx->alg);
- params.CryptoMode = lookaside_match_alg_mode(ctx->mode);
- params.KeyByteCount = ctx->key_len;
- params.Key_p = (uint8_t *) ctx->key;
- if (params.CryptoMode == SAB_CRYPTO_MODE_GCM && ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) {
- params.Nonce_p = (uint8_t *) &ctx->nonce;
- params.IVSrc = SAB_IV_SRC_TOKEN;
- params.flags |= SAB_FLAG_COPY_IV;
- memcpy(gcm_iv, &ctx->nonce, 4);
- memcpy(gcm_iv + 4, iv, ivsize);
- gcm_iv[15] = 1;
- } else if (params.CryptoMode == SAB_CRYPTO_MODE_GMAC) {
- params.Nonce_p = (uint8_t *) &ctx->nonce;
- params.IVSrc = SAB_IV_SRC_TOKEN;
- memcpy(gcm_iv, &ctx->nonce, 4);
- memcpy(gcm_iv + 4, iv, ivsize);
- gcm_iv[15] = 1;
- } else if (params.CryptoMode == SAB_CRYPTO_MODE_GCM) {
- params.IVSrc = SAB_IV_SRC_TOKEN;
- memcpy(gcm_iv, iv, ivsize);
- gcm_iv[15] = 1;
- } else if (params.CryptoMode == SAB_CRYPTO_MODE_CCM) {
- params.IVSrc = SAB_IV_SRC_SA;
- params.Nonce_p = (uint8_t *) &ctx->nonce + 1;
- params.IV_p = iv;
- } else {
- params.IVSrc = SAB_IV_SRC_SA;
- params.IV_p = iv;
- }
-
- if (params.CryptoMode == SAB_CRYPTO_MODE_CTR)
- params.Nonce_p = (uint8_t *) &ctx->nonce;
-
- params.AuthAlgo = aead_hash_match(ctx->hash_alg);
- params.AuthKey1_p = (uint8_t *) ctx->ipad;
- params.AuthKey2_p = (uint8_t *) ctx->opad;
-
- ProtocolParams.ICVByteCount = digestsize;
-
- rc = SABuilder_GetSizes(¶ms, &SAWords, NULL, NULL);
- if (rc) {
- CRYPTO_ERR("SA not created because of size errors: %d\n", rc);
- goto error_remove_sg;
- }
-
- DMAProperties.fCached = true;
- DMAProperties.Alignment = MTK_EIP197_INLINE_DMA_ALIGNMENT_BYTE_COUNT;
- DMAProperties.Bank = MTK_EIP197_INLINE_BANK_TRANSFORM;
- DMAProperties.Size = MAX(4*SAWords, 256);
-
- DMAStatus = DMABuf_Alloc(DMAProperties, &SAHostAddress, &SAHandle);
- if (DMAStatus != DMABUF_STATUS_OK) {
- rc = 1;
- CRYPTO_ERR("Allocation of SA failed: %d\n", DMAStatus);
- goto error_remove_sg;
- }
-
- rc = SABuilder_BuildSA(¶ms, (u32 *)SAHostAddress.p, NULL, NULL);
- if (rc) {
- CRYPTO_ERR("SA not created because of errors: %d\n", rc);
- goto error_remove_sg;
- }
-
- /* Check dst buffer has enough size */
- mtk_req->nr_src = sg_nents_for_len(src, totlen_src);
- mtk_req->nr_dst = sg_nents_for_len(dst, totlen_dst);
-
- if (src == dst) {
- mtk_req->nr_src = max(mtk_req->nr_src, mtk_req->nr_dst);
- mtk_req->nr_dst = mtk_req->nr_src;
- if (unlikely((totlen_src || totlen_dst) && (mtk_req->nr_src <= 0))) {
- CRYPTO_ERR("In-place buffer not large enough\n");
- return -EINVAL;
- }
- dma_map_sg(crypto_dev, src, mtk_req->nr_src, DMA_BIDIRECTIONAL);
- } else {
- if (unlikely(totlen_src && (mtk_req->nr_src <= 0))) {
- CRYPTO_ERR("Source buffer not large enough\n");
- return -EINVAL;
- }
- dma_map_sg(crypto_dev, src, mtk_req->nr_src, DMA_TO_DEVICE);
+ PEC_ResultDescriptor_t Res;
+ uint32_t OutputToken[IOTOKEN_OUT_WORD_COUNT];
+ int ret = 0;
- if (unlikely(totlen_dst && (mtk_req->nr_dst <= 0))) {
- CRYPTO_ERR("Dest buffer not large enough\n");
- dma_unmap_sg(crypto_dev, src, mtk_req->nr_src, DMA_TO_DEVICE);
- return -EINVAL;
+ while (true) {
+ spin_lock_bh(&priv->mtk_eip_ring[2].ring_lock);
+ if (list_empty(&priv->mtk_eip_ring[2].list)) {
+ spin_unlock_bh(&priv->mtk_eip_ring[2].ring_lock);
+ return;
}
- dma_map_sg(crypto_dev, dst, mtk_req->nr_dst, DMA_FROM_DEVICE);
- }
-
- if (params.CryptoMode == SAB_CRYPTO_MODE_CCM ||
- (params.CryptoMode == SAB_CRYPTO_MODE_GCM &&
- ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP)) {
+ rd = list_first_entry(&priv->mtk_eip_ring[2].list, struct mtk_crypto_result, list);
+ spin_unlock_bh(&priv->mtk_eip_ring[2].ring_lock);
- aad = kmalloc(assoclen, GFP_KERNEL);
- if (!aad)
- goto error_remove_sg;
- sg_copy_to_buffer(src, mtk_req->nr_src, aad, assoclen);
- src_pkt -= assoclen;
- pass_assoc = assoclen;
- }
-
- /* Assign sg list */
- rc = PEC_SGList_Create(MAX(mtk_req->nr_src, 1), &SrcSGListHandle);
- if (rc != PEC_STATUS_OK) {
- CRYPTO_ERR("PEC_SGList_Create src failed with rc = %d\n", rc);
- goto error_remove_sg;
- }
-
- pass_id = 0;
- DMAProperties.fCached = true;
- DMAProperties.Alignment = MTK_EIP197_INLINE_DMA_ALIGNMENT_BYTE_COUNT;
- DMAProperties.Bank = MTK_EIP197_INLINE_BANK_PACKET;
- for_each_sg(src, sg, mtk_req->nr_src, i) {
- int len = sg_dma_len(sg);
- DMABuf_Handle_t sg_handle;
- DMABuf_HostAddress_t host;
-
- if (totlen_src < len)
- len = totlen_src;
+ ctx = crypto_tfm_ctx(rd->async->tfm);
+ if (crypto_pe_get_one(&OutTokenDscr, OutputToken, &Res, ctx->ring) < 1) {
+ PEC_NotifyFunction_t CBFunc;
- if (pass_assoc) {
- if (pass_assoc >= len) {
- pass_assoc -= len;
- pass_id++;
+ CBFunc = mtk_crypto_ring2_handler;
+ if (OutTokenDscr.ErrorCode == 0)
continue;
- }
- DMAProperties.Size = MAX(len - pass_assoc, 1);
- rc = DMABuf_Particle_Alloc(DMAProperties, sg_dma_address(sg) + pass_assoc,
- &host, &sg_handle);
- if (rc != DMABUF_STATUS_OK) {
- CRYPTO_ERR("DMABuf_Particle_Alloc failed rc = %d\n", rc);
- goto error_remove_sg;
- }
- rc = PEC_SGList_Write(SrcSGListHandle, i - pass_id, sg_handle,
- len - pass_assoc);
- if (rc != PEC_STATUS_OK)
- pr_notice("PEC_SGList_Write failed rc = %d\n", rc);
- pass_assoc = 0;
- } else {
- DMAProperties.Size = MAX(len, 1);
- rc = DMABuf_Particle_Alloc(DMAProperties, sg_dma_address(sg),
- &host, &sg_handle);
- if (rc != DMABUF_STATUS_OK) {
- CRYPTO_ERR("DMABuf_Particle_Alloc failed rc = %d\n", rc);
- goto error_remove_sg;
- }
+ else if (OutTokenDscr.ErrorCode & BIT(9))
+ ret = -EBADMSG;
+ else if (OutTokenDscr.ErrorCode == 0x4003)
+ ret = 0;
+ else
+ ret = 1;
- rc = PEC_SGList_Write(SrcSGListHandle, i - pass_id, sg_handle, len);
- if (rc != PEC_STATUS_OK)
- pr_notice("PEC_SGList_Write failed rc = %d\n", rc);
+ CRYPTO_ERR("error from crypto_pe_get_one: %d\n", ret);
}
- totlen_src -= len;
- if (!totlen_src)
- break;
- }
+ ret = ctx->handle_result(rd, ret);
- /* Alloc sg list for result */
- rc = PEC_SGList_Create(MAX(mtk_req->nr_dst, 1), &DstSGListHandle);
- if (rc != PEC_STATUS_OK) {
- CRYPTO_ERR("PEC_SGList_Create dst failed with rc = %d\n", rc);
- goto error_remove_sg;
+ spin_lock_bh(&priv->mtk_eip_ring[2].ring_lock);
+ list_del(&rd->list);
+ spin_unlock_bh(&priv->mtk_eip_ring[2].ring_lock);
+ kfree(rd);
}
-
- for_each_sg(dst, sg, mtk_req->nr_dst, i) {
- int len = sg_dma_len(sg);
- DMABuf_Handle_t sg_handle;
- DMABuf_HostAddress_t host;
+}
- if (len > totlen_dst)
- len = totlen_dst;
+void mtk_crypto_ring1_handler(void)
+{
+ struct mtk_crypto_result *rd;
+ struct mtk_crypto_context *ctx;
+ IOToken_Output_Dscr_t OutTokenDscr;
+ PEC_ResultDescriptor_t Res;
+ uint32_t OutputToken[IOTOKEN_OUT_WORD_COUNT];
+ int ret = 0;
- DMAProperties.Size = MAX(len, 1);
- rc = DMABuf_Particle_Alloc(DMAProperties, sg_dma_address(sg), &host, &sg_handle);
- if (rc != DMABUF_STATUS_OK) {
- CRYPTO_ERR("DMABuf_Particle_Alloc failed rc = %d\n", rc);
- goto error_remove_sg;
+ while (true) {
+ spin_lock_bh(&priv->mtk_eip_ring[1].ring_lock);
+ if (list_empty(&priv->mtk_eip_ring[1].list)) {
+ spin_unlock_bh(&priv->mtk_eip_ring[1].ring_lock);
+ return;
}
- rc = PEC_SGList_Write(DstSGListHandle, i, sg_handle, len);
- if (rc != PEC_STATUS_OK)
- pr_notice("PEC_SGList_Write failed rc = %d\n", rc);
-
- if (unlikely(!len))
- break;
- totlen_dst -= len;
- }
+ rd = list_first_entry(&priv->mtk_eip_ring[1].list, struct mtk_crypto_result, list);
+ spin_unlock_bh(&priv->mtk_eip_ring[1].ring_lock);
- /* Build Token */
- rc = TokenBuilder_GetContextSize(¶ms, &TCRWords);
- if (rc) {
- CRYPTO_ERR("TokenBuilder_GetContextSize returned errors: %d\n", rc);
- goto error_remove_sg;
- }
-
- TCRData = kmalloc(4 * TCRWords, GFP_KERNEL);
- if (!TCRData) {
- rc = 1;
- CRYPTO_ERR("Allocation of TCR failed\n");
- goto error_remove_sg;
- }
-
- rc = TokenBuilder_BuildContext(¶ms, TCRData);
- if (rc) {
- CRYPTO_ERR("TokenBuilder_BuildContext failed: %d\n", rc);
- goto error_remove_sg;
- }
-
- rc = TokenBuilder_GetSize(TCRData, &TokenMaxWords);
- if (rc) {
- CRYPTO_ERR("TokenBuilder_GetSize failed: %d\n", rc);
- goto error_remove_sg;
- }
-
- DMAProperties.fCached = true;
- DMAProperties.Alignment = MTK_EIP197_INLINE_DMA_ALIGNMENT_BYTE_COUNT;
- DMAProperties.Bank = MTK_EIP197_INLINE_BANK_TOKEN;
- DMAProperties.Size = 4*TokenMaxWords;
-
- DMAStatus = DMABuf_Alloc(DMAProperties, &TokenHostAddress, &TokenHandle);
- if (DMAStatus != DMABUF_STATUS_OK) {
- rc = 1;
- CRYPTO_ERR("Allocation of token builder failed: %d\n", DMAStatus);
- goto error_remove_sg;
- }
-
- rc = PEC_SA_Register(PEC_INTERFACE_ID, SAHandle, DMABuf_NULLHandle,
- DMABuf_NULLHandle);
- if (rc != PEC_STATUS_OK) {
- CRYPTO_ERR("PEC_SA_Register failed: %d\n", rc);
- goto error_remove_sg;
- }
-
- ZEROINIT(TokenParams);
-
- if (params.CryptoMode == SAB_CRYPTO_MODE_GCM || params.CryptoMode == SAB_CRYPTO_MODE_GMAC)
- TokenParams.IV_p = gcm_iv;
-
- if ((params.CryptoMode == SAB_CRYPTO_MODE_GCM && ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) ||
- params.CryptoMode == SAB_CRYPTO_MODE_CCM) {
- TokenParams.AdditionalValue = assoclen - ivsize;
- TokenParams.AAD_p = aad;
- } else if (params.CryptoMode != SAB_CRYPTO_MODE_GMAC)
- TokenParams.AdditionalValue = assoclen;
-
-
- PktHostAddress.p = kmalloc(sizeof(uint8_t), GFP_KERNEL);
- rc = TokenBuilder_BuildToken(TCRData, (uint8_t *)PktHostAddress.p, src_pkt,
- &TokenParams, (uint32_t *)TokenHostAddress.p,
- &TokenWords, &TokenHeaderWord);
- kfree(PktHostAddress.p);
- if (rc != TKB_STATUS_OK) {
- CRYPTO_ERR("Token builder failed: %d\n", rc);
- goto error_exit_unregister;
- }
-
- ZEROINIT(Cmd);
- Cmd.Token_Handle = TokenHandle;
- Cmd.Token_WordCount = TokenWords;
- Cmd.SrcPkt_Handle = SrcSGListHandle;
- Cmd.SrcPkt_ByteCount = src_pkt;
- Cmd.DstPkt_Handle = DstSGListHandle;
- Cmd.SA_Handle1 = SAHandle;
- Cmd.SA_Handle2 = DMABuf_NULLHandle;
-
- #if defined(CRYPTO_IOTOKEN_EXT)
- InTokenDscrExt.HW_Services = IOTOKEN_CMD_PKT_LAC;
-#endif
- InTokenDscr.TknHdrWordInit = TokenHeaderWord;
+ ctx = crypto_tfm_ctx(rd->async->tfm);
+ if (crypto_pe_get_one(&OutTokenDscr, OutputToken, &Res, ctx->ring) < 1) {
+ PEC_NotifyFunction_t CBFunc;
- if (!crypto_iotoken_create(&InTokenDscr,
- InTokenDscrExt_p,
- InputToken,
- &Cmd)) {
- rc = 1;
- goto error_exit_unregister;
- }
+ CBFunc = mtk_crypto_ring1_handler;
+ if (OutTokenDscr.ErrorCode == 0)
+ continue;
+ else if (OutTokenDscr.ErrorCode & BIT(9))
+ ret = -EBADMSG;
+ else if (OutTokenDscr.ErrorCode == 0x4003)
+ ret = 0;
+ else
+ ret = 1;
- rc = PEC_Packet_Put(PEC_INTERFACE_ID, &Cmd, 1, &count);
- if (rc != PEC_STATUS_OK && count != 1)
- goto error_exit_unregister;
+ CRYPTO_ERR("error from crypto_pe_get_one: %d\n", ret);
+ }
- result = kmalloc(sizeof(struct mtk_crypto_result), GFP_KERNEL);
- if (!result) {
- rc = 1;
- CRYPTO_ERR("No memory for result\n");
- goto error_exit_unregister;
- }
- INIT_LIST_HEAD(&result->list);
- result->eip.sa = SAHandle.p;
- result->eip.token = TokenHandle.p;
- result->eip.token_context = TCRData;
- result->eip.pkt_handle = SrcSGListHandle.p;
- result->async = async;
- result->dst = DstSGListHandle.p;
+ ret = ctx->handle_result(rd, ret);
- spin_lock_bh(&add_lock);
- list_add_tail(&result->list, &result_list);
- spin_unlock_bh(&add_lock);
- CBFunc = mtk_crypto_interrupt_handler;
- rc = PEC_ResultNotify_Request(PEC_INTERFACE_ID, CBFunc, 1);
- if (rc != PEC_STATUS_OK) {
- CRYPTO_ERR("PEC_ResultNotify_Request failed with rc = %d\n", rc);
- goto error_exit_unregister;
+ spin_lock_bh(&priv->mtk_eip_ring[1].ring_lock);
+ list_del(&rd->list);
+ spin_unlock_bh(&priv->mtk_eip_ring[1].ring_lock);
+ kfree(rd);
}
+}
- return rc;
+void mtk_crypto_ring0_handler(void)
+{
+ struct mtk_crypto_result *rd;
+ struct mtk_crypto_context *ctx;
+ IOToken_Output_Dscr_t OutTokenDscr;
+ PEC_ResultDescriptor_t Res;
+ uint32_t OutputToken[IOTOKEN_OUT_WORD_COUNT];
+ int ret = 0;
-error_exit_unregister:
- PEC_SA_UnRegister(PEC_INTERFACE_ID, SAHandle, DMABuf_NULLHandle,
- DMABuf_NULLHandle);
-error_remove_sg:
- if (src == dst) {
- dma_unmap_sg(crypto_dev, src, mtk_req->nr_src, DMA_BIDIRECTIONAL);
- } else {
- dma_unmap_sg(crypto_dev, src, mtk_req->nr_src, DMA_TO_DEVICE);
- dma_unmap_sg(crypto_dev, dst, mtk_req->nr_dst, DMA_FROM_DEVICE);
- }
+ while (true) {
+ spin_lock_bh(&priv->mtk_eip_ring[0].ring_lock);
+ if (list_empty(&priv->mtk_eip_ring[0].list)) {
+ spin_unlock_bh(&priv->mtk_eip_ring[0].ring_lock);
+ return;
+ }
+ rd = list_first_entry(&priv->mtk_eip_ring[0].list, struct mtk_crypto_result, list);
+ spin_unlock_bh(&priv->mtk_eip_ring[0].ring_lock);
- if (aad != NULL)
- kfree(aad);
+ ctx = crypto_tfm_ctx(rd->async->tfm);
+ if (crypto_pe_get_one(&OutTokenDscr, OutputToken, &Res, ctx->ring) < 1) {
+ PEC_NotifyFunction_t CBFunc;
- crypto_free_sglist(SrcSGListHandle.p);
- crypto_free_sglist(DstSGListHandle.p);
+ CBFunc = mtk_crypto_ring0_handler;
+ if (OutTokenDscr.ErrorCode == 0)
+ continue;
+ else if (OutTokenDscr.ErrorCode & BIT(9))
+ ret = -EBADMSG;
+ else if (OutTokenDscr.ErrorCode == 0x4003)
+ ret = 0;
+ else
+ ret = 1;
-error_exit:
- DMABuf_Release(SAHandle);
- DMABuf_Release(TokenHandle);
+ CRYPTO_ERR("error from crypto_pe_get_one: %d\n", ret);
+ }
- if (TCRData != NULL)
- kfree(TCRData);
+ ret = ctx->handle_result(rd, ret);
- return rc;
+ spin_lock_bh(&priv->mtk_eip_ring[0].ring_lock);
+ list_del(&rd->list);
+ spin_unlock_bh(&priv->mtk_eip_ring[0].ring_lock);
+ kfree(rd);
+ }
}
+void (*mtk_crypto_interrupt_handler[])(void) = {
+ mtk_crypto_ring0_handler,
+ mtk_crypto_ring1_handler,
+ mtk_crypto_ring2_handler,
+ mtk_crypto_ring3_handler
+};
+
-int crypto_basic_cipher(struct crypto_async_request *async, struct mtk_crypto_cipher_req *mtk_req,
- struct scatterlist *src, struct scatterlist *dst, unsigned int cryptlen,
- unsigned int assoclen, unsigned int digestsize, u8 *iv, unsigned int ivsize)
+int mtk_crypto_ddk_alloc_buff(struct mtk_crypto_cipher_ctx *ctx, int dir, unsigned int digestsize,
+ struct mtk_crypto_engine_data *data)
{
- struct mtk_crypto_cipher_ctx *ctx = crypto_tfm_ctx(async->tfm);
- struct skcipher_request *areq = skcipher_request_cast(async);
- struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(areq);
- struct mtk_crypto_result *result;
- struct scatterlist *sg;
- unsigned int totlen_src = cryptlen + assoclen;
- unsigned int totlen_dst = totlen_src;
- unsigned int blksize = crypto_skcipher_blocksize(skcipher);
- int rc;
- int i;
SABuilder_Params_t params;
SABuilder_Params_Basic_t ProtocolParams;
- unsigned int SAWords = 0;
DMABuf_Status_t DMAStatus;
DMABuf_Properties_t DMAProperties = {0, 0, 0, 0};
DMABuf_HostAddress_t SAHostAddress;
DMABuf_HostAddress_t TokenHostAddress;
- DMABuf_HostAddress_t PktHostAddress;
DMABuf_Handle_t SAHandle = {0};
DMABuf_Handle_t TokenHandle = {0};
- DMABuf_Handle_t SrcSGListHandle = {0};
- DMABuf_Handle_t DstSGListHandle = {0};
+ unsigned int SAWords = 0;
unsigned int TCRWords = 0;
- void *TCRData = 0;
- unsigned int TokenWords = 0;
- unsigned int TokenHeaderWord;
unsigned int TokenMaxWords = 0;
- TokenBuilder_Params_t TokenParams;
- PEC_CommandDescriptor_t Cmd;
- unsigned int count;
-
IOToken_Input_Dscr_t InTokenDscr;
IOToken_Output_Dscr_t OutTokenDscr;
- uint32_t InputToken[IOTOKEN_IN_WORD_COUNT];
void *InTokenDscrExt_p = NULL;
- PEC_NotifyFunction_t CBFunc;
+ int rc;
#ifdef CRYPTO_IOTOKEN_EXT
IOToken_Input_Dscr_Ext_t InTokenDscrExt;
@@ -761,18 +471,20 @@
ZEROINIT(InTokenDscr);
ZEROINIT(OutTokenDscr);
- /* If the data is not aligned with block size, return invalid */
- if (!IS_ALIGNED(cryptlen, blksize))
- return -EINVAL;
-
- /* Init SA */
- if (mtk_req->direction == MTK_CRYPTO_ENCRYPT)
+ if (dir == MTK_CRYPTO_ENCRYPT)
rc = SABuilder_Init_Basic(¶ms, &ProtocolParams, SAB_DIRECTION_OUTBOUND);
else
rc = SABuilder_Init_Basic(¶ms, &ProtocolParams, SAB_DIRECTION_INBOUND);
+
+ if (data->sa_handle) {
+ crypto_free_sa(data->sa_handle, ctx->base.ring);
+ crypto_free_token(data->token_handle);
+ kfree(data->token_context);
+ }
+
if (rc) {
CRYPTO_ERR("SABuilder_Init_Basic failed: %d\n", rc);
- goto error_exit;
+ return rc;
}
/* Build SA */
@@ -780,81 +492,202 @@
params.CryptoMode = lookaside_match_alg_mode(ctx->mode);
params.KeyByteCount = ctx->key_len;
params.Key_p = (uint8_t *) ctx->key;
- params.IVSrc = SAB_IV_SRC_SA;
+ if (params.CryptoMode == SAB_CRYPTO_MODE_GCM && ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) {
+ params.Nonce_p = (uint8_t *) &ctx->nonce;
+ params.IVSrc = SAB_IV_SRC_TOKEN;
+ params.flags |= SAB_FLAG_COPY_IV;
+ } else if (params.CryptoMode == SAB_CRYPTO_MODE_GMAC) {
+ params.Nonce_p = (uint8_t *) &ctx->nonce;
+ params.IVSrc = SAB_IV_SRC_TOKEN;
+ } else if (params.CryptoMode == SAB_CRYPTO_MODE_GCM) {
+ params.IVSrc = SAB_IV_SRC_TOKEN;
+ } else if (params.CryptoMode == SAB_CRYPTO_MODE_CCM) { /* Todo, use token for ccm */
+ params.IVSrc = SAB_IV_SRC_TOKEN;
+ params.Nonce_p = (uint8_t *) &ctx->nonce + 1;
+ } else {
+ params.IVSrc = SAB_IV_SRC_TOKEN;
+ }
+
if (params.CryptoMode == SAB_CRYPTO_MODE_CTR)
params.Nonce_p = (uint8_t *) &ctx->nonce;
- params.IV_p = iv;
+
+ params.AuthAlgo = aead_hash_match(ctx->hash_alg);
+ params.AuthKey1_p = (uint8_t *) ctx->ipad;
+ params.AuthKey2_p = (uint8_t *) ctx->opad;
+
+ ProtocolParams.ICVByteCount = digestsize;
rc = SABuilder_GetSizes(¶ms, &SAWords, NULL, NULL);
if (rc) {
CRYPTO_ERR("SA not created because of size errors: %d\n", rc);
- goto error_exit;
+ return rc;
}
+ data->sa_size = SAWords;
+
DMAProperties.fCached = true;
DMAProperties.Alignment = MTK_EIP197_INLINE_DMA_ALIGNMENT_BYTE_COUNT;
DMAProperties.Bank = MTK_EIP197_INLINE_BANK_TRANSFORM;
- DMAProperties.Size = MAX(4*SAWords, 256);
+ DMAProperties.Size = MAX(4 * SAWords, 256);
DMAStatus = DMABuf_Alloc(DMAProperties, &SAHostAddress, &SAHandle);
if (DMAStatus != DMABUF_STATUS_OK) {
rc = 1;
CRYPTO_ERR("Allocation of SA failed: %d\n", DMAStatus);
- goto error_exit;
+ return rc;
}
- rc = SABuilder_BuildSA(¶ms, (u32 *)SAHostAddress.p, NULL, NULL);
+ rc = SABuilder_BuildSA(¶ms, (u32 *) SAHostAddress.p, NULL, NULL);
if (rc) {
CRYPTO_ERR("SA not created because of errors: %d\n", rc);
- goto error_exit;
+ goto release_sa;
+ }
+
+ rc = PEC_SA_Register(ctx->base.ring, SAHandle, DMABuf_NULLHandle,
+ DMABuf_NULLHandle);
+ if (rc != PEC_STATUS_OK) {
+ CRYPTO_ERR("PEC_SA_Register failed: %d\n", rc);
+ goto release_sa;
}
+ data->sa_addr = SAHostAddress.p;
+ data->sa_handle = SAHandle.p;
+
/* Build Token */
rc = TokenBuilder_GetContextSize(¶ms, &TCRWords);
if (rc) {
- CRYPTO_ERR("TokenBuilder_GetContextSize returned errors: %d\n", rc);
- goto error_exit;
+ CRYPTO_ERR("TokenBuilder_GetContextSize returned errors: f%d\n", rc);
+ goto release_sa;
}
- TCRData = kmalloc(4 * TCRWords, GFP_KERNEL);
- if (!TCRData) {
+ data->token_context = kmalloc(4 * TCRWords, GFP_KERNEL);
+ if (!data->token_context) {
rc = 1;
CRYPTO_ERR("Allocation of TCR failed\n");
- goto error_exit;
+ goto release_sa;
}
- rc = TokenBuilder_BuildContext(¶ms, TCRData);
+ rc = TokenBuilder_BuildContext(¶ms, data->token_context);
if (rc) {
CRYPTO_ERR("TokenBuilder_BuildContext failed: %d\n", rc);
- goto error_exit;
+ goto release_context;
}
- rc = TokenBuilder_GetSize(TCRData, &TokenMaxWords);
+ rc = TokenBuilder_GetSize(data->token_context, &TokenMaxWords);
if (rc) {
CRYPTO_ERR("TokenBuilder_GetSize failed: %d\n", rc);
- goto error_exit;
+ goto release_context;
}
+ data->token_size = TokenMaxWords;
DMAProperties.fCached = true;
DMAProperties.Alignment = MTK_EIP197_INLINE_DMA_ALIGNMENT_BYTE_COUNT;
DMAProperties.Bank = MTK_EIP197_INLINE_BANK_TOKEN;
- DMAProperties.Size = 4*TokenMaxWords;
+ DMAProperties.Size = 4 * TokenMaxWords;
DMAStatus = DMABuf_Alloc(DMAProperties, &TokenHostAddress, &TokenHandle);
if (DMAStatus != DMABUF_STATUS_OK) {
rc = 1;
CRYPTO_ERR("Allocation of token builder failed: %d\n", DMAStatus);
- goto error_exit;
+ goto release_context;
}
+ data->token_addr = TokenHostAddress.p;
+ data->token_handle = TokenHandle.p;
+ data->valid = 1;
- rc = PEC_SA_Register(PEC_INTERFACE_ID, SAHandle, DMABuf_NULLHandle,
- DMABuf_NULLHandle);
- if (rc != PEC_STATUS_OK) {
- CRYPTO_ERR("PEC_SA_Register failed: %d\n", rc);
- goto error_exit;
+ return rc;
+
+release_context:
+ kfree(data->token_context);
+release_sa:
+ DMABuf_Release(SAHandle);
+
+ return rc;
+}
+
+int mtk_crypto_basic_cipher(struct crypto_async_request *async,
+ struct mtk_crypto_cipher_req *mtk_req, struct scatterlist *src,
+ struct scatterlist *dst, unsigned int cryptlen,
+ unsigned int assoclen, unsigned int digestsize, u8 *iv,
+ unsigned int ivsize)
+{
+ struct mtk_crypto_cipher_ctx *ctx = crypto_tfm_ctx(async->tfm);
+ struct mtk_crypto_engine_data *data;
+ struct mtk_crypto_result *result;
+ struct scatterlist *sg;
+ unsigned int totlen_src;
+ unsigned int totlen_dst;
+ unsigned int src_pkt = cryptlen + assoclen;
+ unsigned int pass_assoc = 0;
+ int pass_id;
+ int rc;
+ int i;
+ int ring = ctx->base.ring;
+
+ DMABuf_Properties_t DMAProperties = {0, 0, 0, 0};
+
+ DMABuf_Handle_t SAHandle = {0};
+ DMABuf_Handle_t TokenHandle = {0};
+ DMABuf_Handle_t SrcSGListHandle = {0};
+ DMABuf_Handle_t DstSGListHandle = {0};
+
+ unsigned int TokenWords = 0;
+ unsigned int TokenHeaderWord;
+
+ TokenBuilder_Params_t TokenParams;
+ PEC_CommandDescriptor_t Cmd;
+ PEC_NotifyFunction_t CBFunc;
+ unsigned int count;
+
+ IOToken_Input_Dscr_t InTokenDscr;
+ IOToken_Output_Dscr_t OutTokenDscr;
+ uint32_t InputToken[IOTOKEN_IN_WORD_COUNT];
+ void *InTokenDscrExt_p = NULL;
+ uint8_t token_iv[16] = {0};
+ uint8_t *aad = NULL;
+
+#ifdef CRYPTO_IOTOKEN_EXT
+ IOToken_Input_Dscr_Ext_t InTokenDscrExt;
+
+ ZEROINIT(InTokenDscrExt);
+ InTokenDscrExt_p = &InTokenDscrExt;
+#endif
+ ZEROINIT(InTokenDscr);
+ ZEROINIT(OutTokenDscr);
+
+ /* Init SA */
+ if (mtk_req->direction == MTK_CRYPTO_ENCRYPT) {
+ totlen_src = cryptlen + assoclen;
+ totlen_dst = totlen_src + digestsize;
+ data = &ctx->enc;
+ } else {
+ totlen_src = cryptlen + assoclen;
+ totlen_dst = totlen_src - digestsize;
+ data = &ctx->dec;
+ }
+
+ SAHandle.p = data->sa_handle;
+ TokenHandle.p = data->token_handle;
+
+ if ((ctx->mode == MTK_CRYPTO_MODE_GCM && ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP)
+ || ctx->mode == MTK_CRYPTO_MODE_GMAC) {
+ memcpy(token_iv, &ctx->nonce, 4);
+ memcpy(token_iv + 4, iv, ivsize);
+ token_iv[15] = 1;
+ } else if (ctx->mode == MTK_CRYPTO_MODE_GCM) {
+ memcpy(token_iv, iv, ivsize);
+ token_iv[15] = 1;
+ } else if (ctx->mode == MTK_CRYPTO_MODE_CCM) {
+ memcpy(token_iv, (uint8_t *) &ctx->nonce, 4);
+ memcpy(token_iv + 4, iv, ivsize);
+ token_iv[15] = 0;
+ } else if (ctx->mode == MTK_CRYPTO_MODE_CTR) {
+ memcpy(token_iv, &ctx->nonce, 4);
+ memcpy(token_iv + 4, iv, ivsize);
+ token_iv[15] = 1;
}
- /* Check buffer has enough size for output */
+ /* Check dst buffer has enough size */
mtk_req->nr_src = sg_nents_for_len(src, totlen_src);
mtk_req->nr_dst = sg_nents_for_len(dst, totlen_dst);
@@ -863,33 +696,43 @@
mtk_req->nr_dst = mtk_req->nr_src;
if (unlikely((totlen_src || totlen_dst) && (mtk_req->nr_src <= 0))) {
CRYPTO_ERR("In-place buffer not large enough\n");
- kfree(TCRData);
- return -EINVAL;
+ goto error_remove_sg;
}
dma_map_sg(crypto_dev, src, mtk_req->nr_src, DMA_BIDIRECTIONAL);
} else {
if (unlikely(totlen_src && (mtk_req->nr_src <= 0))) {
CRYPTO_ERR("Source buffer not large enough\n");
- kfree(TCRData);
- return -EINVAL;
+ goto error_remove_sg;
}
dma_map_sg(crypto_dev, src, mtk_req->nr_src, DMA_TO_DEVICE);
if (unlikely(totlen_dst && (mtk_req->nr_dst <= 0))) {
CRYPTO_ERR("Dest buffer not large enough\n");
dma_unmap_sg(crypto_dev, src, mtk_req->nr_src, DMA_TO_DEVICE);
- kfree(TCRData);
- return -EINVAL;
+ goto error_remove_sg;
}
dma_map_sg(crypto_dev, dst, mtk_req->nr_dst, DMA_FROM_DEVICE);
}
+ if (ctx->mode == MTK_CRYPTO_MODE_CCM ||
+ (ctx->mode == MTK_CRYPTO_MODE_GCM && ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP)) {
+
+ aad = kmalloc(assoclen, GFP_KERNEL);
+ if (!aad)
+ goto error_remove_sg;
+ sg_copy_to_buffer(src, mtk_req->nr_src, aad, assoclen);
+ src_pkt -= assoclen;
+ pass_assoc = assoclen;
+ }
+
+ /* Assign sg list */
rc = PEC_SGList_Create(MAX(mtk_req->nr_src, 1), &SrcSGListHandle);
if (rc != PEC_STATUS_OK) {
CRYPTO_ERR("PEC_SGList_Create src failed with rc = %d\n", rc);
goto error_remove_sg;
}
+ pass_id = 0;
DMAProperties.fCached = true;
DMAProperties.Alignment = MTK_EIP197_INLINE_DMA_ALIGNMENT_BYTE_COUNT;
DMAProperties.Bank = MTK_EIP197_INLINE_BANK_PACKET;
@@ -901,21 +744,44 @@
if (totlen_src < len)
len = totlen_src;
- DMAProperties.Size = MAX(len, 1);
- rc = DMABuf_Particle_Alloc(DMAProperties, sg_dma_address(sg), &host, &sg_handle);
- if (rc != DMABUF_STATUS_OK) {
- CRYPTO_ERR("DMABuf_Particle_Alloc failed rc = %d\n", rc);
- goto error_remove_sg;
+ if (pass_assoc) {
+ if (pass_assoc >= len) {
+ pass_assoc -= len;
+ pass_id++;
+ continue;
+ }
+ DMAProperties.Size = MAX(len - pass_assoc, 1);
+ rc = DMABuf_Particle_Alloc(DMAProperties, sg_dma_address(sg) + pass_assoc,
+ &host, &sg_handle);
+ if (rc != DMABUF_STATUS_OK) {
+ CRYPTO_ERR("DMABuf_Particle_Alloc failed rc = %d\n", rc);
+ goto error_remove_sg;
+ }
+ rc = PEC_SGList_Write(SrcSGListHandle, i - pass_id, sg_handle,
+ len - pass_assoc);
+ if (rc != PEC_STATUS_OK)
+ pr_notice("PEC_SGList_Write failed rc = %d\n", rc);
+ pass_assoc = 0;
+ } else {
+ DMAProperties.Size = MAX(len, 1);
+ rc = DMABuf_Particle_Alloc(DMAProperties, sg_dma_address(sg),
+ &host, &sg_handle);
+ if (rc != DMABUF_STATUS_OK) {
+ CRYPTO_ERR("DMABuf_Particle_Alloc failed rc = %d\n", rc);
+ goto error_remove_sg;
+ }
+
+ rc = PEC_SGList_Write(SrcSGListHandle, i - pass_id, sg_handle, len);
+ if (rc != PEC_STATUS_OK)
+ pr_notice("PEC_SGList_Write failed rc = %d\n", rc);
}
- rc = PEC_SGList_Write(SrcSGListHandle, i, sg_handle, len);
- if (rc != PEC_STATUS_OK)
- pr_notice("PEC_SGList_Write failed rc = %d\n", rc);
totlen_src -= len;
if (!totlen_src)
break;
}
+ /* Alloc sg list for result */
rc = PEC_SGList_Create(MAX(mtk_req->nr_dst, 1), &DstSGListHandle);
if (rc != PEC_STATUS_OK) {
CRYPTO_ERR("PEC_SGList_Create dst failed with rc = %d\n", rc);
@@ -937,32 +803,47 @@
goto error_remove_sg;
}
rc = PEC_SGList_Write(DstSGListHandle, i, sg_handle, len);
+ if (rc != PEC_STATUS_OK)
+ pr_notice("PEC_SGList_Write failed rc = %d\n", rc);
if (unlikely(!len))
break;
totlen_dst -= len;
}
- if (params.CryptoMode == SAB_CRYPTO_MODE_CBC &&
- mtk_req->direction == MTK_CRYPTO_DECRYPT)
- sg_pcopy_to_buffer(src, mtk_req->nr_src, iv, ivsize, cryptlen - ivsize);
-
- PktHostAddress.p = kmalloc(sizeof(uint8_t), GFP_KERNEL);
+ /* Build Token */
ZEROINIT(TokenParams);
- rc = TokenBuilder_BuildToken(TCRData, (uint8_t *)PktHostAddress.p, cryptlen,
- &TokenParams, (uint32_t *)TokenHostAddress.p,
+
+ if (ctx->mode == MTK_CRYPTO_MODE_GCM || ctx->mode == MTK_CRYPTO_MODE_GMAC ||
+ ctx->mode == MTK_CRYPTO_MODE_CCM || ctx->mode == MTK_CRYPTO_MODE_CTR)
+ TokenParams.IV_p = token_iv;
+ else
+ TokenParams.IV_p = iv;
+
+ if ((ctx->mode == MTK_CRYPTO_MODE_GCM && ctx->aead == EIP197_AEAD_TYPE_IPSEC_ESP) ||
+ ctx->mode == MTK_CRYPTO_MODE_CCM) {
+ TokenParams.AdditionalValue = assoclen - ivsize;
+ TokenParams.AAD_p = aad;
+ } else if (ctx->mode != MTK_CRYPTO_MODE_GMAC)
+ TokenParams.AdditionalValue = assoclen;
+
+ rc = TokenBuilder_BuildToken(data->token_context, aad, src_pkt,
+ &TokenParams, (uint32_t *) data->token_addr,
&TokenWords, &TokenHeaderWord);
- kfree(PktHostAddress.p);
if (rc != TKB_STATUS_OK) {
CRYPTO_ERR("Token builder failed: %d\n", rc);
goto error_remove_sg;
}
+ if (ctx->mode == MTK_CRYPTO_MODE_CBC &&
+ mtk_req->direction == MTK_CRYPTO_DECRYPT)
+ sg_pcopy_to_buffer(src, mtk_req->nr_src, iv, ivsize, cryptlen - ivsize);
+
ZEROINIT(Cmd);
Cmd.Token_Handle = TokenHandle;
Cmd.Token_WordCount = TokenWords;
Cmd.SrcPkt_Handle = SrcSGListHandle;
- Cmd.SrcPkt_ByteCount = cryptlen;
+ Cmd.SrcPkt_ByteCount = src_pkt;
Cmd.DstPkt_Handle = DstSGListHandle;
Cmd.SA_Handle1 = SAHandle;
Cmd.SA_Handle2 = DMABuf_NULLHandle;
@@ -980,10 +861,8 @@
goto error_remove_sg;
}
- rc = PEC_Packet_Put(PEC_INTERFACE_ID, &Cmd, 1, &count);
+ rc = PEC_Packet_Put(ring, &Cmd, 1, &count);
if (rc != PEC_STATUS_OK && count != 1) {
- rc = 1;
- CRYPTO_ERR("PEC_Packet_Put error: %d\n", rc);
goto error_remove_sg;
}
@@ -994,23 +873,21 @@
goto error_remove_sg;
}
INIT_LIST_HEAD(&result->list);
- result->eip.sa = SAHandle.p;
- result->eip.token = TokenHandle.p;
- result->eip.token_context = TCRData;
result->eip.pkt_handle = SrcSGListHandle.p;
result->async = async;
result->dst = DstSGListHandle.p;
- spin_lock_bh(&add_lock);
- list_add_tail(&result->list, &result_list);
- spin_unlock_bh(&add_lock);
- CBFunc = mtk_crypto_interrupt_handler;
- rc = PEC_ResultNotify_Request(PEC_INTERFACE_ID, CBFunc, 1);
+ spin_lock_bh(&priv->mtk_eip_ring[ring].ring_lock);
+ list_add_tail(&result->list, &priv->mtk_eip_ring[ring].list);
+ spin_unlock_bh(&priv->mtk_eip_ring[ring].ring_lock);
+ CBFunc = mtk_crypto_interrupt_handler[ring];
+ rc = PEC_ResultNotify_Request(ring, CBFunc, 1);
if (rc != PEC_STATUS_OK) {
CRYPTO_ERR("PEC_ResultNotify_Request failed with rc = %d\n", rc);
goto error_remove_sg;
}
- return 0;
+
+ return rc;
error_remove_sg:
if (src == dst) {
@@ -1020,19 +897,12 @@
dma_unmap_sg(crypto_dev, dst, mtk_req->nr_dst, DMA_FROM_DEVICE);
}
+ if (aad != NULL)
+ kfree(aad);
+
crypto_free_sglist(SrcSGListHandle.p);
crypto_free_sglist(DstSGListHandle.p);
- PEC_SA_UnRegister(PEC_INTERFACE_ID, SAHandle, DMABuf_NULLHandle,
- DMABuf_NULLHandle);
-
-error_exit:
- DMABuf_Release(SAHandle);
- DMABuf_Release(TokenHandle);
-
- if (TCRData != NULL)
- kfree(TCRData);
-
return rc;
}
@@ -1068,6 +938,8 @@
uint8_t *Input_p, unsigned int InputByteCount, bool finish)
{
struct mtk_crypto_result *result;
+ struct ahash_request *areq = ahash_request_cast(async);
+ struct mtk_crypto_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
DMABuf_Properties_t DMAProperties = {0, 0, 0, 0};
DMABuf_HostAddress_t TokenHostAddress;
@@ -1082,6 +954,7 @@
unsigned int TokenHeaderWord;
unsigned int TokenWords = 0;
void *TCRData = 0;
+ int ring = ctx->base.ring;
TokenBuilder_Params_t TokenParams;
PEC_CommandDescriptor_t Cmd;
@@ -1174,7 +1047,7 @@
goto error_exit_unregister;
}
- rc = PEC_Packet_Put(PEC_INTERFACE_ID, &Cmd, 1, &count);
+ rc = PEC_Packet_Put(ring, &Cmd, 1, &count);
if (rc != PEC_STATUS_OK && count != 1) {
rc = 1;
CRYPTO_ERR("PEC_Packet_Put error: %d\n", rc);
@@ -1188,21 +1061,21 @@
goto error_exit_unregister;
}
INIT_LIST_HEAD(&result->list);
- result->eip.token = TokenHandle.p;
+ result->eip.token_handle = TokenHandle.p;
result->eip.pkt_handle = PktHandle.p;
result->async = async;
result->dst = PktHostAddress.p;
- spin_lock_bh(&add_lock);
- list_add_tail(&result->list, &result_list);
- spin_unlock_bh(&add_lock);
- CBFunc = mtk_crypto_interrupt_handler;
- rc = PEC_ResultNotify_Request(PEC_INTERFACE_ID, CBFunc, 1);
+ spin_lock_bh(&priv->mtk_eip_ring[ring].ring_lock);
+ list_add_tail(&result->list, &priv->mtk_eip_ring[ring].list);
+ spin_unlock_bh(&priv->mtk_eip_ring[ring].ring_lock);
+ CBFunc = mtk_crypto_interrupt_handler[ring];
+ rc = PEC_ResultNotify_Request(ring, CBFunc, 1);
return rc;
error_exit_unregister:
- PEC_SA_UnRegister(PEC_INTERFACE_ID, SAHandle, DMABuf_NULLHandle,
+ PEC_SA_UnRegister(ring, SAHandle, DMABuf_NULLHandle,
DMABuf_NULLHandle);
error_exit:
@@ -1225,6 +1098,7 @@
SABuilder_Params_t params;
unsigned int SAWords = 0;
int rc;
+ int ring = ctx->base.ring;
DMABuf_Properties_t DMAProperties = {0, 0, 0, 0};
DMABuf_HostAddress_t TokenHostAddress;
@@ -1360,7 +1234,7 @@
goto error_exit;
}
- rc = PEC_SA_Register(PEC_INTERFACE_ID, SAHandle, DMABuf_NULLHandle,
+ rc = PEC_SA_Register(ring, SAHandle, DMABuf_NULLHandle,
DMABuf_NULLHandle);
if (rc != PEC_STATUS_OK) {
CRYPTO_ERR("PEC_SA_Register failed: %d\n", rc);
@@ -1400,7 +1274,7 @@
goto error_exit_unregister;
}
- rc = PEC_Packet_Put(PEC_INTERFACE_ID, &Cmd, 1, &count);
+ rc = PEC_Packet_Put(ring, &Cmd, 1, &count);
if (rc != PEC_STATUS_OK && count != 1) {
rc = 1;
CRYPTO_ERR("PEC_Packet_Put error: %d\n", rc);
@@ -1414,20 +1288,20 @@
goto error_exit_unregister;
}
INIT_LIST_HEAD(&result->list);
- result->eip.sa = SAHandle.p;
- result->eip.token = TokenHandle.p;
+ result->eip.sa_handle = SAHandle.p;
+ result->eip.token_handle = TokenHandle.p;
result->eip.token_context = TCRData;
result->eip.pkt_handle = PktHandle.p;
result->async = async;
result->dst = PktHostAddress.p;
result->size = InputByteCount;
- spin_lock_bh(&add_lock);
- list_add_tail(&result->list, &result_list);
- spin_unlock_bh(&add_lock);
+ spin_lock_bh(&priv->mtk_eip_ring[ring].ring_lock);
+ list_add_tail(&result->list, &priv->mtk_eip_ring[ring].list);
+ spin_unlock_bh(&priv->mtk_eip_ring[ring].ring_lock);
- CBFunc = mtk_crypto_interrupt_handler;
- rc = PEC_ResultNotify_Request(PEC_INTERFACE_ID, CBFunc, 1);
+ CBFunc = mtk_crypto_interrupt_handler[ring];
+ rc = PEC_ResultNotify_Request(ring, CBFunc, 1);
if (rc != PEC_STATUS_OK) {
CRYPTO_ERR("PEC_ResultNotify_Request failed with rc = %d\n", rc);
goto error_exit_unregister;
@@ -1435,7 +1309,7 @@
return 0;
error_exit_unregister:
- PEC_SA_UnRegister(PEC_INTERFACE_ID, SAHandle, DMABuf_NULLHandle,
+ PEC_SA_UnRegister(ring, SAHandle, DMABuf_NULLHandle,
DMABuf_NULLHandle);
error_exit:
@@ -1460,6 +1334,7 @@
unsigned int SAWords = 0;
static uint8_t DummyAuthKey[64];
int rc;
+ int ring = ctx->base.ring;
DMABuf_Properties_t DMAProperties = {0, 0, 0, 0};
DMABuf_HostAddress_t TokenHostAddress;
@@ -1605,7 +1480,7 @@
goto error_exit;
}
- rc = PEC_SA_Register(PEC_INTERFACE_ID, SAHandle, DMABuf_NULLHandle,
+ rc = PEC_SA_Register(ring, SAHandle, DMABuf_NULLHandle,
DMABuf_NULLHandle);
if (rc != PEC_STATUS_OK) {
CRYPTO_ERR("PEC_SA_Register failed: %d\n", rc);
@@ -1652,7 +1527,7 @@
goto error_exit_unregister;
}
- rc = PEC_Packet_Put(PEC_INTERFACE_ID, &Cmd, 1, &count);
+ rc = PEC_Packet_Put(ring, &Cmd, 1, &count);
if (rc != PEC_STATUS_OK && count != 1) {
rc = 1;
CRYPTO_ERR("PEC_Packet_Put error: %d\n", rc);
@@ -1666,21 +1541,21 @@
goto error_exit_unregister;
}
INIT_LIST_HEAD(&result->list);
- result->eip.token = TokenHandle.p;
+ result->eip.token_handle = TokenHandle.p;
result->eip.pkt_handle = PktHandle.p;
result->async = async;
result->dst = PktHostAddress.p;
- spin_lock_bh(&add_lock);
- list_add_tail(&result->list, &result_list);
- spin_unlock_bh(&add_lock);
- CBFunc = mtk_crypto_interrupt_handler;
- rc = PEC_ResultNotify_Request(PEC_INTERFACE_ID, CBFunc, 1);
+ spin_lock_bh(&priv->mtk_eip_ring[ring].ring_lock);
+ list_add_tail(&result->list, &priv->mtk_eip_ring[ring].list);
+ spin_unlock_bh(&priv->mtk_eip_ring[ring].ring_lock);
+ CBFunc = mtk_crypto_interrupt_handler[ring];
+ rc = PEC_ResultNotify_Request(ring, CBFunc, 1);
return rc;
error_exit_unregister:
- PEC_SA_UnRegister(PEC_INTERFACE_ID, SAHandle, DMABuf_NULLHandle,
+ PEC_SA_UnRegister(ring, SAHandle, DMABuf_NULLHandle,
DMABuf_NULLHandle);
error_exit:
@@ -1881,7 +1756,7 @@
goto error_exit_unregister;
}
- if (crypto_pe_busy_get_one(&OutTokenDscr, OutputToken, &Res) < 1) {
+ if (crypto_pe_busy_get_one(&OutTokenDscr, OutputToken, &Res, PEC_INTERFACE_ID) < 1) {
rc = 1;
CRYPTO_ERR("error from crypto_pe_busy_get_one\n");
goto error_exit_unregister;
@@ -2147,6 +2022,7 @@
PEC_Capabilities_t pec_cap;
PEC_Status_t pec_sta;
u32 i = MTK_EIP197_INLINE_NOF_TRIES;
+ u32 j;
#ifdef PEC_PCL_EIP197
PCL_Status_t pcl_sta;
#endif
@@ -2164,17 +2040,19 @@
return -1;
}
#endif
- while (i) {
- pec_sta = PEC_Init(PEC_INTERFACE_ID, &pec_init_blk);
- if (pec_sta == PEC_STATUS_OK) {
- CRYPTO_INFO("PEC_INIT ok!\n");
- break;
- } else if (pec_sta != PEC_STATUS_OK && pec_sta != PEC_STATUS_BUSY) {
- return pec_sta;
- }
+ for (j = 0; j < PEC_MAX_INTERFACE_NUM; j++) {
+ while (i) {
+ pec_sta = PEC_Init(j, &pec_init_blk);
+ if (pec_sta == PEC_STATUS_OK) {
+ CRYPTO_INFO("PEC_INIT interface %d ok!\n", j);
+ break;
+ } else if (pec_sta != PEC_STATUS_OK && pec_sta != PEC_STATUS_BUSY) {
+ return pec_sta;
+ }
- mdelay(MTK_EIP197_INLINE_RETRY_DELAY_MS);
- i--;
+ mdelay(MTK_EIP197_INLINE_RETRY_DELAY_MS);
+ i--;
+ }
}
if (!i) {
@@ -2200,23 +2078,26 @@
{
unsigned int LoopCounter = MTK_EIP197_INLINE_NOF_TRIES;
PEC_Status_t PEC_Status;
+ int j;
- while (LoopCounter > 0) {
- PEC_Status = PEC_UnInit(PEC_INTERFACE_ID);
- if (PEC_Status == PEC_STATUS_OK)
- break;
- else if (PEC_Status != PEC_STATUS_OK && PEC_Status != PEC_STATUS_BUSY) {
- CRYPTO_ERR("PEC could not be un-initialized, error=%d\n", PEC_Status);
+ for (j = 0; j < PEC_MAX_INTERFACE_NUM; j++) {
+ while (LoopCounter > 0) {
+ PEC_Status = PEC_UnInit(j);
+ if (PEC_Status == PEC_STATUS_OK)
+ break;
+ else if (PEC_Status != PEC_STATUS_OK && PEC_Status != PEC_STATUS_BUSY) {
+ CRYPTO_ERR("PEC could not deinit, error=%d\n", PEC_Status);
+ return;
+ }
+ // Wait for MTK_EIP197_INLINE_RETRY_DELAY_MS milliseconds
+ udelay(MTK_EIP197_INLINE_RETRY_DELAY_MS * 1000);
+ LoopCounter--;
+ }
+ // Check for timeout
+ if (LoopCounter == 0) {
+ CRYPTO_ERR("PEC could not be un-initialized, timeout\n");
return;
}
- // Wait for MTK_EIP197_INLINE_RETRY_DELAY_MS milliseconds
- udelay(MTK_EIP197_INLINE_RETRY_DELAY_MS * 1000);
- LoopCounter--;
- }
- // Check for timeout
- if (LoopCounter == 0) {
- CRYPTO_ERR("PEC could not be un-initialized, timeout\n");
- return;
}
#ifdef PEC_PCL_EIP197
@@ -2424,7 +2305,7 @@
goto error_exit_unregister;
}
- if (crypto_pe_busy_get_one(&OutTokenDscr, OutputToken, &Res) < 1) {
+ if (crypto_pe_busy_get_one(&OutTokenDscr, OutputToken, &Res, PEC_INTERFACE_ID) < 1) {
rc = 1;
CRYPTO_ERR("error from crypto_pe_busy_get_one\n");
goto error_exit_unregister;
@@ -2506,7 +2387,7 @@
}
// Receive the result packet ... do we care about contents ?
- if (crypto_pe_busy_get_one(&OutTokenDscr, OutputToken, &Res) < 1) {
+ if (crypto_pe_busy_get_one(&OutTokenDscr, OutputToken, &Res, PEC_INTERFACE_ID) < 1) {
CRYPTO_ERR("%s: crypto_pe_busy_get_one() failed\n", __func__);
return false;
}
diff --git a/feed/kernel/crypto-eip/src/inc/crypto-eip/crypto-eip.h b/feed/kernel/crypto-eip/src/inc/crypto-eip/crypto-eip.h
index 5a413b6..7d5d21a 100644
--- a/feed/kernel/crypto-eip/src/inc/crypto-eip/crypto-eip.h
+++ b/feed/kernel/crypto-eip/src/inc/crypto-eip/crypto-eip.h
@@ -19,7 +19,6 @@
struct mtk_crypto;
extern struct mtk_crypto mcrypto;
-extern spinlock_t add_lock;
#define TRANSFORM_RECORD_LEN 64
diff --git a/feed/kernel/crypto-eip/src/inc/crypto-eip/crypto-eip197-inline-ddk.h b/feed/kernel/crypto-eip/src/inc/crypto-eip/crypto-eip197-inline-ddk.h
index 25cd724..42975c2 100644
--- a/feed/kernel/crypto-eip/src/inc/crypto-eip/crypto-eip197-inline-ddk.h
+++ b/feed/kernel/crypto-eip/src/inc/crypto-eip/crypto-eip197-inline-ddk.h
@@ -50,6 +50,7 @@
#define MTK_EIP197_INLINE_DMA_ALIGNMENT_BYTE_COUNT 16
/* PEC Configuration */
+#define PEC_MAX_INTERFACE_NUM 4
#ifdef DDK_PEC_IF_ID
#define PEC_INTERFACE_ID DDK_PEC_IF_ID
#else
diff --git a/feed/kernel/crypto-eip/src/inc/crypto-eip/ddk-wrapper.h b/feed/kernel/crypto-eip/src/inc/crypto-eip/ddk-wrapper.h
index e301b53..99f33ba 100644
--- a/feed/kernel/crypto-eip/src/inc/crypto-eip/ddk-wrapper.h
+++ b/feed/kernel/crypto-eip/src/inc/crypto-eip/ddk-wrapper.h
@@ -13,14 +13,13 @@
#include "lookaside.h"
#include "crypto-eip197-inline-ddk.h"
-void mtk_crypto_interrupt_handler(void);
u32 *mtk_ddk_tr_ipsec_build(struct mtk_xfrm_params *xfrm_params, u32 ipsec_mod);
-int crypto_basic_cipher(struct crypto_async_request *async, struct mtk_crypto_cipher_req *mtk_req,
- struct scatterlist *src, struct scatterlist *dst, unsigned int cryptlen,
- unsigned int assoclen, unsigned int digestsize, u8 *iv, unsigned int ivsize);
-int crypto_aead_cipher(struct crypto_async_request *async, struct mtk_crypto_cipher_req *mtk_req,
- struct scatterlist *src, struct scatterlist *dst, unsigned int cryptlen,
- unsigned int assoclen, unsigned int digestsize, u8 *iv, unsigned int ivsize);
+int mtk_crypto_ddk_alloc_buff(struct mtk_crypto_cipher_ctx *ctx, int dir, unsigned int digestsize,
+ struct mtk_crypto_engine_data *data);
+int mtk_crypto_basic_cipher(struct crypto_async_request *async,
+ struct mtk_crypto_cipher_req *mtk_req, struct scatterlist *src,
+ struct scatterlist *dst, unsigned int cryptlen, unsigned int assoclen,
+ unsigned int digestsize, u8 *iv, unsigned int ivsize);
int crypto_ahash_token_req(struct crypto_async_request *async,
struct mtk_crypto_ahash_req *mtk_req, uint8_t *Input_p,
unsigned int InputByteCount, /*uint8_t *Output_p,*/
@@ -37,7 +36,7 @@
unsigned int AuthKeyByteCount,
uint8_t *Inner_p,
uint8_t *Outer_p);
-void crypto_free_sa(void *sa_pointer);
+void crypto_free_sa(void *sa_pointer, int ring);
void crypto_free_token(void *token);
void crypto_free_pkt(void *pkt);
void crypto_free_sglist(void *sglist);
diff --git a/feed/kernel/crypto-eip/src/inc/crypto-eip/lookaside.h b/feed/kernel/crypto-eip/src/inc/crypto-eip/lookaside.h
index cc9bc29..11ac275 100644
--- a/feed/kernel/crypto-eip/src/inc/crypto-eip/lookaside.h
+++ b/feed/kernel/crypto-eip/src/inc/crypto-eip/lookaside.h
@@ -28,6 +28,8 @@
#define EIP197_AEAD_IPSEC_IV_SIZE 8
#define EIP197_AEAD_IPSEC_CCM_NONCE_SIZE 3
+extern struct mtk_crypto_priv *priv;
+
extern struct mtk_crypto_alg_template mtk_crypto_cbc_aes;
extern struct mtk_crypto_alg_template mtk_crypto_ofb_aes;
extern struct mtk_crypto_alg_template mtk_crypto_ecb_aes;
@@ -80,10 +82,12 @@
struct mtk_crypto_work_data {
struct work_struct work;
struct mtk_crypto_priv *priv;
+ int ring;
};
-struct mtk_crypto_queue {
- spinlock_t lock;
+struct mtk_crypto_ring {
+ struct list_head list;
+ spinlock_t ring_lock;
struct workqueue_struct *workqueue;
struct mtk_crypto_work_data work_data;
@@ -93,13 +97,19 @@
};
struct mtk_crypto_priv {
- struct mtk_crypto_queue mtk_eip_queue;
+ struct mtk_crypto_ring *mtk_eip_ring;
+ atomic_t ring_used;
};
struct mtk_crypto_engine_data {
- void *sa;
- void *token;
+ int valid;
+ void *sa_handle;
+ void *sa_addr;
+ uint32_t sa_size;
+ void *token_handle;
+ void *token_addr;
void *token_context;
+ uint32_t token_size;
void *pkt_handle;
};
@@ -131,6 +141,7 @@
struct mtk_crypto_context {
int (*send)(struct crypto_async_request *req);
int (*handle_result)(struct mtk_crypto_result *req, int err);
+ int ring;
};
enum mtk_crypto_cipher_direction {
@@ -194,6 +205,9 @@
struct crypto_cipher *hkaes;
struct crypto_aead *fback;
+
+ struct mtk_crypto_engine_data enc;
+ struct mtk_crypto_engine_data dec;
};
enum mtk_crypto_ahash_digest {
@@ -264,12 +278,14 @@
u32 state[SHA512_DIGEST_SIZE / sizeof(u32)];
u8 cache[HASH_CACHE_SIZE];
+ int ring;
void *sa_pointer;
void *token_context;
};
void mtk_crypto_dequeue_work(struct work_struct *work);
-void mtk_crypto_dequeue(struct mtk_crypto_priv *priv);
+void mtk_crypto_dequeue(struct mtk_crypto_priv *priv, int ring);
+int mtk_crypto_select_ring(struct mtk_crypto_priv *priv);
int mtk_crypto_hmac_setkey(const char *alg, const u8 *key, unsigned int keylen,
void *istate, void *ostate);
diff --git a/feed/kernel/crypto-eip/src/init.c b/feed/kernel/crypto-eip/src/init.c
index 757f2ec..fc7b868 100644
--- a/feed/kernel/crypto-eip/src/init.c
+++ b/feed/kernel/crypto-eip/src/init.c
@@ -34,7 +34,6 @@
struct mtk_crypto mcrypto;
struct device *crypto_dev;
struct mtk_crypto_priv *priv;
-spinlock_t add_lock;
static struct mtk_crypto_alg_template *mtk_crypto_algs[] = {
&mtk_crypto_cbc_aes,
@@ -294,6 +293,7 @@
static int __init mtk_crypto_lookaside_data_init(struct platform_device *pdev)
{
struct device *dev = &pdev->dev;
+ int i;
priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
if (!priv)
@@ -301,18 +301,41 @@
platform_set_drvdata(pdev, priv);
- priv->mtk_eip_queue.work_data.priv = priv;
- INIT_WORK(&priv->mtk_eip_queue.work_data.work, mtk_crypto_dequeue_work);
-
- priv->mtk_eip_queue.workqueue = create_singlethread_workqueue("mtk_crypto_work");
- if (!priv->mtk_eip_queue.workqueue)
+ priv->mtk_eip_ring = devm_kcalloc(dev, PEC_MAX_INTERFACE_NUM,
+ sizeof(*priv->mtk_eip_ring), GFP_KERNEL);
+ if (!priv->mtk_eip_ring)
return -ENOMEM;
+ for (i = 0; i < PEC_MAX_INTERFACE_NUM; i++) {
+ char wq_name[17] = {0};
+ char irq_name[6] = {0};
+ int irq, cpu;
+
+ // init workqueue for all rings
+ priv->mtk_eip_ring[i].work_data.priv = priv;
+ priv->mtk_eip_ring[i].work_data.ring = i;
+ INIT_WORK(&priv->mtk_eip_ring[i].work_data.work, mtk_crypto_dequeue_work);
+
- crypto_init_queue(&priv->mtk_eip_queue.queue, EIP197_DEFAULT_RING_SIZE);
+ snprintf(wq_name, 17, "mtk_crypto_work%d", i);
+ priv->mtk_eip_ring[i].workqueue = create_singlethread_workqueue(wq_name);
+ if (!priv->mtk_eip_ring[i].workqueue)
+ return -ENOMEM;
- spin_lock_init(&priv->mtk_eip_queue.lock);
- spin_lock_init(&priv->mtk_eip_queue.queue_lock);
- spin_lock_init(&add_lock);
+ crypto_init_queue(&priv->mtk_eip_ring[i].queue, EIP197_DEFAULT_RING_SIZE);
+
+ spin_lock_init(&priv->mtk_eip_ring[i].ring_lock);
+ spin_lock_init(&priv->mtk_eip_ring[i].queue_lock);
+ INIT_LIST_HEAD(&priv->mtk_eip_ring[i].list);
+
+ // setup irq affinity
+ snprintf(irq_name, 6, "ring%d", i);
+ irq = platform_get_irq_byname(pdev, irq_name);
+ if (irq < 0)
+ return irq;
+
+ cpu = cpumask_local_spread(i, NUMA_NO_NODE);
+ irq_set_affinity_hint(irq, get_cpu_mask(cpu));
+ }
return 0;
};
diff --git a/feed/kernel/crypto-eip/src/lookaside-cipher.c b/feed/kernel/crypto-eip/src/lookaside-cipher.c
index df46d47..7527d44 100644
--- a/feed/kernel/crypto-eip/src/lookaside-cipher.c
+++ b/feed/kernel/crypto-eip/src/lookaside-cipher.c
@@ -29,13 +29,23 @@
struct skcipher_request *req = skcipher_request_cast(async);
struct mtk_crypto_cipher_req *mtk_req = skcipher_request_ctx(req);
struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
+ unsigned int blksize = crypto_skcipher_blocksize(skcipher);
int ret = 0;
- ret = crypto_basic_cipher(async, mtk_req, req->src, req->dst, req->cryptlen,
+ if (!IS_ALIGNED(req->cryptlen, blksize)) {
+ ret = -EINVAL;
+ goto end;
+ }
+
+ ret = mtk_crypto_basic_cipher(async, mtk_req, req->src, req->dst, req->cryptlen,
0, 0, req->iv, skcipher->ivsize);
- if (ret != 0)
+end:
+ if (ret != 0) {
+ local_bh_disable();
async->complete(async, ret);
+ local_bh_enable();
+ }
return ret;
}
@@ -59,13 +69,12 @@
dma_unmap_sg(crypto_dev, req->dst, mtk_req->nr_dst, DMA_FROM_DEVICE);
}
+ local_bh_disable();
async->complete(async, err);
+ local_bh_enable();
crypto_free_sglist(res->eip.pkt_handle);
crypto_free_sglist(res->dst);
- crypto_free_sa(res->eip.sa);
- crypto_free_token(res->eip.token);
- kfree(res->eip.token_context);
return 0;
}
@@ -77,12 +86,15 @@
struct mtk_crypto_cipher_req *mtk_req = aead_request_ctx(req);
int ret;
- ret = crypto_aead_cipher(async, mtk_req, req->src, req->dst, req->cryptlen,
+ ret = mtk_crypto_basic_cipher(async, mtk_req, req->src, req->dst, req->cryptlen,
req->assoclen, crypto_aead_authsize(tfm), req->iv,
crypto_aead_ivsize(tfm));
- if (ret != 0)
+ if (ret != 0) {
+ local_bh_disable();
async->complete(async, ret);
+ local_bh_enable();
+ }
return ret;
}
@@ -121,11 +133,10 @@
crypto_free_sglist(res->eip.pkt_handle);
crypto_free_sglist(res->dst);
- crypto_free_sa(res->eip.sa);
- crypto_free_token(res->eip.token);
- kfree(res->eip.token_context);
+ local_bh_disable();
async->complete(async, err);
+ local_bh_enable();
return 0;
}
@@ -142,6 +153,9 @@
ctx->base.send = mtk_crypto_skcipher_send;
ctx->base.handle_result = mtk_crypto_skcipher_handle_result;
+ ctx->base.ring = -1;
+ ctx->enc.valid = 0;
+ ctx->dec.valid = 0;
return 0;
}
@@ -172,17 +186,35 @@
enum mtk_crypto_cipher_direction dir)
{
struct mtk_crypto_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
+ struct crypto_aead *tfm = crypto_aead_reqtfm(aead_request_cast(base));
struct mtk_crypto_priv *priv = ctx->priv;
+ struct mtk_crypto_engine_data *data;
+ int ring;
int ret;
mtk_req->direction = dir;
- spin_lock_bh(&priv->mtk_eip_queue.queue_lock);
- ret = crypto_enqueue_request(&priv->mtk_eip_queue.queue, base);
- spin_unlock_bh(&priv->mtk_eip_queue.queue_lock);
+ if (ctx->base.ring < 0) {
+ ring = mtk_crypto_select_ring(priv);
+ ctx->base.ring = ring;
+ } else
+ ring = ctx->base.ring;
- queue_work(priv->mtk_eip_queue.workqueue,
- &priv->mtk_eip_queue.work_data.work);
+
+ if (dir == MTK_CRYPTO_ENCRYPT)
+ data = &ctx->enc;
+ else
+ data = &ctx->dec;
+
+ if (!data->valid)
+ mtk_crypto_ddk_alloc_buff(ctx, dir, crypto_aead_authsize(tfm), data);
+
+ spin_lock_bh(&priv->mtk_eip_ring[ring].queue_lock);
+ ret = crypto_enqueue_request(&priv->mtk_eip_ring[ring].queue, base);
+ spin_unlock_bh(&priv->mtk_eip_ring[ring].queue_lock);
+
+ queue_work(priv->mtk_eip_ring[ring].workqueue,
+ &priv->mtk_eip_ring[ring].work_data.work);
return ret;
}
@@ -216,6 +248,8 @@
ctx->key_len = len;
memzero_explicit(&aes, sizeof(aes));
+ ctx->enc.valid = 0;
+ ctx->dec.valid = 0;
return 0;
}
@@ -223,6 +257,18 @@
{
struct mtk_crypto_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
+ if (ctx->enc.sa_handle) {
+ crypto_free_sa(ctx->enc.sa_handle, ctx->base.ring);
+ crypto_free_token(ctx->enc.token_handle);
+ kfree(ctx->enc.token_context);
+ }
+
+ if (ctx->dec.sa_handle) {
+ crypto_free_sa(ctx->dec.sa_handle, ctx->base.ring);
+ crypto_free_token(ctx->dec.token_handle);
+ kfree(ctx->dec.token_context);
+ }
+
memzero_explicit(ctx->key, sizeof(ctx->key));
}
@@ -368,6 +414,8 @@
ctx->key_len = keylen;
memzero_explicit(&aes, sizeof(aes));
+ ctx->enc.valid = 0;
+ ctx->dec.valid = 0;
return 0;
}
@@ -418,6 +466,8 @@
return ret;
memcpy(ctx->key, key, len);
ctx->key_len = len;
+ ctx->enc.valid = 0;
+ ctx->dec.valid = 0;
return 0;
}
@@ -505,6 +555,8 @@
memcpy(ctx->key, key, len);
ctx->key_len = len;
+ ctx->enc.valid = 0;
+ ctx->dec.valid = 0;
return 0;
}
@@ -614,6 +666,9 @@
ctx->aead = true;
ctx->base.send = mtk_crypto_aead_send;
ctx->base.handle_result = mtk_crypto_aead_handle_result;
+ ctx->base.ring = -1;
+ ctx->enc.valid = 0;
+ ctx->dec.valid = 0;
return 0;
}
@@ -716,13 +771,17 @@
memcpy(ctx->opad, &ostate.state, ctx->state_sz);
if (istate.sa_pointer)
- crypto_free_sa(istate.sa_pointer);
+ crypto_free_sa(istate.sa_pointer, istate.ring);
kfree(istate.token_context);
if (ostate.sa_pointer)
- crypto_free_sa(ostate.sa_pointer);
+ crypto_free_sa(ostate.sa_pointer, ostate.ring);
kfree(ostate.token_context);
memzero_explicit(&keys, sizeof(keys));
+
+ ctx->enc.valid = 0;
+ ctx->dec.valid = 0;
+
return 0;
badkey:
@@ -1466,6 +1525,10 @@
memzero_explicit(hashkey, AES_BLOCK_SIZE);
memzero_explicit(&aes, sizeof(aes));
+
+ ctx->enc.valid = 0;
+ ctx->dec.valid = 0;
+
return 0;
}
@@ -1656,6 +1719,10 @@
ctx->hash_alg = MTK_CRYPTO_ALG_CCM;
memzero_explicit(&aes, sizeof(aes));
+
+ ctx->enc.valid = 0;
+ ctx->dec.valid = 0;
+
return 0;
}
diff --git a/feed/kernel/crypto-eip/src/lookaside-hash.c b/feed/kernel/crypto-eip/src/lookaside-hash.c
index 8e0ed27..947ec8a 100644
--- a/feed/kernel/crypto-eip/src/lookaside-hash.c
+++ b/feed/kernel/crypto-eip/src/lookaside-hash.c
@@ -29,12 +29,15 @@
struct mtk_crypto_priv *priv = ctx->priv;
int ret;
- spin_lock_bh(&priv->mtk_eip_queue.queue_lock);
- ret = crypto_enqueue_request(&priv->mtk_eip_queue.queue, &areq->base);
- spin_unlock_bh(&priv->mtk_eip_queue.queue_lock);
+ if (ctx->base.ring < 0)
+ ctx->base.ring = mtk_crypto_select_ring(priv);
- queue_work(priv->mtk_eip_queue.workqueue,
- &priv->mtk_eip_queue.work_data.work);
+ spin_lock_bh(&priv->mtk_eip_ring[ctx->base.ring].queue_lock);
+ ret = crypto_enqueue_request(&priv->mtk_eip_ring[ctx->base.ring].queue, &areq->base);
+ spin_unlock_bh(&priv->mtk_eip_ring[ctx->base.ring].queue_lock);
+
+ queue_work(priv->mtk_eip_ring[ctx->base.ring].workqueue,
+ &priv->mtk_eip_ring[ctx->base.ring].work_data.work);
return ret;
}
@@ -127,7 +130,7 @@
kfree(cur_req);
if (ret) {
if (req->sa_pointer)
- crypto_free_sa(req->sa_pointer);
+ crypto_free_sa(req->sa_pointer, ctx->base.ring);
kfree(req->token_context);
CRYPTO_ERR("Fail on ahash_aes_cbc process\n");
goto exit;
@@ -147,7 +150,7 @@
if (ret) {
if (req->sa_pointer)
- crypto_free_sa(req->sa_pointer);
+ crypto_free_sa(req->sa_pointer, ctx->base.ring);
CRYPTO_ERR("Fail on ahash_req process\n");
goto exit;
}
@@ -158,7 +161,7 @@
zero_length_hmac:
if (req->sa_pointer)
- crypto_free_sa(req->sa_pointer);
+ crypto_free_sa(req->sa_pointer, ctx->base.ring);
kfree(req->token_context);
/* complete the final hash with opad for hmac*/
@@ -171,7 +174,9 @@
return 0;
exit:
+ local_bh_disable();
async->complete(async, ret);
+ local_bh_enable();
return 0;
}
@@ -180,18 +185,19 @@
{
struct crypto_async_request *async = res->async;
struct ahash_request *areq = ahash_request_cast(async);
+ struct mtk_crypto_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
struct mtk_crypto_ahash_req *req = ahash_request_ctx(areq);
struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
int cache_len;
if (req->xcbcmac) {
memcpy(req->state, res->dst + res->size - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
- crypto_free_sa(res->eip.sa);
+ crypto_free_sa(res->eip.sa_handle, ctx->base.ring);
kfree(res->eip.token_context);
} else
memcpy(req->state, res->dst, req->digest_sz);
- crypto_free_token(res->eip.token);
+ crypto_free_token(res->eip.token_handle);
crypto_free_pkt(res->eip.pkt_handle);
if (req->finish) {
@@ -201,7 +207,7 @@
return 0;
}
if (req->sa_pointer)
- crypto_free_sa(req->sa_pointer);
+ crypto_free_sa(req->sa_pointer, ctx->base.ring);
kfree(req->token_context);
@@ -211,7 +217,10 @@
cache_len = mtk_crypto_queued_len(req);
if (cache_len)
memcpy(req->cache, req->cache_next, cache_len);
+
+ local_bh_disable();
async->complete(async, 0);
+ local_bh_enable();
return 0;
}
@@ -322,6 +331,7 @@
static int mtk_crypto_ahash_export(struct ahash_request *areq, void *out)
{
struct mtk_crypto_ahash_req *req = ahash_request_ctx(areq);
+ struct mtk_crypto_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
struct mtk_crypto_ahash_export_state *export = out;
export->len = req->len;
@@ -330,6 +340,7 @@
export->digest = req->digest;
export->sa_pointer = req->sa_pointer;
export->token_context = req->token_context;
+ export->ring = ctx->base.ring;
memcpy(export->state, req->state, req->state_sz);
memcpy(export->cache, req->cache, HASH_CACHE_SIZE);
@@ -372,6 +383,7 @@
ctx->priv = tmpl->priv;
ctx->base.send = mtk_crypto_ahash_send;
ctx->base.handle_result = mtk_crypto_ahash_handle_result;
+ ctx->base.ring = -1;
crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
sizeof(struct mtk_crypto_ahash_req));
diff --git a/feed/kernel/crypto-eip/src/lookaside.c b/feed/kernel/crypto-eip/src/lookaside.c
index 7eae63d..fe2cb15 100644
--- a/feed/kernel/crypto-eip/src/lookaside.c
+++ b/feed/kernel/crypto-eip/src/lookaside.c
@@ -15,7 +15,12 @@
#include "crypto-eip/lookaside.h"
#include "crypto-eip/internal.h"
-void mtk_crypto_dequeue(struct mtk_crypto_priv *priv)
+inline int mtk_crypto_select_ring(struct mtk_crypto_priv *priv)
+{
+ return (atomic_inc_return(&priv->ring_used) % PEC_MAX_INTERFACE_NUM);
+}
+
+void mtk_crypto_dequeue(struct mtk_crypto_priv *priv, int ring)
{
struct crypto_async_request *req;
struct crypto_async_request *backlog;
@@ -23,10 +28,10 @@
int ret;
while (true) {
- spin_lock_bh(&priv->mtk_eip_queue.queue_lock);
- backlog = crypto_get_backlog(&priv->mtk_eip_queue.queue);
- req = crypto_dequeue_request(&priv->mtk_eip_queue.queue);
- spin_unlock_bh(&priv->mtk_eip_queue.queue_lock);
+ spin_lock_bh(&priv->mtk_eip_ring[ring].queue_lock);
+ backlog = crypto_get_backlog(&priv->mtk_eip_ring[ring].queue);
+ req = crypto_dequeue_request(&priv->mtk_eip_ring[ring].queue);
+ spin_unlock_bh(&priv->mtk_eip_ring[ring].queue_lock);
if (!req)
goto finalize;
@@ -48,5 +53,5 @@
{
struct mtk_crypto_work_data *data =
container_of(work, struct mtk_crypto_work_data, work);
- mtk_crypto_dequeue(data->priv);
+ mtk_crypto_dequeue(data->priv, data->ring);
}