/* * This file is provided under a dual BSD/GPLv2 license. When using or * redistributing this file, you may do so under either license. * * GPL LICENSE SUMMARY * Copyright(c) 2015-2016 Intel Corporation. * This program is free software; you can redistribute it and/or modify * it under the terms of version 2 of the GNU General Public License as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * Contact Information: * qat-linux@intel.com * * BSD LICENSE * Copyright(c) 2015-2016 Intel Corporation. * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * Neither the name of Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include "../qat_logs.h" #include /* Needed to calculate pre-compute values */ #include /* Needed to calculate pre-compute values */ #include /* Needed to calculate pre-compute values */ #include "qat_algs.h" /* returns block size in bytes per cipher algo */ int qat_cipher_get_block_size(enum icp_qat_hw_cipher_algo qat_cipher_alg) { switch (qat_cipher_alg) { case ICP_QAT_HW_CIPHER_ALGO_DES: return ICP_QAT_HW_DES_BLK_SZ; case ICP_QAT_HW_CIPHER_ALGO_3DES: return ICP_QAT_HW_3DES_BLK_SZ; case ICP_QAT_HW_CIPHER_ALGO_AES128: case ICP_QAT_HW_CIPHER_ALGO_AES192: case ICP_QAT_HW_CIPHER_ALGO_AES256: return ICP_QAT_HW_AES_BLK_SZ; default: PMD_DRV_LOG(ERR, "invalid block cipher alg %u", qat_cipher_alg); return -EFAULT; }; return -EFAULT; } /* * Returns size in bytes per hash algo for state1 size field in cd_ctrl * This is digest size rounded up to nearest quadword */ static int qat_hash_get_state1_size(enum icp_qat_hw_auth_algo qat_hash_alg) { switch (qat_hash_alg) { case ICP_QAT_HW_AUTH_ALGO_SHA1: return QAT_HW_ROUND_UP(ICP_QAT_HW_SHA1_STATE1_SZ, QAT_HW_DEFAULT_ALIGNMENT); case ICP_QAT_HW_AUTH_ALGO_SHA224: return QAT_HW_ROUND_UP(ICP_QAT_HW_SHA224_STATE1_SZ, QAT_HW_DEFAULT_ALIGNMENT); case ICP_QAT_HW_AUTH_ALGO_SHA256: return QAT_HW_ROUND_UP(ICP_QAT_HW_SHA256_STATE1_SZ, QAT_HW_DEFAULT_ALIGNMENT); case ICP_QAT_HW_AUTH_ALGO_SHA384: return QAT_HW_ROUND_UP(ICP_QAT_HW_SHA384_STATE1_SZ, QAT_HW_DEFAULT_ALIGNMENT); case ICP_QAT_HW_AUTH_ALGO_SHA512: return QAT_HW_ROUND_UP(ICP_QAT_HW_SHA512_STATE1_SZ, QAT_HW_DEFAULT_ALIGNMENT); case ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC: return QAT_HW_ROUND_UP(ICP_QAT_HW_AES_XCBC_MAC_STATE1_SZ, QAT_HW_DEFAULT_ALIGNMENT); case ICP_QAT_HW_AUTH_ALGO_GALOIS_128: case ICP_QAT_HW_AUTH_ALGO_GALOIS_64: return QAT_HW_ROUND_UP(ICP_QAT_HW_GALOIS_128_STATE1_SZ, QAT_HW_DEFAULT_ALIGNMENT); case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3: return QAT_HW_ROUND_UP(ICP_QAT_HW_ZUC_3G_EIA3_STATE1_SZ, QAT_HW_DEFAULT_ALIGNMENT); case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2: return QAT_HW_ROUND_UP(ICP_QAT_HW_SNOW_3G_UIA2_STATE1_SZ, QAT_HW_DEFAULT_ALIGNMENT); case ICP_QAT_HW_AUTH_ALGO_MD5: return QAT_HW_ROUND_UP(ICP_QAT_HW_MD5_STATE1_SZ, QAT_HW_DEFAULT_ALIGNMENT); case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9: return QAT_HW_ROUND_UP(ICP_QAT_HW_KASUMI_F9_STATE1_SZ, QAT_HW_DEFAULT_ALIGNMENT); case ICP_QAT_HW_AUTH_ALGO_NULL: return QAT_HW_ROUND_UP(ICP_QAT_HW_NULL_STATE1_SZ, QAT_HW_DEFAULT_ALIGNMENT); case ICP_QAT_HW_AUTH_ALGO_DELIMITER: /* return maximum state1 size in this case */ return QAT_HW_ROUND_UP(ICP_QAT_HW_SHA512_STATE1_SZ, QAT_HW_DEFAULT_ALIGNMENT); default: PMD_DRV_LOG(ERR, "invalid hash alg %u", qat_hash_alg); return -EFAULT; }; return -EFAULT; } /* returns digest size in bytes per hash algo */ static int qat_hash_get_digest_size(enum icp_qat_hw_auth_algo qat_hash_alg) { switch (qat_hash_alg) { case ICP_QAT_HW_AUTH_ALGO_SHA1: return ICP_QAT_HW_SHA1_STATE1_SZ; case ICP_QAT_HW_AUTH_ALGO_SHA224: return ICP_QAT_HW_SHA224_STATE1_SZ; case ICP_QAT_HW_AUTH_ALGO_SHA256: return ICP_QAT_HW_SHA256_STATE1_SZ; case ICP_QAT_HW_AUTH_ALGO_SHA384: return ICP_QAT_HW_SHA384_STATE1_SZ; case ICP_QAT_HW_AUTH_ALGO_SHA512: return ICP_QAT_HW_SHA512_STATE1_SZ; case ICP_QAT_HW_AUTH_ALGO_MD5: return ICP_QAT_HW_MD5_STATE1_SZ; case ICP_QAT_HW_AUTH_ALGO_DELIMITER: /* return maximum digest size in this case */ return ICP_QAT_HW_SHA512_STATE1_SZ; default: PMD_DRV_LOG(ERR, "invalid hash alg %u", qat_hash_alg); return -EFAULT; }; return -EFAULT; } /* returns block size in byes per hash algo */ static int qat_hash_get_block_size(enum icp_qat_hw_auth_algo qat_hash_alg) { switch (qat_hash_alg) { case ICP_QAT_HW_AUTH_ALGO_SHA1: return SHA_CBLOCK; case ICP_QAT_HW_AUTH_ALGO_SHA224: return SHA256_CBLOCK; case ICP_QAT_HW_AUTH_ALGO_SHA256: return SHA256_CBLOCK; case ICP_QAT_HW_AUTH_ALGO_SHA384: return SHA512_CBLOCK; case ICP_QAT_HW_AUTH_ALGO_SHA512: return SHA512_CBLOCK; case ICP_QAT_HW_AUTH_ALGO_GALOIS_128: return 16; case ICP_QAT_HW_AUTH_ALGO_MD5: return MD5_CBLOCK; case ICP_QAT_HW_AUTH_ALGO_DELIMITER: /* return maximum block size in this case */ return SHA512_CBLOCK; default: PMD_DRV_LOG(ERR, "invalid hash alg %u", qat_hash_alg); return -EFAULT; }; return -EFAULT; } static int partial_hash_sha1(uint8_t *data_in, uint8_t *data_out) { SHA_CTX ctx; if (!SHA1_Init(&ctx)) return -EFAULT; SHA1_Transform(&ctx, data_in); rte_memcpy(data_out, &ctx, SHA_DIGEST_LENGTH); return 0; } static int partial_hash_sha224(uint8_t *data_in, uint8_t *data_out) { SHA256_CTX ctx; if (!SHA224_Init(&ctx)) return -EFAULT; SHA256_Transform(&ctx, data_in); rte_memcpy(data_out, &ctx, SHA256_DIGEST_LENGTH); return 0; } static int partial_hash_sha256(uint8_t *data_in, uint8_t *data_out) { SHA256_CTX ctx; if (!SHA256_Init(&ctx)) return -EFAULT; SHA256_Transform(&ctx, data_in); rte_memcpy(data_out, &ctx, SHA256_DIGEST_LENGTH); return 0; } static int partial_hash_sha384(uint8_t *data_in, uint8_t *data_out) { SHA512_CTX ctx; if (!SHA384_Init(&ctx)) return -EFAULT; SHA512_Transform(&ctx, data_in); rte_memcpy(data_out, &ctx, SHA512_DIGEST_LENGTH); return 0; } static int partial_hash_sha512(uint8_t *data_in, uint8_t *data_out) { SHA512_CTX ctx; if (!SHA512_Init(&ctx)) return -EFAULT; SHA512_Transform(&ctx, data_in); rte_memcpy(data_out, &ctx, SHA512_DIGEST_LENGTH); return 0; } static int partial_hash_md5(uint8_t *data_in, uint8_t *data_out) { MD5_CTX ctx; if (!MD5_Init(&ctx)) return -EFAULT; MD5_Transform(&ctx, data_in); rte_memcpy(data_out, &ctx, MD5_DIGEST_LENGTH); return 0; } static int partial_hash_compute(enum icp_qat_hw_auth_algo hash_alg, uint8_t *data_in, uint8_t *data_out) { int digest_size; uint8_t digest[qat_hash_get_digest_size( ICP_QAT_HW_AUTH_ALGO_DELIMITER)]; uint32_t *hash_state_out_be32; uint64_t *hash_state_out_be64; int i; PMD_INIT_FUNC_TRACE(); digest_size = qat_hash_get_digest_size(hash_alg); if (digest_size <= 0) return -EFAULT; hash_state_out_be32 = (uint32_t *)data_out; hash_state_out_be64 = (uint64_t *)data_out; switch (hash_alg) { case ICP_QAT_HW_AUTH_ALGO_SHA1: if (partial_hash_sha1(data_in, digest)) return -EFAULT; for (i = 0; i < digest_size >> 2; i++, hash_state_out_be32++) *hash_state_out_be32 = rte_bswap32(*(((uint32_t *)digest)+i)); break; case ICP_QAT_HW_AUTH_ALGO_SHA224: if (partial_hash_sha224(data_in, digest)) return -EFAULT; for (i = 0; i < digest_size >> 2; i++, hash_state_out_be32++) *hash_state_out_be32 = rte_bswap32(*(((uint32_t *)digest)+i)); break; case ICP_QAT_HW_AUTH_ALGO_SHA256: if (partial_hash_sha256(data_in, digest)) return -EFAULT; for (i = 0; i < digest_size >> 2; i++, hash_state_out_be32++) *hash_state_out_be32 = rte_bswap32(*(((uint32_t *)digest)+i)); break; case ICP_QAT_HW_AUTH_ALGO_SHA384: if (partial_hash_sha384(data_in, digest)) return -EFAULT; for (i = 0; i < digest_size >> 3; i++, hash_state_out_be64++) *hash_state_out_be64 = rte_bswap64(*(((uint64_t *)digest)+i)); break; case ICP_QAT_HW_AUTH_ALGO_SHA512: if (partial_hash_sha512(data_in, digest)) return -EFAULT; for (i = 0; i < digest_size >> 3; i++, hash_state_out_be64++) *hash_state_out_be64 = rte_bswap64(*(((uint64_t *)digest)+i)); break; case ICP_QAT_HW_AUTH_ALGO_MD5: if (partial_hash_md5(data_in, data_out)) return -EFAULT; break; default: PMD_DRV_LOG(ERR, "invalid hash alg %u", hash_alg); return -EFAULT; } return 0; } #define HMAC_IPAD_VALUE 0x36 #define HMAC_OPAD_VALUE 0x5c #define HASH_XCBC_PRECOMP_KEY_NUM 3 static int qat_alg_do_precomputes(enum icp_qat_hw_auth_algo hash_alg, const uint8_t *auth_key, uint16_t auth_keylen, uint8_t *p_state_buf, uint16_t *p_state_len) { int block_size; uint8_t ipad[qat_hash_get_block_size(ICP_QAT_HW_AUTH_ALGO_DELIMITER)]; uint8_t opad[qat_hash_get_block_size(ICP_QAT_HW_AUTH_ALGO_DELIMITER)]; int i; PMD_INIT_FUNC_TRACE(); if (hash_alg == ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC) { static uint8_t qat_aes_xcbc_key_seed[ ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ] = { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, }; uint8_t *in = NULL; uint8_t *out = p_state_buf; int x; AES_KEY enc_key; in = rte_zmalloc("working mem for key", ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ, 16); rte_memcpy(in, qat_aes_xcbc_key_seed, ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ); for (x = 0; x < HASH_XCBC_PRECOMP_KEY_NUM; x++) { if (AES_set_encrypt_key(auth_key, auth_keylen << 3, &enc_key) != 0) { rte_free(in - (x * ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ)); memset(out - (x * ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ), 0, ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ); return -EFAULT; } AES_encrypt(in, out, &enc_key); in += ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ; out += ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ; } *p_state_len = ICP_QAT_HW_AES_XCBC_MAC_STATE2_SZ; rte_free(in - x*ICP_QAT_HW_AES_XCBC_MAC_KEY_SZ); return 0; } else if ((hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128) || (hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64)) { uint8_t *in = NULL; uint8_t *out = p_state_buf; AES_KEY enc_key; memset(p_state_buf, 0, ICP_QAT_HW_GALOIS_H_SZ + ICP_QAT_HW_GALOIS_LEN_A_SZ + ICP_QAT_HW_GALOIS_E_CTR0_SZ); in = rte_zmalloc("working mem for key", ICP_QAT_HW_GALOIS_H_SZ, 16); memset(in, 0, ICP_QAT_HW_GALOIS_H_SZ); if (AES_set_encrypt_key(auth_key, auth_keylen << 3, &enc_key) != 0) { return -EFAULT; } AES_encrypt(in, out, &enc_key); *p_state_len = ICP_QAT_HW_GALOIS_H_SZ + ICP_QAT_HW_GALOIS_LEN_A_SZ + ICP_QAT_HW_GALOIS_E_CTR0_SZ; rte_free(in); return 0; } block_size = qat_hash_get_block_size(hash_alg); if (block_size <= 0) return -EFAULT; /* init ipad and opad from key and xor with fixed values */ memset(ipad, 0, block_size); memset(opad, 0, block_size); if (auth_keylen > (unsigned int)block_size) { PMD_DRV_LOG(ERR, "invalid keylen %u", auth_keylen); return -EFAULT; } rte_memcpy(ipad, auth_key, auth_keylen); rte_memcpy(opad, auth_key, auth_keylen); for (i = 0; i < block_size; i++) { uint8_t *ipad_ptr = ipad + i; uint8_t *opad_ptr = opad + i; *ipad_ptr ^= HMAC_IPAD_VALUE; *opad_ptr ^= HMAC_OPAD_VALUE; } /* do partial hash of ipad and copy to state1 */ if (partial_hash_compute(hash_alg, ipad, p_state_buf)) { memset(ipad, 0, block_size); memset(opad, 0, block_size); PMD_DRV_LOG(ERR, "ipad precompute failed"); return -EFAULT; } /* * State len is a multiple of 8, so may be larger than the digest. * Put the partial hash of opad state_len bytes after state1 */ *p_state_len = qat_hash_get_state1_size(hash_alg); if (partial_hash_compute(hash_alg, opad, p_state_buf + *p_state_len)) { memset(ipad, 0, block_size); memset(opad, 0, block_size); PMD_DRV_LOG(ERR, "opad precompute failed"); return -EFAULT; } /* don't leave data lying around */ memset(ipad, 0, block_size); memset(opad, 0, block_size); return 0; } void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header, enum qat_crypto_proto_flag proto_flags) { PMD_INIT_FUNC_TRACE(); header->hdr_flags = ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET); header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_LA; header->comn_req_flags = ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR, QAT_COMN_PTR_TYPE_FLAT); ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags, ICP_QAT_FW_LA_PARTIAL_NONE); ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags, ICP_QAT_FW_CIPH_IV_16BYTE_DATA); switch (proto_flags) { case QAT_CRYPTO_PROTO_FLAG_NONE: ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags, ICP_QAT_FW_LA_NO_PROTO); break; case QAT_CRYPTO_PROTO_FLAG_CCM: ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags, ICP_QAT_FW_LA_CCM_PROTO); break; case QAT_CRYPTO_PROTO_FLAG_GCM: ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags, ICP_QAT_FW_LA_GCM_PROTO); break; case QAT_CRYPTO_PROTO_FLAG_SNOW3G: ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags, ICP_QAT_FW_LA_SNOW_3G_PROTO); break; case QAT_CRYPTO_PROTO_FLAG_ZUC: ICP_QAT_FW_LA_ZUC_3G_PROTO_FLAG_SET(header->serv_specif_flags, ICP_QAT_FW_LA_ZUC_3G_PROTO); break; } ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags, ICP_QAT_FW_LA_NO_UPDATE_STATE); ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, ICP_QAT_FW_LA_NO_DIGEST_IN_BUFFER); } /* * Snow3G and ZUC should never use this function * and set its protocol flag in both cipher and auth part of content * descriptor building function */ static enum qat_crypto_proto_flag qat_get_crypto_proto_flag(uint16_t flags) { int proto = ICP_QAT_FW_LA_PROTO_GET(flags); enum qat_crypto_proto_flag qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_NONE; switch (proto) { case ICP_QAT_FW_LA_GCM_PROTO: qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_GCM; break; case ICP_QAT_FW_LA_CCM_PROTO: qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_CCM; break; } return qat_proto_flag; } int qat_alg_aead_session_create_content_desc_cipher(struct qat_session *cdesc, uint8_t *cipherkey, uint32_t cipherkeylen) { struct icp_qat_hw_cipher_algo_blk *cipher; struct icp_qat_fw_la_bulk_req *req_tmpl = &cdesc->fw_req; struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; void *ptr = &req_tmpl->cd_ctrl; struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr; struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr; enum icp_qat_hw_cipher_convert key_convert; enum qat_crypto_proto_flag qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_NONE; uint32_t total_key_size; uint16_t cipher_offset, cd_size; uint32_t wordIndex = 0; uint32_t *temp_key = NULL; PMD_INIT_FUNC_TRACE(); if (cdesc->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) { cd_pars->u.s.content_desc_addr = cdesc->cd_paddr; ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER); ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, ICP_QAT_FW_LA_NO_RET_AUTH_RES); ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, ICP_QAT_FW_LA_NO_CMP_AUTH_RES); cdesc->cd_cur_ptr = (uint8_t *)&cdesc->cd; } else if (cdesc->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) { cd_pars->u.s.content_desc_addr = cdesc->cd_paddr; ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER); ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_AUTH); ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH); ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); cdesc->cd_cur_ptr = (uint8_t *)&cdesc->cd; } else if (cdesc->qat_cmd != ICP_QAT_FW_LA_CMD_HASH_CIPHER) { PMD_DRV_LOG(ERR, "Invalid param, must be a cipher command."); return -EFAULT; } if (cdesc->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE) { /* * CTR Streaming ciphers are a special case. Decrypt = encrypt * Overriding default values previously set */ cdesc->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT; key_convert = ICP_QAT_HW_CIPHER_NO_CONVERT; } else if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 || cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) key_convert = ICP_QAT_HW_CIPHER_KEY_CONVERT; else if (cdesc->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) key_convert = ICP_QAT_HW_CIPHER_NO_CONVERT; else key_convert = ICP_QAT_HW_CIPHER_KEY_CONVERT; if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2) { total_key_size = ICP_QAT_HW_SNOW_3G_UEA2_KEY_SZ + ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ; cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ >> 3; qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_SNOW3G; } else if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI) { total_key_size = ICP_QAT_HW_KASUMI_F8_KEY_SZ; cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_KASUMI_BLK_SZ >> 3; cipher_cd_ctrl->cipher_padding_sz = (2 * ICP_QAT_HW_KASUMI_BLK_SZ) >> 3; } else if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_3DES) { total_key_size = ICP_QAT_HW_3DES_KEY_SZ; cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_3DES_BLK_SZ >> 3; qat_proto_flag = qat_get_crypto_proto_flag(header->serv_specif_flags); } else if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_DES) { total_key_size = ICP_QAT_HW_DES_KEY_SZ; cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_DES_BLK_SZ >> 3; qat_proto_flag = qat_get_crypto_proto_flag(header->serv_specif_flags); } else if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) { total_key_size = ICP_QAT_HW_ZUC_3G_EEA3_KEY_SZ + ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ; cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ >> 3; qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_ZUC; } else { total_key_size = cipherkeylen; cipher_cd_ctrl->cipher_state_sz = ICP_QAT_HW_AES_BLK_SZ >> 3; qat_proto_flag = qat_get_crypto_proto_flag(header->serv_specif_flags); } cipher_cd_ctrl->cipher_key_sz = total_key_size >> 3; cipher_offset = cdesc->cd_cur_ptr-((uint8_t *)&cdesc->cd); cipher_cd_ctrl->cipher_cfg_offset = cipher_offset >> 3; header->service_cmd_id = cdesc->qat_cmd; qat_alg_init_common_hdr(header, qat_proto_flag); cipher = (struct icp_qat_hw_cipher_algo_blk *)cdesc->cd_cur_ptr; cipher->cipher_config.val = ICP_QAT_HW_CIPHER_CONFIG_BUILD(cdesc->qat_mode, cdesc->qat_cipher_alg, key_convert, cdesc->qat_dir); if (cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI) { temp_key = (uint32_t *)(cdesc->cd_cur_ptr + sizeof(struct icp_qat_hw_cipher_config) + cipherkeylen); memcpy(cipher->key, cipherkey, cipherkeylen); memcpy(temp_key, cipherkey, cipherkeylen); /* XOR Key with KASUMI F8 key modifier at 4 bytes level */ for (wordIndex = 0; wordIndex < (cipherkeylen >> 2); wordIndex++) temp_key[wordIndex] ^= KASUMI_F8_KEY_MODIFIER_4_BYTES; cdesc->cd_cur_ptr += sizeof(struct icp_qat_hw_cipher_config) + cipherkeylen + cipherkeylen; } else { memcpy(cipher->key, cipherkey, cipherkeylen); cdesc->cd_cur_ptr += sizeof(struct icp_qat_hw_cipher_config) + cipherkeylen; } if (total_key_size > cipherkeylen) { uint32_t padding_size = total_key_size-cipherkeylen; if ((cdesc->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_3DES) && (cipherkeylen == QAT_3DES_KEY_SZ_OPT2)) /* K3 not provided so use K1 = K3*/ memcpy(cdesc->cd_cur_ptr, cipherkey, padding_size); else memset(cdesc->cd_cur_ptr, 0, padding_size); cdesc->cd_cur_ptr += padding_size; } cd_size = cdesc->cd_cur_ptr-(uint8_t *)&cdesc->cd; cd_pars->u.s.content_desc_params_sz = RTE_ALIGN_CEIL(cd_size, 8) >> 3; return 0; } int qat_alg_aead_session_create_content_desc_auth(struct qat_session *cdesc, uint8_t *authkey, uint32_t authkeylen, uint32_t add_auth_data_length, uint32_t digestsize, unsigned int operation) { struct icp_qat_hw_auth_setup *hash; struct icp_qat_hw_cipher_algo_blk *cipherconfig; struct icp_qat_fw_la_bulk_req *req_tmpl = &cdesc->fw_req; struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; void *ptr = &req_tmpl->cd_ctrl; struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr; struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr; struct icp_qat_fw_la_auth_req_params *auth_param = (struct icp_qat_fw_la_auth_req_params *) ((char *)&req_tmpl->serv_specif_rqpars + sizeof(struct icp_qat_fw_la_cipher_req_params)); uint16_t state1_size = 0, state2_size = 0; uint16_t hash_offset, cd_size; uint32_t *aad_len = NULL; uint32_t wordIndex = 0; uint32_t *pTempKey; enum qat_crypto_proto_flag qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_NONE; PMD_INIT_FUNC_TRACE(); if (cdesc->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH) { ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH); ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); cdesc->cd_cur_ptr = (uint8_t *)&cdesc->cd; } else if (cdesc->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER) { ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH); ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER); ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER); ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); cdesc->cd_cur_ptr = (uint8_t *)&cdesc->cd; } else if (cdesc->qat_cmd != ICP_QAT_FW_LA_CMD_CIPHER_HASH) { PMD_DRV_LOG(ERR, "Invalid param, must be a hash command."); return -EFAULT; } if (operation == RTE_CRYPTO_AUTH_OP_VERIFY) { ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, ICP_QAT_FW_LA_NO_RET_AUTH_RES); ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, ICP_QAT_FW_LA_CMP_AUTH_RES); cdesc->auth_op = ICP_QAT_HW_AUTH_VERIFY; } else { ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, ICP_QAT_FW_LA_RET_AUTH_RES); ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, ICP_QAT_FW_LA_NO_CMP_AUTH_RES); cdesc->auth_op = ICP_QAT_HW_AUTH_GENERATE; } /* * Setup the inner hash config */ hash_offset = cdesc->cd_cur_ptr-((uint8_t *)&cdesc->cd); hash = (struct icp_qat_hw_auth_setup *)cdesc->cd_cur_ptr; hash->auth_config.reserved = 0; hash->auth_config.config = ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1, cdesc->qat_hash_alg, digestsize); if (cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 || cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 || cdesc->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3) hash->auth_counter.counter = 0; else hash->auth_counter.counter = rte_bswap32( qat_hash_get_block_size(cdesc->qat_hash_alg)); cdesc->cd_cur_ptr += sizeof(struct icp_qat_hw_auth_setup); /* * cd_cur_ptr now points at the state1 information. */ switch (cdesc->qat_hash_alg) { case ICP_QAT_HW_AUTH_ALGO_SHA1: if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA1, authkey, authkeylen, cdesc->cd_cur_ptr, &state1_size)) { PMD_DRV_LOG(ERR, "(SHA)precompute failed"); return -EFAULT; } state2_size = RTE_ALIGN_CEIL(ICP_QAT_HW_SHA1_STATE2_SZ, 8); break; case ICP_QAT_HW_AUTH_ALGO_SHA224: if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA224, authkey, authkeylen, cdesc->cd_cur_ptr, &state1_size)) { PMD_DRV_LOG(ERR, "(SHA)precompute failed"); return -EFAULT; } state2_size = ICP_QAT_HW_SHA224_STATE2_SZ; break; case ICP_QAT_HW_AUTH_ALGO_SHA256: if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA256, authkey, authkeylen, cdesc->cd_cur_ptr, &state1_size)) { PMD_DRV_LOG(ERR, "(SHA)precompute failed"); return -EFAULT; } state2_size = ICP_QAT_HW_SHA256_STATE2_SZ; break; case ICP_QAT_HW_AUTH_ALGO_SHA384: if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA384, authkey, authkeylen, cdesc->cd_cur_ptr, &state1_size)) { PMD_DRV_LOG(ERR, "(SHA)precompute failed"); return -EFAULT; } state2_size = ICP_QAT_HW_SHA384_STATE2_SZ; break; case ICP_QAT_HW_AUTH_ALGO_SHA512: if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_SHA512, authkey, authkeylen, cdesc->cd_cur_ptr, &state1_size)) { PMD_DRV_LOG(ERR, "(SHA)precompute failed"); return -EFAULT; } state2_size = ICP_QAT_HW_SHA512_STATE2_SZ; break; case ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC: state1_size = ICP_QAT_HW_AES_XCBC_MAC_STATE1_SZ; if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC, authkey, authkeylen, cdesc->cd_cur_ptr + state1_size, &state2_size)) { PMD_DRV_LOG(ERR, "(XCBC)precompute failed"); return -EFAULT; } break; case ICP_QAT_HW_AUTH_ALGO_GALOIS_128: case ICP_QAT_HW_AUTH_ALGO_GALOIS_64: qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_GCM; state1_size = ICP_QAT_HW_GALOIS_128_STATE1_SZ; if (qat_alg_do_precomputes(cdesc->qat_hash_alg, authkey, authkeylen, cdesc->cd_cur_ptr + state1_size, &state2_size)) { PMD_DRV_LOG(ERR, "(GCM)precompute failed"); return -EFAULT; } /* * Write (the length of AAD) into bytes 16-19 of state2 * in big-endian format. This field is 8 bytes */ auth_param->u2.aad_sz = RTE_ALIGN_CEIL(add_auth_data_length, 16); auth_param->hash_state_sz = (auth_param->u2.aad_sz) >> 3; aad_len = (uint32_t *)(cdesc->cd_cur_ptr + ICP_QAT_HW_GALOIS_128_STATE1_SZ + ICP_QAT_HW_GALOIS_H_SZ); *aad_len = rte_bswap32(add_auth_data_length); break; case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2: qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_SNOW3G; state1_size = qat_hash_get_state1_size( ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2); state2_size = ICP_QAT_HW_SNOW_3G_UIA2_STATE2_SZ; memset(cdesc->cd_cur_ptr, 0, state1_size + state2_size); cipherconfig = (struct icp_qat_hw_cipher_algo_blk *) (cdesc->cd_cur_ptr + state1_size + state2_size); cipherconfig->cipher_config.val = ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_ECB_MODE, ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2, ICP_QAT_HW_CIPHER_KEY_CONVERT, ICP_QAT_HW_CIPHER_ENCRYPT); memcpy(cipherconfig->key, authkey, authkeylen); memset(cipherconfig->key + authkeylen, 0, ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ); cdesc->cd_cur_ptr += sizeof(struct icp_qat_hw_cipher_config) + authkeylen + ICP_QAT_HW_SNOW_3G_UEA2_IV_SZ; auth_param->hash_state_sz = RTE_ALIGN_CEIL(add_auth_data_length, 16) >> 3; break; case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3: hash->auth_config.config = ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE0, cdesc->qat_hash_alg, digestsize); qat_proto_flag = QAT_CRYPTO_PROTO_FLAG_ZUC; state1_size = qat_hash_get_state1_size( ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3); state2_size = ICP_QAT_HW_ZUC_3G_EIA3_STATE2_SZ; memset(cdesc->cd_cur_ptr, 0, state1_size + state2_size + ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ); memcpy(cdesc->cd_cur_ptr + state1_size, authkey, authkeylen); cdesc->cd_cur_ptr += state1_size + state2_size + ICP_QAT_HW_ZUC_3G_EEA3_IV_SZ; auth_param->hash_state_sz = RTE_ALIGN_CEIL(add_auth_data_length, 16) >> 3; break; case ICP_QAT_HW_AUTH_ALGO_MD5: if (qat_alg_do_precomputes(ICP_QAT_HW_AUTH_ALGO_MD5, authkey, authkeylen, cdesc->cd_cur_ptr, &state1_size)) { PMD_DRV_LOG(ERR, "(MD5)precompute failed"); return -EFAULT; } state2_size = ICP_QAT_HW_MD5_STATE2_SZ; break; case ICP_QAT_HW_AUTH_ALGO_NULL: state1_size = qat_hash_get_state1_size( ICP_QAT_HW_AUTH_ALGO_NULL); state2_size = ICP_QAT_HW_NULL_STATE2_SZ; break; case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9: state1_size = qat_hash_get_state1_size( ICP_QAT_HW_AUTH_ALGO_KASUMI_F9); state2_size = ICP_QAT_HW_KASUMI_F9_STATE2_SZ; memset(cdesc->cd_cur_ptr, 0, state1_size + state2_size); pTempKey = (uint32_t *)(cdesc->cd_cur_ptr + state1_size + authkeylen); /* * The Inner Hash Initial State2 block must contain IK * (Initialisation Key), followed by IK XOR-ed with KM * (Key Modifier): IK||(IK^KM). */ /* write the auth key */ memcpy(cdesc->cd_cur_ptr + state1_size, authkey, authkeylen); /* initialise temp key with auth key */ memcpy(pTempKey, authkey, authkeylen); /* XOR Key with KASUMI F9 key modifier at 4 bytes level */ for (wordIndex = 0; wordIndex < (authkeylen >> 2); wordIndex++) pTempKey[wordIndex] ^= KASUMI_F9_KEY_MODIFIER_4_BYTES; break; default: PMD_DRV_LOG(ERR, "Invalid HASH alg %u", cdesc->qat_hash_alg); return -EFAULT; } /* Request template setup */ qat_alg_init_common_hdr(header, qat_proto_flag); header->service_cmd_id = cdesc->qat_cmd; /* Auth CD config setup */ hash_cd_ctrl->hash_cfg_offset = hash_offset >> 3; hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED; hash_cd_ctrl->inner_res_sz = digestsize; hash_cd_ctrl->final_sz = digestsize; hash_cd_ctrl->inner_state1_sz = state1_size; auth_param->auth_res_sz = digestsize; hash_cd_ctrl->inner_state2_sz = state2_size; hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset + ((sizeof(struct icp_qat_hw_auth_setup) + RTE_ALIGN_CEIL(hash_cd_ctrl->inner_state1_sz, 8)) >> 3); cdesc->cd_cur_ptr += state1_size + state2_size; cd_size = cdesc->cd_cur_ptr-(uint8_t *)&cdesc->cd; cd_pars->u.s.content_desc_addr = cdesc->cd_paddr; cd_pars->u.s.content_desc_params_sz = RTE_ALIGN_CEIL(cd_size, 8) >> 3; return 0; } int qat_alg_validate_aes_key(int key_len, enum icp_qat_hw_cipher_algo *alg) { switch (key_len) { case ICP_QAT_HW_AES_128_KEY_SZ: *alg = ICP_QAT_HW_CIPHER_ALGO_AES128; break; case ICP_QAT_HW_AES_192_KEY_SZ: *alg = ICP_QAT_HW_CIPHER_ALGO_AES192; break; case ICP_QAT_HW_AES_256_KEY_SZ: *alg = ICP_QAT_HW_CIPHER_ALGO_AES256; break; default: return -EINVAL; } return 0; } int qat_alg_validate_aes_docsisbpi_key(int key_len, enum icp_qat_hw_cipher_algo *alg) { switch (key_len) { case ICP_QAT_HW_AES_128_KEY_SZ: *alg = ICP_QAT_HW_CIPHER_ALGO_AES128; break; default: return -EINVAL; } return 0; } int qat_alg_validate_snow3g_key(int key_len, enum icp_qat_hw_cipher_algo *alg) { switch (key_len) { case ICP_QAT_HW_SNOW_3G_UEA2_KEY_SZ: *alg = ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2; break; default: return -EINVAL; } return 0; } int qat_alg_validate_kasumi_key(int key_len, enum icp_qat_hw_cipher_algo *alg) { switch (key_len) { case ICP_QAT_HW_KASUMI_KEY_SZ: *alg = ICP_QAT_HW_CIPHER_ALGO_KASUMI; break; default: return -EINVAL; } return 0; } int qat_alg_validate_des_key(int key_len, enum icp_qat_hw_cipher_algo *alg) { switch (key_len) { case ICP_QAT_HW_DES_KEY_SZ: *alg = ICP_QAT_HW_CIPHER_ALGO_DES; break; default: return -EINVAL; } return 0; } int qat_alg_validate_3des_key(int key_len, enum icp_qat_hw_cipher_algo *alg) { switch (key_len) { case QAT_3DES_KEY_SZ_OPT1: case QAT_3DES_KEY_SZ_OPT2: *alg = ICP_QAT_HW_CIPHER_ALGO_3DES; break; default: return -EINVAL; } return 0; } int qat_alg_validate_zuc_key(int key_len, enum icp_qat_hw_cipher_algo *alg) { switch (key_len) { case ICP_QAT_HW_ZUC_3G_EEA3_KEY_SZ: *alg = ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3; break; default: return -EINVAL; } return 0; }