_(IP4_INPUT, "ip4-input-no-checksum") \
_(IP6_INPUT, "ip6-input") \
_(L2_INPUT, "l2-input") \
-_(HANDOFF, "handoff") \
-_(PENDING, "pending")
+_(HANDOFF, "handoff")
#define _(v, s) ESP_DECRYPT_NEXT_##v,
typedef enum
/* allocate buffer failed, will not add to frame and drop */
b->error = node->errors[ESP_DECRYPT_ERROR_NO_BUFFERS];
next[0] = ESP_DECRYPT_NEXT_DROP;
- return 0;
+ return -1;
}
}
else
*async_pd = *pd;
*async_pd2 = *pd2;
pd->protect_index = current_protect_index;
- next[0] = ESP_DECRYPT_NEXT_PENDING;
/* for AEAD integ_len - crypto_len will be negative, it is ok since it
* is ignored by the engine. */
}
}
-/* when submitting a frame is failed, drop all buffers in the frame */
-static_always_inline void
-esp_async_recycle_failed_submit (vnet_crypto_async_frame_t * f,
- vlib_buffer_t ** b, u16 * next)
-{
- u32 n_drop = f->n_elts;
- while (--n_drop)
- {
- (b - n_drop)[0]->error = ESP_DECRYPT_ERROR_CRYPTO_ENGINE_ERROR;
- (next - n_drop)[0] = ESP_DECRYPT_NEXT_DROP;
- }
- vnet_crypto_async_reset_frame (f);
-}
-
always_inline uword
esp_decrypt_inline (vlib_main_t * vm,
vlib_node_runtime_t * node, vlib_frame_t * from_frame,
vnet_crypto_async_frame_t *async_frame = 0;
int is_async = im->async_mode;
vnet_crypto_async_op_id_t last_async_op = ~0;
+ u16 n_async_drop = 0;
vlib_get_buffers (vm, from, b, n_left);
if (!is_async)
if (n_bufs == 0)
{
b[0]->error = node->errors[ESP_DECRYPT_ERROR_NO_BUFFERS];
+ esp_set_next_index (is_async, from, nexts, from[b - bufs],
+ &n_async_drop, ESP_DECRYPT_NEXT_DROP, next);
next[0] = ESP_DECRYPT_NEXT_DROP;
goto next;
}
if (async_frame && async_frame->n_elts)
{
if (vnet_crypto_async_submit_open_frame (vm, async_frame))
- esp_async_recycle_failed_submit (async_frame, b, next);
+ esp_async_recycle_failed_submit (async_frame, b, from,
+ nexts, &n_async_drop,
+ ESP_DECRYPT_NEXT_DROP,
+ ESP_DECRYPT_ERROR_CRYPTO_ENGINE_ERROR);
}
async_frame =
vnet_crypto_async_get_frame (vm, sa0->crypto_async_dec_op_id);
if (PREDICT_TRUE (thread_index != sa0->decrypt_thread_index))
{
+ esp_set_next_index (is_async, from, nexts, from[b - bufs],
+ &n_async_drop, ESP_DECRYPT_NEXT_HANDOFF, next);
next[0] = ESP_DECRYPT_NEXT_HANDOFF;
goto next;
}
if (ipsec_sa_anti_replay_check (sa0, pd->seq))
{
b[0]->error = node->errors[ESP_DECRYPT_ERROR_REPLAY];
- next[0] = ESP_DECRYPT_NEXT_DROP;
+ esp_set_next_index (is_async, from, nexts, from[b - bufs],
+ &n_async_drop, ESP_DECRYPT_NEXT_DROP, next);
goto next;
}
if (pd->current_length < cpd.icv_sz + esp_sz + cpd.iv_sz)
{
b[0]->error = node->errors[ESP_DECRYPT_ERROR_RUNT];
- next[0] = ESP_DECRYPT_NEXT_DROP;
+ esp_set_next_index (is_async, from, nexts, from[b - bufs],
+ &n_async_drop, ESP_DECRYPT_NEXT_DROP, next);
goto next;
}
b[0], next, async_next);
if (PREDICT_FALSE (ret < 0))
{
- esp_async_recycle_failed_submit (async_frame, b, next);
+ b[0]->error = ESP_DECRYPT_ERROR_CRYPTO_ENGINE_ERROR;
+ esp_set_next_index (1, from, nexts, from[b - bufs],
+ &n_async_drop, ESP_DECRYPT_NEXT_DROP, next);
+ /* when next[0] is ESP_DECRYPT_NEXT_DROP we only have to drop
+ * the current packet. Otherwise it is frame submission error
+ * thus we have to drop the whole frame.
+ */
+ if (next[0] != ESP_DECRYPT_NEXT_DROP && async_frame->n_elts)
+ esp_async_recycle_failed_submit (async_frame, b, from,
+ nexts, &n_async_drop,
+ ESP_DECRYPT_NEXT_DROP,
+ ESP_DECRYPT_ERROR_CRYPTO_ENGINE_ERROR);
goto next;
}
}
if (async_frame && async_frame->n_elts)
{
if (vnet_crypto_async_submit_open_frame (vm, async_frame) < 0)
- esp_async_recycle_failed_submit (async_frame, b, next);
+ esp_async_recycle_failed_submit (async_frame, b, from, nexts,
+ &n_async_drop,
+ ESP_DECRYPT_NEXT_DROP,
+ ESP_DECRYPT_ERROR_CRYPTO_ENGINE_ERROR);
}
/* no post process in async */
- n_left = from_frame->n_vectors;
vlib_node_increment_counter (vm, node->node_index,
ESP_DECRYPT_ERROR_RX_PKTS, n_left);
- vlib_buffer_enqueue_to_next (vm, node, from, nexts, n_left);
+ if (n_async_drop)
+ vlib_buffer_enqueue_to_next (vm, node, from, nexts, n_async_drop);
return n_left;
}
return esp_decrypt_post_inline (vm, node, from_frame, 1, 1);
}
-VLIB_NODE_FN (esp_decrypt_pending_node) (vlib_main_t * vm,
- vlib_node_runtime_t * node,
- vlib_frame_t * from_frame)
-{
- return from_frame->n_vectors;
-}
-
-/* *INDENT-OFF* */
-VLIB_REGISTER_NODE (esp_decrypt_pending_node) = {
- .name = "esp-decrypt-pending",
- .vector_size = sizeof (u32),
- .type = VLIB_NODE_TYPE_INTERNAL,
-
- .n_next_nodes = 0
-};
-/* *INDENT-ON* */
-
/* *INDENT-OFF* */
VLIB_REGISTER_NODE (esp4_decrypt_node) = {
.name = "esp4-decrypt",
[ESP_DECRYPT_NEXT_IP6_INPUT] = "ip6-input",
[ESP_DECRYPT_NEXT_L2_INPUT] = "l2-input",
[ESP_DECRYPT_NEXT_HANDOFF] = "esp4-decrypt-handoff",
- [ESP_DECRYPT_NEXT_PENDING] = "esp-decrypt-pending"
},
};
[ESP_DECRYPT_NEXT_IP6_INPUT] = "ip6-input",
[ESP_DECRYPT_NEXT_L2_INPUT] = "l2-input",
[ESP_DECRYPT_NEXT_HANDOFF]= "esp6-decrypt-handoff",
- [ESP_DECRYPT_NEXT_PENDING] = "esp-decrypt-pending"
},
};
[ESP_DECRYPT_NEXT_IP6_INPUT] = "ip6-input",
[ESP_DECRYPT_NEXT_L2_INPUT] = "l2-input",
[ESP_DECRYPT_NEXT_HANDOFF] = "esp4-decrypt-tun-handoff",
- [ESP_DECRYPT_NEXT_PENDING] = "esp-decrypt-pending"
},
};
[ESP_DECRYPT_NEXT_IP6_INPUT] = "ip6-input",
[ESP_DECRYPT_NEXT_L2_INPUT] = "l2-input",
[ESP_DECRYPT_NEXT_HANDOFF]= "esp6-decrypt-tun-handoff",
- [ESP_DECRYPT_NEXT_PENDING] = "esp-decrypt-pending"
},
};
#define foreach_esp_encrypt_next \
_(DROP4, "ip4-drop") \
_(DROP6, "ip6-drop") \
-_(PENDING, "pending") \
_(HANDOFF4, "handoff4") \
_(HANDOFF6, "handoff6") \
_(INTERFACE_OUTPUT, "interface-output")
vnet_crypto_async_frame_t ** async_frame,
ipsec_sa_t * sa, vlib_buffer_t * b,
esp_header_t * esp, u8 * payload, u32 payload_len,
- u8 iv_sz, u8 icv_sz, u32 bi, u16 * next, u32 hdr_len,
+ u8 iv_sz, u8 icv_sz, u32 bi, u16 next, u32 hdr_len,
u16 async_next, vlib_buffer_t * lb)
{
esp_post_data_t *post = esp_post_data (b);
i16 crypto_start_offset, integ_start_offset = 0;
u16 crypto_total_len, integ_total_len;
- post->next_index = next[0];
- next[0] = ESP_ENCRYPT_NEXT_PENDING;
+ post->next_index = next;
/* crypto */
crypto_start_offset = payload - b->data;
iv, tag, aad, flag);
}
-/* when submitting a frame is failed, drop all buffers in the frame */
-static_always_inline void
-esp_async_recycle_failed_submit (vnet_crypto_async_frame_t * f,
- vlib_buffer_t ** b, u16 * next,
- u16 drop_next)
-{
- u32 n_drop = f->n_elts;
- while (--n_drop)
- {
- (b - n_drop)[0]->error = ESP_ENCRYPT_ERROR_CRYPTO_ENGINE_ERROR;
- (next - n_drop)[0] = drop_next;
- }
- vnet_crypto_async_reset_frame (f);
-}
-
always_inline uword
esp_encrypt_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
vlib_frame_t * frame, int is_ip6, int is_tun,
int is_async = im->async_mode;
vnet_crypto_async_op_id_t last_async_op = ~0;
u16 drop_next = (is_ip6 ? ESP_ENCRYPT_NEXT_DROP6 : ESP_ENCRYPT_NEXT_DROP4);
+ u16 n_async_drop = 0;
vlib_get_buffers (vm, from, b, n_left);
if (!is_async)
{
if (async_frame && async_frame->n_elts)
{
- if (vnet_crypto_async_submit_open_frame (vm, async_frame)
- < 0)
- esp_async_recycle_failed_submit (async_frame, b,
- next, drop_next);
+ if (vnet_crypto_async_submit_open_frame (vm, async_frame))
+ esp_async_recycle_failed_submit (async_frame, b, from,
+ nexts, &n_async_drop,
+ drop_next,
+ ESP_ENCRYPT_ERROR_CRYPTO_ENGINE_ERROR);
}
async_frame =
vnet_crypto_async_get_frame (vm, sa0->crypto_async_enc_op_id);
if (PREDICT_TRUE (thread_index != sa0->encrypt_thread_index))
{
- next[0] = (is_ip6 ?
- ESP_ENCRYPT_NEXT_HANDOFF6 : ESP_ENCRYPT_NEXT_HANDOFF4);
+ esp_set_next_index (is_async, from, nexts, from[b - bufs],
+ &n_async_drop,
+ (is_ip6 ? ESP_ENCRYPT_NEXT_HANDOFF6 :
+ ESP_ENCRYPT_NEXT_HANDOFF4), next);
goto trace;
}
if (n_bufs == 0)
{
b[0]->error = node->errors[ESP_ENCRYPT_ERROR_NO_BUFFERS];
- next[0] = drop_next;
+ esp_set_next_index (is_async, from, nexts, from[b - bufs],
+ &n_async_drop, drop_next, next);
goto trace;
}
if (PREDICT_FALSE (esp_seq_advance (sa0)))
{
b[0]->error = node->errors[ESP_ENCRYPT_ERROR_SEQ_CYCLED];
- next[0] = drop_next;
+ esp_set_next_index (is_async, from, nexts, from[b - bufs],
+ &n_async_drop, drop_next, next);
goto trace;
}
if (!next_hdr_ptr)
{
b[0]->error = node->errors[ESP_ENCRYPT_ERROR_NO_BUFFERS];
- next[0] = drop_next;
+ esp_set_next_index (is_async, from, nexts, from[b - bufs],
+ &n_async_drop, drop_next, next);
goto trace;
}
b[0]->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
vlib_buffer_length_in_chain
(vm, b[0]));
if (!next_hdr_ptr)
- goto trace;
+ {
+ esp_set_next_index (is_async, from, nexts, from[b - bufs],
+ &n_async_drop, drop_next, next);
+ goto trace;
+ }
b[0]->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
payload_len = b[0]->current_length;
if (is_async)
{
if (PREDICT_FALSE (sa0->crypto_async_enc_op_id == 0))
- goto trace;
+ {
+ esp_set_next_index (is_async, from, nexts, from[b - bufs],
+ &n_async_drop, drop_next, next);
+ goto trace;
+ }
if (esp_prepare_async_frame (vm, ptd, &async_frame, sa0, b[0], esp,
payload, payload_len, iv_sz,
- icv_sz, from[b - bufs], next, hdr_len,
- async_next, lb))
+ icv_sz, from[b - bufs], next[0],
+ hdr_len, async_next, lb))
{
- esp_async_recycle_failed_submit (async_frame, b, next,
- drop_next);
+ /* The fail only caused by submission, free the whole frame. */
+ if (async_frame->n_elts)
+ esp_async_recycle_failed_submit (async_frame, b, from, nexts,
+ &n_async_drop, drop_next,
+ ESP_ENCRYPT_ERROR_CRYPTO_ENGINE_ERROR);
+ b[0]->error = ESP_ENCRYPT_ERROR_CRYPTO_ENGINE_ERROR;
+ esp_set_next_index (1, from, nexts, from[b - bufs],
+ &n_async_drop, drop_next, next);
goto trace;
}
}
esp_process_chained_ops (vm, node, ptd->chained_integ_ops, bufs, nexts,
ptd->chunks, drop_next);
}
- else if (async_frame && async_frame->n_elts)
+ else
{
- if (vnet_crypto_async_submit_open_frame (vm, async_frame) < 0)
- esp_async_recycle_failed_submit (async_frame, b, next, drop_next);
+ if (async_frame && async_frame->n_elts)
+ {
+ if (vnet_crypto_async_submit_open_frame (vm, async_frame) < 0)
+ esp_async_recycle_failed_submit (async_frame, b, from, nexts,
+ &n_async_drop, drop_next,
+ ESP_ENCRYPT_ERROR_CRYPTO_ENGINE_ERROR);
+ }
+ vlib_node_increment_counter (vm, node->node_index,
+ ESP_ENCRYPT_ERROR_RX_PKTS,
+ frame->n_vectors);
+ if (n_async_drop)
+ vlib_buffer_enqueue_to_next (vm, node, from, nexts, n_async_drop);
+
+ return frame->n_vectors;
}
vlib_node_increment_counter (vm, node->node_index,
[ESP_ENCRYPT_NEXT_DROP6] = "ip6-drop",
[ESP_ENCRYPT_NEXT_HANDOFF4] = "esp4-encrypt-handoff",
[ESP_ENCRYPT_NEXT_HANDOFF6] = "esp6-encrypt-handoff",
- [ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT] = "interface-output",
- [ESP_ENCRYPT_NEXT_PENDING] = "esp-encrypt-pending",
+ [ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT] = "interface-output"
},
};
/* *INDENT-ON* */
[ESP_ENCRYPT_NEXT_HANDOFF4] = "esp4-encrypt-tun-handoff",
[ESP_ENCRYPT_NEXT_HANDOFF6] = "error-drop",
[ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT] = "adj-midchain-tx",
- [ESP_ENCRYPT_NEXT_PENDING] = "esp-encrypt-pending",
},
};
[ESP_ENCRYPT_NEXT_DROP6] = "ip6-drop",
[ESP_ENCRYPT_NEXT_HANDOFF4] = "error-drop",
[ESP_ENCRYPT_NEXT_HANDOFF6] = "esp6-encrypt-tun-handoff",
- [ESP_ENCRYPT_NEXT_PENDING] = "esp-encrypt-pending",
[ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT] = "adj-midchain-tx",
},
};
};
/* *INDENT-ON* */
-VLIB_NODE_FN (esp_encrypt_pending_node) (vlib_main_t * vm,
- vlib_node_runtime_t * node,
- vlib_frame_t * from_frame)
-{
- return from_frame->n_vectors;
-}
-
-/* *INDENT-OFF* */
-VLIB_REGISTER_NODE (esp_encrypt_pending_node) = {
- .name = "esp-encrypt-pending",
- .vector_size = sizeof (u32),
- .type = VLIB_NODE_TYPE_INTERNAL,
-
- .n_next_nodes = 0
-};
-/* *INDENT-ON* */
-
/*
* fd.io coding-style-patch-verification: ON
*
Code Review / vpp.git / commitdiff