dpdk-cryptodev: improve cryptodev cache ring implementation

Sw ring is renamed to the cache ring. This name better reflects the
puropse of this ring. We've introduced push/pop functions, as well as
other utility functions which remove code repetition. Error handlig
is improved: previously in case of an error all frame elements were
marked as bad, now only these for which errors occured have the error
status set.
Unnecessary stats counters have been removed.

Type: improvement
Signed-off-by: Piotr Bronowski <piotrx.bronowski@intel.com>
Change-Id: I2fd42a529ac84ce5ad260611d6b35a861d441c79

src/plugins/dpdk/cryptodev/cryptodev.c

@@ -667,37 +667,66 @@ VLIB_CLI_COMMAND (show_cryptodev_assignment, static) = {
 };
 
 static clib_error_t *
-cryptodev_show_sw_rings_fn (vlib_main_t *vm, unformat_input_t *input,
-			    vlib_cli_command_t *cmd)
+cryptodev_show_cache_rings_fn (vlib_main_t *vm, unformat_input_t *input,
+			       vlib_cli_command_t *cmd)
 {
   cryptodev_main_t *cmt = &cryptodev_main;
   u32 thread_index = 0;
   vec_foreach_index (thread_index, cmt->per_thread_data)
     {
       cryptodev_engine_thread_t *cet = cmt->per_thread_data + thread_index;
+      cryptodev_cache_ring_t *ring = &cet->cache_ring;
+      u16 head = ring->head;
+      u16 tail = ring->tail;
+      u16 n_cached = ((head == tail) && (ring->frames[head].f == 0)) ?
+			     0 :
+		     ((head == tail) && (ring->frames[head].f != 0)) ?
+			     (CRYPTODEV_CACHE_QUEUE_MASK + 1) :
+		     (head > tail) ?
+			     (head - tail) :
+			     (CRYPTODEV_CACHE_QUEUE_MASK - tail + head);
+
+      u16 enq_head = ring->enq_head;
+      u16 deq_tail = ring->deq_tail;
+      u16 n_frames_inflight =
+	((enq_head == deq_tail) && (ring->frames[enq_head].f == 0)) ?
+		0 :
+	((enq_head == deq_tail) && (ring->frames[enq_head].f != 0)) ?
+		CRYPTODEV_CACHE_QUEUE_MASK + 1 :
+	(enq_head > deq_tail) ?
+		(enq_head - deq_tail) :
+		(CRYPTODEV_CACHE_QUEUE_MASK - deq_tail + enq_head);
+
+      u16 n_frames_processed =
+	((tail == deq_tail) && (ring->frames[deq_tail].f == 0)) ?
+		0 :
+	((tail == deq_tail) && (ring->frames[deq_tail].f != 0)) ?
+				  (CRYPTODEV_CACHE_QUEUE_MASK + 1) :
+	(deq_tail > tail) ? (deq_tail - tail) :
+				  (CRYPTODEV_CACHE_QUEUE_MASK - tail + deq_tail);
+
       if (vlib_num_workers () > 0 && thread_index == 0)
 	continue;
       vlib_cli_output (vm, "\n\n");
-      vlib_cli_output (vm, "Frames total: %d", cet->frames_on_ring);
-      vlib_cli_output (vm, "Frames pending in a ring: %d",
-		       cet->frames_on_ring - cet->enqueued_not_dequeueq -
-			 cet->deqeued_not_returned);
+      vlib_cli_output (vm, "Frames total: %d", n_cached);
+      vlib_cli_output (vm, "Frames pending in the ring: %d",
+		       n_cached - n_frames_inflight - n_frames_processed);
       vlib_cli_output (vm, "Frames enqueued but not dequeued: %d",
-		       cet->enqueued_not_dequeueq);
+		       n_frames_inflight);
       vlib_cli_output (vm, "Frames dequed but not returned: %d",
-		       cet->deqeued_not_returned);
+		       n_frames_processed);
       vlib_cli_output (vm, "inflight: %d", cet->inflight);
-      vlib_cli_output (vm, "Head: %d", cet->frame_ring.head);
-      vlib_cli_output (vm, "Tail: %d", cet->frame_ring.tail);
+      vlib_cli_output (vm, "Head: %d", ring->head);
+      vlib_cli_output (vm, "Tail: %d", ring->tail);
       vlib_cli_output (vm, "\n\n");
     }
   return 0;
 }
 
 VLIB_CLI_COMMAND (show_cryptodev_sw_rings, static) = {
-  .path = "show cryptodev sw-ring status",
-  .short_help = "show status of all cryptodev software rings",
-  .function = cryptodev_show_sw_rings_fn,
+  .path = "show cryptodev cache status",
+  .short_help = "show status of all cryptodev cache rings",
+  .function = cryptodev_show_cache_rings_fn,
 };
 
 static clib_error_t *

src/plugins/dpdk/cryptodev/cryptodev.h

@@ -156,26 +156,51 @@ typedef struct
 typedef struct
 {
   vnet_crypto_async_frame_t *f;
+  union
+  {
+    struct
+    {
+      /* index of frame elt where enque to
+       * the crypto engine is happening */
+      u8 enq_elts_head;
+      /* index of the frame elt where dequeue
+       * from the crypto engine is happening */
+      u8 deq_elts_tail;
+      u8 elts_inflight;
 
-  u8 enqueued;
-  u8 dequeued;
-  u8 deq_state;
-  u8 frame_inflight;
+      u8 op_type;
+      u8 aad_len;
+      u8 n_elts;
+      u16 reserved;
+    };
+    u64 raw;
+  };
 
-  u8 op_type;
-  u8 aad_len;
-  u8 n_elts;
-  u8 reserved;
-} cryptodev_async_ring_elt;
+  u64 frame_elts_errs_mask;
+} cryptodev_cache_ring_elt_t;
 
 typedef struct
 {
-  cryptodev_async_ring_elt frames[VNET_CRYPTO_FRAME_POOL_SIZE];
-  uint16_t head;
-  uint16_t tail;
-  uint16_t enq; /*record the frame currently being enqueued */
-  uint16_t deq; /*record the frame currently being dequeued */
-} cryptodev_async_frame_sw_ring;
+  cryptodev_cache_ring_elt_t frames[VNET_CRYPTO_FRAME_POOL_SIZE];
+
+  union
+  {
+    struct
+    {
+      /* head of the cache ring */
+      u16 head;
+      /* tail of the cache ring */
+      u16 tail;
+      /* index of the frame where enqueue
+       * to the crypto engine is happening */
+      u16 enq_head;
+      /* index of the frame where dequeue
+       * from the crypto engine is happening */
+      u16 deq_tail;
+    };
+    u64 raw;
+  };
+} cryptodev_cache_ring_t;
 
 typedef struct
 {
@@ -194,13 +219,9 @@ typedef struct
     };
   };
 
-  cryptodev_async_frame_sw_ring frame_ring;
+  cryptodev_cache_ring_t cache_ring;
   u16 cryptodev_id;
   u16 cryptodev_q;
-  u16 frames_on_ring;
-  u16 enqueued_not_dequeueq;
-  u16 deqeued_not_returned;
-  u16 pending_to_qat;
   u16 inflight;
 } cryptodev_engine_thread_t;
 
@@ -224,16 +245,107 @@ typedef struct
 
 extern cryptodev_main_t cryptodev_main;
 
-static_always_inline void
-cryptodev_mark_frame_err_status (vnet_crypto_async_frame_t *f,
-				 vnet_crypto_op_status_t s,
-				 vnet_crypto_async_frame_state_t fs)
-{
-  u32 n_elts = f->n_elts, i;
+#define CRYPTODEV_CACHE_RING_GET_FRAME(r, i)                                  \
+  ((r)->frames[(i) &CRYPTODEV_CACHE_QUEUE_MASK].f)
 
-  for (i = 0; i < n_elts; i++)
-    f->elts[i].status = s;
-  f->state = fs;
+#define CRYPTODEV_CACHE_RING_GET_ERR_MASK(r, i)                               \
+  ((r)->frames[(i) &CRYPTODEV_CACHE_QUEUE_MASK].frame_elts_errs_mask)
+
+#define CRYPTODEV_CACHE_RING_GET_FRAME_ELTS_INFLIGHT(r, i)                    \
+  (((r)->frames[(i) &CRYPTODEV_CACHE_QUEUE_MASK].enq_elts_head) -             \
+   ((r)->frames[(i) &CRYPTODEV_CACHE_QUEUE_MASK].deq_elts_tail))
+
+static_always_inline void
+cryptodev_cache_ring_update_enq_head (cryptodev_cache_ring_t *r,
+				      vnet_crypto_async_frame_t *f)
+{
+  if (r->frames[r->enq_head].enq_elts_head == f->n_elts)
+    {
+      r->enq_head++;
+      r->enq_head &= CRYPTODEV_CACHE_QUEUE_MASK;
+      f->state = VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED;
+    }
+}
+
+static_always_inline bool
+cryptodev_cache_ring_update_deq_tail (cryptodev_cache_ring_t *r,
+				      u16 *const deq)
+{
+  if (r->frames[*deq].deq_elts_tail == r->frames[*deq].n_elts)
+    {
+      *deq += 1;
+      *deq &= CRYPTODEV_CACHE_QUEUE_MASK;
+      return 1;
+    }
+
+  return 0;
+}
+static_always_inline u64
+cryptodev_mark_frame_fill_err (vnet_crypto_async_frame_t *f, u64 current_err,
+			       u16 index, u16 n, vnet_crypto_op_status_t op_s)
+{
+  u64 err = current_err;
+  u16 i;
+
+  ERROR_ASSERT (index + n <= VNET_CRYPTO_FRAME_SIZE);
+  ERROR_ASSERT (op_s != VNET_CRYPTO_OP_STATUS_COMPLETED);
+
+  for (i = index; i < (index + n); i++)
+    f->elts[i].status = op_s;
+
+  err |= (~(~(0u) << n) << index);
+
+  return err;
+}
+
+static_always_inline cryptodev_cache_ring_elt_t *
+cryptodev_cache_ring_push (cryptodev_cache_ring_t *r,
+			   vnet_crypto_async_frame_t *f)
+{
+  u16 head = r->head;
+  cryptodev_cache_ring_elt_t *ring_elt = &r->frames[head];
+  /**
+   * in debug mode we do the ring sanity test when a frame is enqueued to
+   * the ring.
+   **/
+#if CLIB_DEBUG > 0
+  u16 tail = r->tail;
+  u16 n_cached = (head >= tail) ? (head - tail) :
+					(CRYPTODEV_CACHE_QUEUE_MASK - tail + head);
+  ERROR_ASSERT (n_cached < VNET_CRYPTO_FRAME_POOL_SIZE);
+  ERROR_ASSERT (r->raw == 0 && r->frames[head].raw == 0 &&
+		r->frames[head].f == 0);
+#endif
+  ring_elt->f = f;
+  ring_elt->n_elts = f->n_elts;
+  /* update head */
+  r->head++;
+  r->head &= CRYPTODEV_CACHE_QUEUE_MASK;
+  return ring_elt;
+}
+
+static_always_inline vnet_crypto_async_frame_t *
+cryptodev_cache_ring_pop (cryptodev_cache_ring_t *r)
+{
+  vnet_crypto_async_frame_t *f;
+  u16 tail = r->tail;
+  cryptodev_cache_ring_elt_t *ring_elt = &r->frames[tail];
+
+  ERROR_ASSERT (r->frames[r->head].raw == 0 ? r->head != tail : 1);
+  ERROR_ASSERT (r->frames[tail].raw != 0);
+  ERROR_ASSERT (ring_elt->deq_elts_tail == ring_elt->enq_elts_head &&
+		ring_elt->deq_elts_tail == ring_elt->n_elts);
+
+  f = CRYPTODEV_CACHE_RING_GET_FRAME (r, tail);
+  f->state = CRYPTODEV_CACHE_RING_GET_ERR_MASK (r, r->tail) == 0 ?
+		     VNET_CRYPTO_FRAME_STATE_SUCCESS :
+		     VNET_CRYPTO_FRAME_STATE_ELT_ERROR;
+
+  clib_memset (ring_elt, 0, sizeof (*ring_elt));
+  r->tail++;
+  r->tail &= CRYPTODEV_CACHE_QUEUE_MASK;
+
+  return f;
 }
 
 int cryptodev_session_create (vlib_main_t *vm, vnet_crypto_key_index_t idx,