Christoph Hellwig | 14d3a3b | 2015-12-11 11:53:03 -0800 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2015 HGST, a Western Digital Company. |
| 3 | * |
| 4 | * This program is free software; you can redistribute it and/or modify it |
| 5 | * under the terms and conditions of the GNU General Public License, |
| 6 | * version 2, as published by the Free Software Foundation. |
| 7 | * |
| 8 | * This program is distributed in the hope it will be useful, but WITHOUT |
| 9 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 10 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
| 11 | * more details. |
| 12 | */ |
| 13 | #include <linux/module.h> |
| 14 | #include <linux/err.h> |
| 15 | #include <linux/slab.h> |
| 16 | #include <rdma/ib_verbs.h> |
| 17 | |
| 18 | /* # of WCs to poll for with a single call to ib_poll_cq */ |
| 19 | #define IB_POLL_BATCH 16 |
| 20 | |
| 21 | /* # of WCs to iterate over before yielding */ |
| 22 | #define IB_POLL_BUDGET_IRQ 256 |
| 23 | #define IB_POLL_BUDGET_WORKQUEUE 65536 |
| 24 | |
| 25 | #define IB_POLL_FLAGS \ |
| 26 | (IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS) |
| 27 | |
| 28 | static int __ib_process_cq(struct ib_cq *cq, int budget) |
| 29 | { |
| 30 | int i, n, completed = 0; |
| 31 | |
| 32 | while ((n = ib_poll_cq(cq, IB_POLL_BATCH, cq->wc)) > 0) { |
| 33 | for (i = 0; i < n; i++) { |
| 34 | struct ib_wc *wc = &cq->wc[i]; |
| 35 | |
| 36 | if (wc->wr_cqe) |
| 37 | wc->wr_cqe->done(cq, wc); |
| 38 | else |
| 39 | WARN_ON_ONCE(wc->status == IB_WC_SUCCESS); |
| 40 | } |
| 41 | |
| 42 | completed += n; |
| 43 | |
| 44 | if (n != IB_POLL_BATCH || |
| 45 | (budget != -1 && completed >= budget)) |
| 46 | break; |
| 47 | } |
| 48 | |
| 49 | return completed; |
| 50 | } |
| 51 | |
| 52 | /** |
| 53 | * ib_process_direct_cq - process a CQ in caller context |
| 54 | * @cq: CQ to process |
| 55 | * @budget: number of CQEs to poll for |
| 56 | * |
| 57 | * This function is used to process all outstanding CQ entries on a |
| 58 | * %IB_POLL_DIRECT CQ. It does not offload CQ processing to a different |
| 59 | * context and does not ask for completion interrupts from the HCA. |
| 60 | * |
| 61 | * Note: for compatibility reasons -1 can be passed in %budget for unlimited |
| 62 | * polling. Do not use this feature in new code, it will be removed soon. |
| 63 | */ |
| 64 | int ib_process_cq_direct(struct ib_cq *cq, int budget) |
| 65 | { |
| 66 | WARN_ON_ONCE(cq->poll_ctx != IB_POLL_DIRECT); |
| 67 | |
| 68 | return __ib_process_cq(cq, budget); |
| 69 | } |
| 70 | EXPORT_SYMBOL(ib_process_cq_direct); |
| 71 | |
| 72 | static void ib_cq_completion_direct(struct ib_cq *cq, void *private) |
| 73 | { |
| 74 | WARN_ONCE(1, "got unsolicited completion for CQ 0x%p\n", cq); |
| 75 | } |
| 76 | |
| 77 | static int ib_poll_handler(struct irq_poll *iop, int budget) |
| 78 | { |
| 79 | struct ib_cq *cq = container_of(iop, struct ib_cq, iop); |
| 80 | int completed; |
| 81 | |
| 82 | completed = __ib_process_cq(cq, budget); |
| 83 | if (completed < budget) { |
| 84 | irq_poll_complete(&cq->iop); |
| 85 | if (ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0) |
| 86 | irq_poll_sched(&cq->iop); |
| 87 | } |
| 88 | |
| 89 | return completed; |
| 90 | } |
| 91 | |
| 92 | static void ib_cq_completion_softirq(struct ib_cq *cq, void *private) |
| 93 | { |
| 94 | irq_poll_sched(&cq->iop); |
| 95 | } |
| 96 | |
| 97 | static void ib_cq_poll_work(struct work_struct *work) |
| 98 | { |
| 99 | struct ib_cq *cq = container_of(work, struct ib_cq, work); |
| 100 | int completed; |
| 101 | |
| 102 | completed = __ib_process_cq(cq, IB_POLL_BUDGET_WORKQUEUE); |
| 103 | if (completed >= IB_POLL_BUDGET_WORKQUEUE || |
| 104 | ib_req_notify_cq(cq, IB_POLL_FLAGS) > 0) |
| 105 | queue_work(ib_comp_wq, &cq->work); |
| 106 | } |
| 107 | |
| 108 | static void ib_cq_completion_workqueue(struct ib_cq *cq, void *private) |
| 109 | { |
| 110 | queue_work(ib_comp_wq, &cq->work); |
| 111 | } |
| 112 | |
| 113 | /** |
| 114 | * ib_alloc_cq - allocate a completion queue |
| 115 | * @dev: device to allocate the CQ for |
| 116 | * @private: driver private data, accessible from cq->cq_context |
| 117 | * @nr_cqe: number of CQEs to allocate |
| 118 | * @comp_vector: HCA completion vectors for this CQ |
| 119 | * @poll_ctx: context to poll the CQ from. |
| 120 | * |
| 121 | * This is the proper interface to allocate a CQ for in-kernel users. A |
| 122 | * CQ allocated with this interface will automatically be polled from the |
| 123 | * specified context. The ULP needs must use wr->wr_cqe instead of wr->wr_id |
| 124 | * to use this CQ abstraction. |
| 125 | */ |
| 126 | struct ib_cq *ib_alloc_cq(struct ib_device *dev, void *private, |
| 127 | int nr_cqe, int comp_vector, enum ib_poll_context poll_ctx) |
| 128 | { |
| 129 | struct ib_cq_init_attr cq_attr = { |
| 130 | .cqe = nr_cqe, |
| 131 | .comp_vector = comp_vector, |
| 132 | }; |
| 133 | struct ib_cq *cq; |
| 134 | int ret = -ENOMEM; |
| 135 | |
| 136 | cq = dev->create_cq(dev, &cq_attr, NULL, NULL); |
| 137 | if (IS_ERR(cq)) |
| 138 | return cq; |
| 139 | |
| 140 | cq->device = dev; |
| 141 | cq->uobject = NULL; |
| 142 | cq->event_handler = NULL; |
| 143 | cq->cq_context = private; |
| 144 | cq->poll_ctx = poll_ctx; |
| 145 | atomic_set(&cq->usecnt, 0); |
| 146 | |
| 147 | cq->wc = kmalloc_array(IB_POLL_BATCH, sizeof(*cq->wc), GFP_KERNEL); |
| 148 | if (!cq->wc) |
| 149 | goto out_destroy_cq; |
| 150 | |
| 151 | switch (cq->poll_ctx) { |
| 152 | case IB_POLL_DIRECT: |
| 153 | cq->comp_handler = ib_cq_completion_direct; |
| 154 | break; |
| 155 | case IB_POLL_SOFTIRQ: |
| 156 | cq->comp_handler = ib_cq_completion_softirq; |
| 157 | |
| 158 | irq_poll_init(&cq->iop, IB_POLL_BUDGET_IRQ, ib_poll_handler); |
| 159 | ib_req_notify_cq(cq, IB_CQ_NEXT_COMP); |
| 160 | break; |
| 161 | case IB_POLL_WORKQUEUE: |
| 162 | cq->comp_handler = ib_cq_completion_workqueue; |
| 163 | INIT_WORK(&cq->work, ib_cq_poll_work); |
| 164 | ib_req_notify_cq(cq, IB_CQ_NEXT_COMP); |
| 165 | break; |
| 166 | default: |
| 167 | ret = -EINVAL; |
| 168 | goto out_free_wc; |
| 169 | } |
| 170 | |
| 171 | return cq; |
| 172 | |
| 173 | out_free_wc: |
| 174 | kfree(cq->wc); |
| 175 | out_destroy_cq: |
| 176 | cq->device->destroy_cq(cq); |
| 177 | return ERR_PTR(ret); |
| 178 | } |
| 179 | EXPORT_SYMBOL(ib_alloc_cq); |
| 180 | |
| 181 | /** |
| 182 | * ib_free_cq - free a completion queue |
| 183 | * @cq: completion queue to free. |
| 184 | */ |
| 185 | void ib_free_cq(struct ib_cq *cq) |
| 186 | { |
| 187 | int ret; |
| 188 | |
| 189 | if (WARN_ON_ONCE(atomic_read(&cq->usecnt))) |
| 190 | return; |
| 191 | |
| 192 | switch (cq->poll_ctx) { |
| 193 | case IB_POLL_DIRECT: |
| 194 | break; |
| 195 | case IB_POLL_SOFTIRQ: |
| 196 | irq_poll_disable(&cq->iop); |
| 197 | break; |
| 198 | case IB_POLL_WORKQUEUE: |
| 199 | flush_work(&cq->work); |
| 200 | break; |
| 201 | default: |
| 202 | WARN_ON_ONCE(1); |
| 203 | } |
| 204 | |
| 205 | kfree(cq->wc); |
| 206 | ret = cq->device->destroy_cq(cq); |
| 207 | WARN_ON_ONCE(ret); |
| 208 | } |
| 209 | EXPORT_SYMBOL(ib_free_cq); |