| /* |
| This file is provided under a dual BSD/GPLv2 license. When using or |
| redistributing this file, you may do so under either license. |
| |
| GPL LICENSE SUMMARY |
| Copyright(c) 2014 Intel Corporation. |
| This program is free software; you can redistribute it and/or modify |
| it under the terms of version 2 of the GNU General Public License as |
| published by the Free Software Foundation. |
| |
| This program is distributed in the hope that it will be useful, but |
| WITHOUT ANY WARRANTY; without even the implied warranty of |
| MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| General Public License for more details. |
| |
| Contact Information: |
| qat-linux@intel.com |
| |
| BSD LICENSE |
| Copyright(c) 2014 Intel Corporation. |
| Redistribution and use in source and binary forms, with or without |
| modification, are permitted provided that the following conditions |
| are met: |
| |
| * Redistributions of source code must retain the above copyright |
| notice, this list of conditions and the following disclaimer. |
| * Redistributions in binary form must reproduce the above copyright |
| notice, this list of conditions and the following disclaimer in |
| the documentation and/or other materials provided with the |
| distribution. |
| * Neither the name of Intel Corporation nor the names of its |
| contributors may be used to endorse or promote products derived |
| from this software without specific prior written permission. |
| |
| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| */ |
| #include <linux/kernel.h> |
| #include <linux/init.h> |
| #include <linux/types.h> |
| #include <linux/pci.h> |
| #include <linux/slab.h> |
| #include <linux/errno.h> |
| #include <linux/interrupt.h> |
| #include "adf_accel_devices.h" |
| #include "adf_common_drv.h" |
| #include "adf_cfg.h" |
| #include "adf_cfg_strings.h" |
| #include "adf_cfg_common.h" |
| #include "adf_transport_access_macros.h" |
| #include "adf_transport_internal.h" |
| |
| static int adf_enable_msix(struct adf_accel_dev *accel_dev) |
| { |
| struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; |
| struct adf_hw_device_data *hw_data = accel_dev->hw_device; |
| u32 msix_num_entries = 1; |
| |
| /* If SR-IOV is disabled, add entries for each bank */ |
| if (!accel_dev->pf.vf_info) { |
| int i; |
| |
| msix_num_entries += hw_data->num_banks; |
| for (i = 0; i < msix_num_entries; i++) |
| pci_dev_info->msix_entries.entries[i].entry = i; |
| } else { |
| pci_dev_info->msix_entries.entries[0].entry = |
| hw_data->num_banks; |
| } |
| |
| if (pci_enable_msix_exact(pci_dev_info->pci_dev, |
| pci_dev_info->msix_entries.entries, |
| msix_num_entries)) { |
| dev_err(&GET_DEV(accel_dev), "Failed to enable MSI-X IRQ(s)\n"); |
| return -EFAULT; |
| } |
| return 0; |
| } |
| |
| static void adf_disable_msix(struct adf_accel_pci *pci_dev_info) |
| { |
| pci_disable_msix(pci_dev_info->pci_dev); |
| } |
| |
| static irqreturn_t adf_msix_isr_bundle(int irq, void *bank_ptr) |
| { |
| struct adf_etr_bank_data *bank = bank_ptr; |
| |
| WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr, bank->bank_number, 0); |
| tasklet_hi_schedule(&bank->resp_handler); |
| return IRQ_HANDLED; |
| } |
| |
| static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr) |
| { |
| struct adf_accel_dev *accel_dev = dev_ptr; |
| |
| #ifdef CONFIG_PCI_IOV |
| /* If SR-IOV is enabled (vf_info is non-NULL), check for VF->PF ints */ |
| if (accel_dev->pf.vf_info) { |
| struct adf_hw_device_data *hw_data = accel_dev->hw_device; |
| struct adf_bar *pmisc = |
| &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; |
| void __iomem *pmisc_bar_addr = pmisc->virt_addr; |
| u32 vf_mask; |
| |
| /* Get the interrupt sources triggered by VFs */ |
| vf_mask = ((ADF_CSR_RD(pmisc_bar_addr, ADF_ERRSOU5) & |
| 0x0000FFFF) << 16) | |
| ((ADF_CSR_RD(pmisc_bar_addr, ADF_ERRSOU3) & |
| 0x01FFFE00) >> 9); |
| |
| if (vf_mask) { |
| struct adf_accel_vf_info *vf_info; |
| bool irq_handled = false; |
| int i; |
| |
| /* Disable VF2PF interrupts for VFs with pending ints */ |
| adf_disable_vf2pf_interrupts(accel_dev, vf_mask); |
| |
| /* |
| * Schedule tasklets to handle VF2PF interrupt BHs |
| * unless the VF is malicious and is attempting to |
| * flood the host OS with VF2PF interrupts. |
| */ |
| for_each_set_bit(i, (const unsigned long *)&vf_mask, |
| (sizeof(vf_mask) * BITS_PER_BYTE)) { |
| vf_info = accel_dev->pf.vf_info + i; |
| |
| if (!__ratelimit(&vf_info->vf2pf_ratelimit)) { |
| dev_info(&GET_DEV(accel_dev), |
| "Too many ints from VF%d\n", |
| vf_info->vf_nr + 1); |
| continue; |
| } |
| |
| /* Tasklet will re-enable ints from this VF */ |
| tasklet_hi_schedule(&vf_info->vf2pf_bh_tasklet); |
| irq_handled = true; |
| } |
| |
| if (irq_handled) |
| return IRQ_HANDLED; |
| } |
| } |
| #endif /* CONFIG_PCI_IOV */ |
| |
| dev_dbg(&GET_DEV(accel_dev), "qat_dev%d spurious AE interrupt\n", |
| accel_dev->accel_id); |
| |
| return IRQ_NONE; |
| } |
| |
| static int adf_request_irqs(struct adf_accel_dev *accel_dev) |
| { |
| struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; |
| struct adf_hw_device_data *hw_data = accel_dev->hw_device; |
| struct msix_entry *msixe = pci_dev_info->msix_entries.entries; |
| struct adf_etr_data *etr_data = accel_dev->transport; |
| int ret, i = 0; |
| char *name; |
| |
| /* Request msix irq for all banks unless SR-IOV enabled */ |
| if (!accel_dev->pf.vf_info) { |
| for (i = 0; i < hw_data->num_banks; i++) { |
| struct adf_etr_bank_data *bank = &etr_data->banks[i]; |
| unsigned int cpu, cpus = num_online_cpus(); |
| |
| name = *(pci_dev_info->msix_entries.names + i); |
| snprintf(name, ADF_MAX_MSIX_VECTOR_NAME, |
| "qat%d-bundle%d", accel_dev->accel_id, i); |
| ret = request_irq(msixe[i].vector, |
| adf_msix_isr_bundle, 0, name, bank); |
| if (ret) { |
| dev_err(&GET_DEV(accel_dev), |
| "failed to enable irq %d for %s\n", |
| msixe[i].vector, name); |
| return ret; |
| } |
| |
| cpu = ((accel_dev->accel_id * hw_data->num_banks) + |
| i) % cpus; |
| irq_set_affinity_hint(msixe[i].vector, |
| get_cpu_mask(cpu)); |
| } |
| } |
| |
| /* Request msix irq for AE */ |
| name = *(pci_dev_info->msix_entries.names + i); |
| snprintf(name, ADF_MAX_MSIX_VECTOR_NAME, |
| "qat%d-ae-cluster", accel_dev->accel_id); |
| ret = request_irq(msixe[i].vector, adf_msix_isr_ae, 0, name, accel_dev); |
| if (ret) { |
| dev_err(&GET_DEV(accel_dev), |
| "failed to enable irq %d, for %s\n", |
| msixe[i].vector, name); |
| return ret; |
| } |
| return ret; |
| } |
| |
| static void adf_free_irqs(struct adf_accel_dev *accel_dev) |
| { |
| struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; |
| struct adf_hw_device_data *hw_data = accel_dev->hw_device; |
| struct msix_entry *msixe = pci_dev_info->msix_entries.entries; |
| struct adf_etr_data *etr_data = accel_dev->transport; |
| int i = 0; |
| |
| if (pci_dev_info->msix_entries.num_entries > 1) { |
| for (i = 0; i < hw_data->num_banks; i++) { |
| irq_set_affinity_hint(msixe[i].vector, NULL); |
| free_irq(msixe[i].vector, &etr_data->banks[i]); |
| } |
| } |
| irq_set_affinity_hint(msixe[i].vector, NULL); |
| free_irq(msixe[i].vector, accel_dev); |
| } |
| |
| static int adf_isr_alloc_msix_entry_table(struct adf_accel_dev *accel_dev) |
| { |
| int i; |
| char **names; |
| struct msix_entry *entries; |
| struct adf_hw_device_data *hw_data = accel_dev->hw_device; |
| u32 msix_num_entries = 1; |
| |
| /* If SR-IOV is disabled (vf_info is NULL), add entries for each bank */ |
| if (!accel_dev->pf.vf_info) |
| msix_num_entries += hw_data->num_banks; |
| |
| entries = kzalloc_node(msix_num_entries * sizeof(*entries), |
| GFP_KERNEL, dev_to_node(&GET_DEV(accel_dev))); |
| if (!entries) |
| return -ENOMEM; |
| |
| names = kcalloc(msix_num_entries, sizeof(char *), GFP_KERNEL); |
| if (!names) { |
| kfree(entries); |
| return -ENOMEM; |
| } |
| for (i = 0; i < msix_num_entries; i++) { |
| *(names + i) = kzalloc(ADF_MAX_MSIX_VECTOR_NAME, GFP_KERNEL); |
| if (!(*(names + i))) |
| goto err; |
| } |
| accel_dev->accel_pci_dev.msix_entries.num_entries = msix_num_entries; |
| accel_dev->accel_pci_dev.msix_entries.entries = entries; |
| accel_dev->accel_pci_dev.msix_entries.names = names; |
| return 0; |
| err: |
| for (i = 0; i < msix_num_entries; i++) |
| kfree(*(names + i)); |
| kfree(entries); |
| kfree(names); |
| return -ENOMEM; |
| } |
| |
| static void adf_isr_free_msix_entry_table(struct adf_accel_dev *accel_dev) |
| { |
| char **names = accel_dev->accel_pci_dev.msix_entries.names; |
| int i; |
| |
| kfree(accel_dev->accel_pci_dev.msix_entries.entries); |
| for (i = 0; i < accel_dev->accel_pci_dev.msix_entries.num_entries; i++) |
| kfree(*(names + i)); |
| kfree(names); |
| } |
| |
| static int adf_setup_bh(struct adf_accel_dev *accel_dev) |
| { |
| struct adf_etr_data *priv_data = accel_dev->transport; |
| struct adf_hw_device_data *hw_data = accel_dev->hw_device; |
| int i; |
| |
| for (i = 0; i < hw_data->num_banks; i++) |
| tasklet_init(&priv_data->banks[i].resp_handler, |
| adf_response_handler, |
| (unsigned long)&priv_data->banks[i]); |
| return 0; |
| } |
| |
| static void adf_cleanup_bh(struct adf_accel_dev *accel_dev) |
| { |
| struct adf_etr_data *priv_data = accel_dev->transport; |
| struct adf_hw_device_data *hw_data = accel_dev->hw_device; |
| int i; |
| |
| for (i = 0; i < hw_data->num_banks; i++) { |
| tasklet_disable(&priv_data->banks[i].resp_handler); |
| tasklet_kill(&priv_data->banks[i].resp_handler); |
| } |
| } |
| |
| /** |
| * adf_isr_resource_free() - Free IRQ for acceleration device |
| * @accel_dev: Pointer to acceleration device. |
| * |
| * Function frees interrupts for acceleration device. |
| */ |
| void adf_isr_resource_free(struct adf_accel_dev *accel_dev) |
| { |
| adf_free_irqs(accel_dev); |
| adf_cleanup_bh(accel_dev); |
| adf_disable_msix(&accel_dev->accel_pci_dev); |
| adf_isr_free_msix_entry_table(accel_dev); |
| } |
| EXPORT_SYMBOL_GPL(adf_isr_resource_free); |
| |
| /** |
| * adf_isr_resource_alloc() - Allocate IRQ for acceleration device |
| * @accel_dev: Pointer to acceleration device. |
| * |
| * Function allocates interrupts for acceleration device. |
| * |
| * Return: 0 on success, error code otherwise. |
| */ |
| int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev) |
| { |
| int ret; |
| |
| ret = adf_isr_alloc_msix_entry_table(accel_dev); |
| if (ret) |
| return ret; |
| if (adf_enable_msix(accel_dev)) |
| goto err_out; |
| |
| if (adf_setup_bh(accel_dev)) |
| goto err_out; |
| |
| if (adf_request_irqs(accel_dev)) |
| goto err_out; |
| |
| return 0; |
| err_out: |
| adf_isr_resource_free(accel_dev); |
| return -EFAULT; |
| } |
| EXPORT_SYMBOL_GPL(adf_isr_resource_alloc); |