staging:gasket: gasket-apex updates Update gasket framework version. Add unique chip id sysfs node. Add dma_bit_mask module parameter. Enable DFS and thermal shutdown by default. Add DFS support. Add thermal sysfs nodes. Fix pci remove/rescan for kernel module build. Various clean ups and fixes. Change-Id: Ie7f0bc1c33397468f83c6bd606bf433d3dd6b01c Signed-off-by: Leonid Lobachev <leonidl@google.com>
diff --git a/drivers/staging/gasket/apex.h b/drivers/staging/gasket/apex.h index 3bbceff..5537d96 100644 --- a/drivers/staging/gasket/apex.h +++ b/drivers/staging/gasket/apex.h
@@ -20,6 +20,19 @@ struct apex_gate_clock_ioctl { u64 force_idle; }; +/* Performance expectation ioctl. */ +enum apex_performance_expectation { + APEX_PERFORMANCE_LOW = 0, + APEX_PERFORMANCE_MED = 1, + APEX_PERFORMANCE_HIGH = 2, + APEX_PERFORMANCE_MAX = 3, +}; + +struct apex_performance_expectation_ioctl { + /* Expected performance from apex. */ + uint32_t performance; +}; + /* Base number for all Apex-common IOCTLs */ #define APEX_IOCTL_BASE 0x7F @@ -27,4 +40,6 @@ struct apex_gate_clock_ioctl { #define APEX_IOCTL_GATE_CLOCK \ _IOW(APEX_IOCTL_BASE, 0, struct apex_gate_clock_ioctl) +#define APEX_IOCTL_PERFORMANCE_EXPECTATION _IOW(APEX_IOCTL_BASE, 1, struct apex_performance_expectation_ioctl) + #endif /* __APEX_H__ */
diff --git a/drivers/staging/gasket/apex_driver.c b/drivers/staging/gasket/apex_driver.c index 0cef1d6..5cc871e 100644 --- a/drivers/staging/gasket/apex_driver.c +++ b/drivers/staging/gasket/apex_driver.c
@@ -5,6 +5,7 @@ * Copyright (C) 2018 Google, Inc. */ +#include <linux/atomic.h> #include <linux/compiler.h> #include <linux/delay.h> #include <linux/device.h> @@ -17,6 +18,7 @@ #include <linux/printk.h> #include <linux/sched.h> #include <linux/uaccess.h> +#include <linux/workqueue.h> #include "apex.h" @@ -27,7 +29,7 @@ /* Constants */ #define APEX_DEVICE_NAME "Apex" -#define APEX_DRIVER_VERSION "1.0" +#define APEX_DRIVER_VERSION "1.1" /* CSRs are in BAR 2. */ #define APEX_BAR_INDEX 2 @@ -62,11 +64,32 @@ /* Wait 100 ms between checks. Total 12 sec wait maximum. */ #define APEX_RESET_DELAY 100 +/* Interval between temperature polls, 0 disables polling */ +#define DEFAULT_APEX_TEMP_POLL_INTERVAL 5000 + +/* apex device private data */ +struct apex_dev { + struct gasket_dev *gasket_dev_ptr; + struct delayed_work check_temperature_work; + u32 adc_trip_points[3]; + atomic_t temp_poll_interval; +}; + /* Enumeration of the supported sysfs entries. */ enum sysfs_attribute_type { ATTR_KERNEL_HIB_PAGE_TABLE_SIZE, ATTR_KERNEL_HIB_SIMPLE_PAGE_TABLE_SIZE, ATTR_KERNEL_HIB_NUM_ACTIVE_PAGES, + ATTR_TEMP, + ATTR_TEMP_WARN1, + ATTR_TEMP_WARN1_EN, + ATTR_TEMP_WARN2, + ATTR_TEMP_WARN2_EN, + ATTR_TEMP_TRIP0, + ATTR_TEMP_TRIP1, + ATTR_TEMP_TRIP2, + ATTR_TEMP_POLL_INTERVAL, + ATTR_UNIQUE_ID, }; /* @@ -98,6 +121,14 @@ enum apex_bar2_regs { APEX_BAR2_REG_USER_HIB_DMA_PAUSED = 0x486E0, APEX_BAR2_REG_IDLEGENERATOR_IDLEGEN_IDLEREGISTER = 0x4A000, APEX_BAR2_REG_KERNEL_HIB_PAGE_TABLE = 0x50000, + APEX_BAR2_REG_OMC0_D0 = 0x01a0d0, + APEX_BAR2_REG_OMC0_D4 = 0x01a0d4, + APEX_BAR2_REG_OMC0_D8 = 0x01a0d8, + APEX_BAR2_REG_OMC0_DC = 0x01a0dc, + APEX_BAR2_REG_EFUSE_DC = 0x01a2dc, + APEX_BAR2_REG_EFUSE_E0 = 0x01a2e0, + APEX_BAR2_REG_EFUSE_E4 = 0x01a2e4, + APEX_BAR2_REG_EFUSE_E8 = 0x01a2e8, /* Error registers - Used mostly for debug */ APEX_BAR2_REG_USER_HIB_ERROR_STATUS = 0x86f0, @@ -138,9 +169,6 @@ static const struct gasket_mappable_region mappable_regions[NUM_REGIONS] = { { 0x48000, 0x1000 }, }; -static const struct gasket_mappable_region cm_mappable_regions[1] = { { 0x0, - APEX_CH_MEM_BYTES } }; - /* Gasket device interrupts enums must be dense (i.e., no empty slots). */ enum apex_interrupt { APEX_INTERRUPT_INSTR_QUEUE = 0, @@ -228,7 +256,6 @@ static struct gasket_interrupt_desc apex_interrupts[] = { }, }; - /* Allows device to enter power save upon driver close(). */ static int allow_power_save = 1; @@ -247,6 +274,38 @@ module_param(allow_sw_clock_gating, int, 0644); module_param(allow_hw_clock_gating, int, 0644); module_param(bypass_top_level, int, 0644); +/* Temperature points in milli C at which DFS is toggled */ +#define DEFAULT_TRIP_POINT0_TEMP 85000 +#define DEFAULT_TRIP_POINT1_TEMP 90000 +#define DEFAULT_TRIP_POINT2_TEMP 95000 + +static int trip_point0_temp = DEFAULT_TRIP_POINT0_TEMP; +static int trip_point1_temp = DEFAULT_TRIP_POINT1_TEMP; +static int trip_point2_temp = DEFAULT_TRIP_POINT2_TEMP; + +module_param(trip_point0_temp, int, 0644); +module_param(trip_point1_temp, int, 0644); +module_param(trip_point2_temp, int, 0644); + +/* Hardware monitored temperature trip points in milli C + Apex chip drives INTR line when reaching hw_temp_warn1 temperature, + and SD_ALARM line when reaching hw_temp_warn2 if corresponding + hw_temp_warn*_en is set to true. + */ +static int hw_temp_warn1 = 100000; +static int hw_temp_warn2 = 100000; +static bool hw_temp_warn1_en = false; +static bool hw_temp_warn2_en = true; + +module_param(hw_temp_warn1, int, 0644); +module_param(hw_temp_warn2, int, 0644); +module_param(hw_temp_warn1_en, bool, 0644); +module_param(hw_temp_warn2_en, bool, 0644); + +/* Temperature poll interval in ms */ +static int temp_poll_interval = DEFAULT_APEX_TEMP_POLL_INTERVAL; +module_param(temp_poll_interval, int, 0644); + /* Check the device status registers and return device status ALIVE or DEAD. */ static int apex_get_status(struct gasket_dev *gasket_dev) { @@ -491,6 +550,62 @@ static long apex_clock_gating(struct gasket_dev *gasket_dev, return 0; } +/* apex_set_performance_expectation: Adjust clock rates for Apex. */ +static long apex_set_performance_expectation( + struct gasket_dev *gasket_dev, + struct apex_performance_expectation_ioctl __user *argp) +{ + struct apex_performance_expectation_ioctl ibuf; + uint32_t rg_gcb_clk_div = 0; + uint32_t rg_axi_clk_fixed = 0; + const int AXI_CLK_FIXED_SHIFT = 2; + const int MCU_CLK_FIXED_SHIFT = 3; + + // 8051 clock is fixed for PCIe, as it's not used at all. + const uint32_t rg_8051_clk_fixed = 1; + + if (bypass_top_level) + return 0; + + if (copy_from_user(&ibuf, argp, sizeof(ibuf))) + return -EFAULT; + + switch (ibuf.performance) { + case APEX_PERFORMANCE_LOW: + rg_gcb_clk_div = 3; + rg_axi_clk_fixed = 0; + break; + + case APEX_PERFORMANCE_MED: + rg_gcb_clk_div = 2; + rg_axi_clk_fixed = 0; + break; + + case APEX_PERFORMANCE_HIGH: + rg_gcb_clk_div = 1; + rg_axi_clk_fixed = 0; + break; + + case APEX_PERFORMANCE_MAX: + rg_gcb_clk_div = 0; + rg_axi_clk_fixed = 0; + break; + + default: + return -EINVAL; + } + + /* + * Set clock rates for GCB, AXI, and 8051: + */ + gasket_read_modify_write_32( + gasket_dev, APEX_BAR_INDEX, APEX_BAR2_REG_SCU_3, + (rg_gcb_clk_div | (rg_axi_clk_fixed << AXI_CLK_FIXED_SHIFT) | (rg_8051_clk_fixed << MCU_CLK_FIXED_SHIFT)), + /*mask_width=*/4, /*mask_shift=*/28); + + return 0; +} + /* Apex-specific ioctl handler. */ static long apex_ioctl(struct file *filp, uint cmd, void __user *argp) { @@ -502,17 +617,32 @@ static long apex_ioctl(struct file *filp, uint cmd, void __user *argp) switch (cmd) { case APEX_IOCTL_GATE_CLOCK: return apex_clock_gating(gasket_dev, argp); + case APEX_IOCTL_PERFORMANCE_EXPECTATION: + return apex_set_performance_expectation(gasket_dev, argp); default: return -ENOTTY; /* unknown command */ } } +/* Linear fit optimized for 25C-100C */ +static int adc_to_millic(int adc) +{ + return (662 - adc) * 250 + 550; +} + +static int millic_to_adc(int millic) +{ + return (550 - millic) / 250 + 662; +} + /* Display driver sysfs entries. */ static ssize_t sysfs_show(struct device *device, struct device_attribute *attr, char *buf) { int ret; + unsigned value, value2, value3, value4; struct gasket_dev *gasket_dev; + struct apex_dev *apex_dev; struct gasket_sysfs_attribute *gasket_attr; enum sysfs_attribute_type type; @@ -522,6 +652,13 @@ static ssize_t sysfs_show(struct device *device, struct device_attribute *attr, return -ENODEV; } + if (!gasket_dev->pci_dev || + !(apex_dev = pci_get_drvdata(gasket_dev->pci_dev))) { + dev_err(device, "Can't find apex_dev data\n"); + gasket_sysfs_put_device_data(device, gasket_dev); + return -ENODEV; + } + gasket_attr = gasket_sysfs_get_attr(device, attr); if (!gasket_attr) { dev_err(device, "No Apex device sysfs attr data found\n"); @@ -529,7 +666,7 @@ static ssize_t sysfs_show(struct device *device, struct device_attribute *attr, return -ENODEV; } - type = (enum sysfs_attribute_type)gasket_sysfs_get_attr(device, attr); + type = (enum sysfs_attribute_type)gasket_attr->data.attr_type; switch (type) { case ATTR_KERNEL_HIB_PAGE_TABLE_SIZE: ret = scnprintf(buf, PAGE_SIZE, "%u\n", @@ -546,6 +683,162 @@ static ssize_t sysfs_show(struct device *device, struct device_attribute *attr, gasket_page_table_num_active_pages( gasket_dev->page_table[0])); break; + case ATTR_TEMP: + value = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_DC); + value = (value >> 16) & ((1 << 10) - 1); + ret = scnprintf(buf, PAGE_SIZE, "%i\n", adc_to_millic(value)); + break; + case ATTR_TEMP_WARN1: + value = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D4); + value = (value >> 16) & ((1 << 10) - 1); + ret = scnprintf(buf, PAGE_SIZE, "%i\n", adc_to_millic(value)); + break; + case ATTR_TEMP_WARN2: + value = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D8); + value = (value >> 16) & ((1 << 10) - 1); + ret = scnprintf(buf, PAGE_SIZE, "%i\n", adc_to_millic(value)); + break; + case ATTR_TEMP_WARN1_EN: + value = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D4); + ret = scnprintf(buf, PAGE_SIZE, "%i\n", value >> 31); + break; + case ATTR_TEMP_WARN2_EN: + value = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D8); + ret = scnprintf(buf, PAGE_SIZE, "%i\n", value >> 31); + break; + case ATTR_TEMP_TRIP0: + ret = scnprintf(buf, PAGE_SIZE, "%i\n", + adc_to_millic(apex_dev->adc_trip_points[0])); + break; + case ATTR_TEMP_TRIP1: + ret = scnprintf(buf, PAGE_SIZE, "%i\n", + adc_to_millic(apex_dev->adc_trip_points[1])); + break; + case ATTR_TEMP_TRIP2: + ret = scnprintf(buf, PAGE_SIZE, "%i\n", + adc_to_millic(apex_dev->adc_trip_points[2])); + break; + case ATTR_TEMP_POLL_INTERVAL: + ret = scnprintf(buf, PAGE_SIZE, "%i\n", + atomic_read(&apex_dev->temp_poll_interval)); + break; + case ATTR_UNIQUE_ID: + value = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_EFUSE_DC); + value2 = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_EFUSE_E0); + value3 = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_EFUSE_E4); + value4 = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_EFUSE_E8); + ret = snprintf(buf, PAGE_SIZE, "%.8x%.8x%.8x%.8x\n", value4, + value3, value2, value); + break; + + default: + dev_dbg(gasket_dev->dev, "Unknown attribute: %s\n", + attr->attr.name); + ret = 0; + break; + } + + gasket_sysfs_put_attr(device, gasket_attr); + gasket_sysfs_put_device_data(device, gasket_dev); + return ret; +} + +/* Set driver sysfs entries. */ +static ssize_t sysfs_store(struct device *device, struct device_attribute *attr, + const char *buf, size_t count) +{ + int ret = count, value; + struct gasket_dev *gasket_dev; + struct apex_dev *apex_dev; + struct gasket_sysfs_attribute *gasket_attr; + enum sysfs_attribute_type type; + + if (kstrtoint(buf, 10, &value)) + return -EINVAL; + + gasket_dev = gasket_sysfs_get_device_data(device); + if (!gasket_dev) { + dev_err(device, "No Apex device sysfs mapping found\n"); + return -ENODEV; + } + + if (!gasket_dev->pci_dev || + !(apex_dev = pci_get_drvdata(gasket_dev->pci_dev))) { + dev_err(device, "Can't find apex_dev data\n"); + gasket_sysfs_put_device_data(device, gasket_dev); + return -ENODEV; + } + + gasket_attr = gasket_sysfs_get_attr(device, attr); + if (!gasket_attr) { + dev_err(device, "No Apex device sysfs attr data found\n"); + gasket_sysfs_put_device_data(device, gasket_dev); + return -ENODEV; + } + + type = (enum sysfs_attribute_type)gasket_attr->data.attr_type; + switch (type) { + case ATTR_TEMP_WARN1: + value = millic_to_adc(value); + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D4, value, 10, + 16); + break; + case ATTR_TEMP_WARN2: + value = millic_to_adc(value); + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D8, value, 10, + 16); + break; + case ATTR_TEMP_WARN1_EN: + value = value > 0 ? 1 : 0; + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D4, value, 1, + 31); + break; + case ATTR_TEMP_WARN2_EN: + value = value > 0 ? 1 : 0; + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D8, value, 1, + 31); + break; + case ATTR_TEMP_TRIP0: + value = millic_to_adc(value); + /* Note: that adc values should be in descending order */ + if (value >= apex_dev->adc_trip_points[1]) { + apex_dev->adc_trip_points[0] = value; + } else ret = -EINVAL; + break; + case ATTR_TEMP_TRIP1: + value = millic_to_adc(value); + if (value <= apex_dev->adc_trip_points[0] && + value >= apex_dev->adc_trip_points[2]) { + apex_dev->adc_trip_points[1] = value; + } else ret = -EINVAL; + break; + case ATTR_TEMP_TRIP2: + value = millic_to_adc(value); + if (value <= apex_dev->adc_trip_points[1]) { + apex_dev->adc_trip_points[2] = value; + } else ret = -EINVAL; + break; + case ATTR_TEMP_POLL_INTERVAL: + cancel_delayed_work_sync(&apex_dev->check_temperature_work); + atomic_set(&apex_dev->temp_poll_interval, value); + if (value > 0) + schedule_delayed_work(&apex_dev->check_temperature_work, + msecs_to_jiffies(value)); + + break; default: dev_dbg(gasket_dev->dev, "Unknown attribute: %s\n", attr->attr.name); @@ -565,9 +858,110 @@ static struct gasket_sysfs_attribute apex_sysfs_attrs[] = { ATTR_KERNEL_HIB_SIMPLE_PAGE_TABLE_SIZE), GASKET_SYSFS_RO(node_0_num_mapped_pages, sysfs_show, ATTR_KERNEL_HIB_NUM_ACTIVE_PAGES), + GASKET_SYSFS_RO(temp, sysfs_show, ATTR_TEMP), + GASKET_SYSFS_RW(hw_temp_warn1, sysfs_show, sysfs_store, + ATTR_TEMP_WARN1), + GASKET_SYSFS_RW(hw_temp_warn1_en, sysfs_show, sysfs_store, + ATTR_TEMP_WARN1_EN), + GASKET_SYSFS_RW(hw_temp_warn2, sysfs_show, sysfs_store, + ATTR_TEMP_WARN2), + GASKET_SYSFS_RW(hw_temp_warn2_en, sysfs_show, sysfs_store, + ATTR_TEMP_WARN2_EN), + GASKET_SYSFS_RW(trip_point0_temp, sysfs_show, sysfs_store, + ATTR_TEMP_TRIP0), + GASKET_SYSFS_RW(trip_point1_temp, sysfs_show, sysfs_store, + ATTR_TEMP_TRIP1), + GASKET_SYSFS_RW(trip_point2_temp, sysfs_show, sysfs_store, + ATTR_TEMP_TRIP2), + GASKET_SYSFS_RW(temp_poll_interval, sysfs_show, sysfs_store, + ATTR_TEMP_POLL_INTERVAL), + GASKET_SYSFS_RO(unique_id, sysfs_show, ATTR_UNIQUE_ID), GASKET_END_OF_ATTR_ARRAY }; +static void apply_module_params(struct apex_dev *apex_dev) { + kernel_param_lock(THIS_MODULE); + + /* use defaults if trip point temperatures are not in ascending order */ + if (trip_point0_temp > trip_point1_temp || + trip_point1_temp > trip_point2_temp) { + dev_warn(apex_dev->gasket_dev_ptr->dev, + "Invalid module parameters for temperature trip points" + ", using defaults\n"); + trip_point0_temp = DEFAULT_TRIP_POINT0_TEMP; + trip_point1_temp = DEFAULT_TRIP_POINT1_TEMP; + trip_point2_temp = DEFAULT_TRIP_POINT2_TEMP; + } + + apex_dev->adc_trip_points[0] = millic_to_adc(trip_point0_temp); + apex_dev->adc_trip_points[1] = millic_to_adc(trip_point1_temp); + apex_dev->adc_trip_points[2] = millic_to_adc(trip_point2_temp); + atomic_set(&apex_dev->temp_poll_interval, temp_poll_interval); + + gasket_read_modify_write_32(apex_dev->gasket_dev_ptr, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D4, + millic_to_adc(hw_temp_warn1), 10, 16); + gasket_read_modify_write_32(apex_dev->gasket_dev_ptr, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D8, + millic_to_adc(hw_temp_warn2), 10, 16); + if (hw_temp_warn1_en) + gasket_read_modify_write_32(apex_dev->gasket_dev_ptr, + APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D4, 1, 1, 31); + + if (hw_temp_warn2_en) + gasket_read_modify_write_32(apex_dev->gasket_dev_ptr, + APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D8, 1, 1, 31); + + kernel_param_unlock(THIS_MODULE); +} + +static void check_temperature_work_handler(struct work_struct *work) { + int i, temp_poll_interval; + u32 adc_temp, clk_div, tmp; + const u32 mask = ((1 << 2) - 1) << 28; + struct apex_dev *apex_dev = + container_of(work, struct apex_dev, + check_temperature_work.work); + struct gasket_dev *gasket_dev = apex_dev->gasket_dev_ptr; + + mutex_lock(&gasket_dev->mutex); + + /* Read current temperature */ + adc_temp = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_DC); + adc_temp = (adc_temp >> 16) & ((1 << 10) - 1); + + /* Find closest trip point + Note: that adc values are in descending order */ + for (i = ARRAY_SIZE(apex_dev->adc_trip_points) - 1; i >= 0; --i) { + if (adc_temp <= apex_dev->adc_trip_points[i]) + break; + } + /* Compute divider value and shift into appropriate bit location */ + clk_div = (i + 1) << 28; + + /* Modify gcb clk divider if it's different from current one */ + tmp = gasket_dev_read_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3); + if (clk_div != (tmp & mask)) { + tmp = (tmp & ~mask) | clk_div; + gasket_dev_write_32(gasket_dev, tmp, APEX_BAR_INDEX, + APEX_BAR2_REG_SCU_3); + dev_warn(gasket_dev->dev, + "Apex performance %sthrottled due to temperature\n", + i == -1 ? "not " : ""); + } + + mutex_unlock(&gasket_dev->mutex); + + temp_poll_interval = atomic_read(&apex_dev->temp_poll_interval); + if (temp_poll_interval > 0) + schedule_delayed_work(&apex_dev->check_temperature_work, + msecs_to_jiffies(temp_poll_interval)); +} + /* On device open, perform a core reinit reset. */ static int apex_device_open_cb(struct gasket_dev *gasket_dev) { @@ -583,17 +977,25 @@ static void apex_pci_fixup_class(struct pci_dev *pdev) pdev->class = (PCI_CLASS_SYSTEM_OTHER << 8) | pdev->class; } DECLARE_PCI_FIXUP_CLASS_HEADER(APEX_PCI_VENDOR_ID, APEX_PCI_DEVICE_ID, - PCI_CLASS_NOT_DEFINED, 8, apex_pci_fixup_class); + PCI_ANY_ID, 8, apex_pci_fixup_class); static int apex_pci_probe(struct pci_dev *pci_dev, const struct pci_device_id *id) { - int ret; + int ret, temp_poll_interval; ulong page_table_ready, msix_table_ready; int retries = 0; struct gasket_dev *gasket_dev; + struct apex_dev *apex_dev; ret = pci_enable_device(pci_dev); +#ifdef MODULE + if (ret) { + apex_pci_fixup_class(pci_dev); + pci_bus_assign_resources(pci_dev->bus); + ret = pci_enable_device(pci_dev); + } +#endif if (ret) { dev_err(&pci_dev->dev, "error enabling PCI device\n"); return ret; @@ -608,7 +1010,18 @@ static int apex_pci_probe(struct pci_dev *pci_dev, return ret; } - pci_set_drvdata(pci_dev, gasket_dev); + apex_dev = kzalloc(sizeof(*apex_dev), GFP_KERNEL); + if (!apex_dev) { + dev_err(&pci_dev->dev, "no memory for device\n"); + ret = -ENOMEM; + goto remove_device; + } + + INIT_DELAYED_WORK(&apex_dev->check_temperature_work, + check_temperature_work_handler); + apex_dev->gasket_dev_ptr = gasket_dev; + apply_module_params(apex_dev); + pci_set_drvdata(pci_dev, apex_dev); apex_reset(gasket_dev); while (retries < APEX_RESET_RETRY) { @@ -633,6 +1046,20 @@ static int apex_pci_probe(struct pci_dev *pci_dev, goto remove_device; } + // Enable thermal sensor clocks + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D0, 0x1, 1, 7); + + // Enable thermal sensor (ENAD ENVR ENBG) + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_D8, 0x7, 3, 0); + + // Enable OMC thermal sensor controller + // This bit should be asserted 100 us after ENAD ENVR ENBG + schedule_timeout(usecs_to_jiffies(100)); + gasket_read_modify_write_32(gasket_dev, APEX_BAR_INDEX, + APEX_BAR2_REG_OMC0_DC, 0x1, 1, 0); + ret = gasket_sysfs_create_entries(gasket_dev->dev_info.device, apex_sysfs_attrs); if (ret) @@ -648,19 +1075,36 @@ static int apex_pci_probe(struct pci_dev *pci_dev, if (allow_power_save) apex_enter_reset(gasket_dev); + /* Enable thermal polling */ + temp_poll_interval = atomic_read(&apex_dev->temp_poll_interval); + if (temp_poll_interval > 0) + schedule_delayed_work(&apex_dev->check_temperature_work, + msecs_to_jiffies(temp_poll_interval)); return 0; remove_device: gasket_pci_remove_device(pci_dev); pci_disable_device(pci_dev); + kfree(apex_dev); return ret; } static void apex_pci_remove(struct pci_dev *pci_dev) { - struct gasket_dev *gasket_dev = pci_get_drvdata(pci_dev); + struct apex_dev *apex_dev = pci_get_drvdata(pci_dev); + struct gasket_dev *gasket_dev; + + if (!apex_dev) { + dev_err(&pci_dev->dev, "NULL apex_dev\n"); + goto remove_device; + } + gasket_dev = apex_dev->gasket_dev_ptr; + + cancel_delayed_work_sync(&apex_dev->check_temperature_work); + kfree(apex_dev); gasket_disable_device(gasket_dev); +remove_device: gasket_pci_remove_device(pci_dev); pci_disable_device(pci_dev); }
diff --git a/drivers/staging/gasket/gasket.h b/drivers/staging/gasket/gasket.h index a0f065c..93e7af1 100644 --- a/drivers/staging/gasket/gasket.h +++ b/drivers/staging/gasket/gasket.h
@@ -38,6 +38,31 @@ struct gasket_page_table_ioctl { }; /* + * Structure for ioctl mapping buffers with flags when using the Gasket + * page_table module. + */ +struct gasket_page_table_ioctl_flags { + struct gasket_page_table_ioctl base; + /* + * Flags indicating status and attribute requests from the host. + * NOTE: STATUS bit does not need to be set in this request. + * Set RESERVED bits to 0 to ensure backwards compatibility. + * + * Bitfields: + * [0] - STATUS: indicates if this entry/slot is free + * 0 = PTE_FREE + * 1 = PTE_INUSE + * [2:1] - DMA_DIRECTION: dma_data_direction requested by host + * 00 = DMA_BIDIRECTIONAL + * 01 = DMA_TO_DEVICE + * 10 = DMA_FROM_DEVICE + * 11 = DMA_NONE + * [31:3] - RESERVED + */ + u32 flags; +}; + +/* * Common structure for ioctls mapping and unmapping buffers when using the * Gasket page_table module. * dma_address: phys addr start of coherent memory, allocated by kernel @@ -119,4 +144,12 @@ struct gasket_coherent_alloc_config_ioctl { #define GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR \ _IOWR(GASKET_IOCTL_BASE, 11, struct gasket_coherent_alloc_config_ioctl) +/* + * Tells the kernel to map size bytes at host_address to device_address in + * page_table_index page table. Passes flags to indicate additional attribute + * requests for the mapped memory. + */ +#define GASKET_IOCTL_MAP_BUFFER_FLAGS \ + _IOW(GASKET_IOCTL_BASE, 12, struct gasket_page_table_ioctl_flags) + #endif /* __GASKET_H__ */
diff --git a/drivers/staging/gasket/gasket_constants.h b/drivers/staging/gasket/gasket_constants.h index 50d87c7..ff113bc 100644 --- a/drivers/staging/gasket/gasket_constants.h +++ b/drivers/staging/gasket/gasket_constants.h
@@ -3,7 +3,7 @@ #ifndef __GASKET_CONSTANTS_H__ #define __GASKET_CONSTANTS_H__ -#define GASKET_FRAMEWORK_VERSION "1.1.2" +#define GASKET_FRAMEWORK_VERSION "1.1.3" /* * The maximum number of simultaneous device types supported by the framework.
diff --git a/drivers/staging/gasket/gasket_core.c b/drivers/staging/gasket/gasket_core.c index d12ab56..16112bb 100644 --- a/drivers/staging/gasket/gasket_core.c +++ b/drivers/staging/gasket/gasket_core.c
@@ -24,6 +24,7 @@ #include <linux/init.h> #include <linux/of.h> #include <linux/pid_namespace.h> +#include <linux/platform_device.h> #include <linux/printk.h> #include <linux/sched.h> @@ -102,6 +103,16 @@ enum gasket_sysfs_attribute_type { ATTR_USER_MEM_RANGES }; +/* On some arm64 systems pcie dma controller can only access lower 4GB of + * addresses. Unfortunately vendor BSP isn't providing any means of determining + * this limitation and there're no errors reported if access to higher addresses + * if being done. This parameter allows to workaround this issue by pretending + * that our device only supports 32 bit addresses. This in turn will cause + * dma driver to use shadow buffers located in low 32 bit address space. + */ +static int dma_bit_mask = 64; +module_param(dma_bit_mask, int, 0644); + /* Perform a standard Gasket callback. */ static inline int check_and_invoke_callback(struct gasket_dev *gasket_dev, @@ -109,8 +120,6 @@ check_and_invoke_callback(struct gasket_dev *gasket_dev, { int ret = 0; - dev_dbg(gasket_dev->dev, "check_and_invoke_callback %p\n", - cb_function); if (cb_function) { mutex_lock(&gasket_dev->mutex); ret = cb_function(gasket_dev); @@ -126,11 +135,8 @@ gasket_check_and_invoke_callback_nolock(struct gasket_dev *gasket_dev, { int ret = 0; - if (cb_function) { - dev_dbg(gasket_dev->dev, - "Invoking device-specific callback.\n"); + if (cb_function) ret = cb_function(gasket_dev); - } return ret; } @@ -189,26 +195,26 @@ static int gasket_find_dev_slot(struct gasket_internal_desc *internal_desc, * Returns 0 if successful, a negative error code otherwise. */ static int gasket_alloc_dev(struct gasket_internal_desc *internal_desc, - struct device *parent, struct gasket_dev **pdev, - const char *kobj_name) + struct device *parent, struct gasket_dev **pdev) { int dev_idx; const struct gasket_driver_desc *driver_desc = internal_desc->driver_desc; struct gasket_dev *gasket_dev; struct gasket_cdev_info *dev_info; + const char *parent_name = dev_name(parent); - pr_debug("Allocating a Gasket device %s.\n", kobj_name); + pr_debug("Allocating a Gasket device, parent %s.\n", parent_name); *pdev = NULL; - dev_idx = gasket_find_dev_slot(internal_desc, kobj_name); + dev_idx = gasket_find_dev_slot(internal_desc, parent_name); if (dev_idx < 0) return dev_idx; gasket_dev = *pdev = kzalloc(sizeof(*gasket_dev), GFP_KERNEL); if (!gasket_dev) { - pr_err("no memory for device %s\n", kobj_name); + pr_err("no memory for device, parent %s\n", parent_name); return -ENOMEM; } internal_desc->devs[dev_idx] = gasket_dev; @@ -217,8 +223,9 @@ static int gasket_alloc_dev(struct gasket_internal_desc *internal_desc, gasket_dev->internal_desc = internal_desc; gasket_dev->dev_idx = dev_idx; - snprintf(gasket_dev->kobj_name, GASKET_NAME_MAX, "%s", kobj_name); + snprintf(gasket_dev->kobj_name, GASKET_NAME_MAX, "%s", parent_name); gasket_dev->dev = get_device(parent); + gasket_dev->dma_dev = get_device(parent); /* gasket_bar_data is uninitialized. */ gasket_dev->num_page_tables = driver_desc->num_page_tables; /* max_page_table_size and *page table are uninit'ed */ @@ -231,10 +238,9 @@ static int gasket_alloc_dev(struct gasket_internal_desc *internal_desc, dev_info->devt = MKDEV(driver_desc->major, driver_desc->minor + gasket_dev->dev_idx); - dev_info->device = device_create(internal_desc->class, parent, - dev_info->devt, gasket_dev, dev_info->name); - - dev_dbg(dev_info->device, "Gasket device allocated.\n"); + dev_info->device = + device_create(internal_desc->class, parent, dev_info->devt, + gasket_dev, dev_info->name); /* cdev has not yet been added; cdev_added is 0 */ dev_info->gasket_dev_ptr = gasket_dev; @@ -252,6 +258,7 @@ static void gasket_free_dev(struct gasket_dev *gasket_dev) internal_desc->devs[gasket_dev->dev_idx] = NULL; mutex_unlock(&internal_desc->mutex); put_device(gasket_dev->dev); + put_device(gasket_dev->dma_dev); kfree(gasket_dev); } @@ -319,8 +326,9 @@ static int gasket_map_pci_bar(struct gasket_dev *gasket_dev, int bar_num) goto fail; } - dma_set_mask(&gasket_dev->pci_dev->dev, DMA_BIT_MASK(64)); - dma_set_coherent_mask(&gasket_dev->pci_dev->dev, DMA_BIT_MASK(64)); + dma_set_mask(&gasket_dev->pci_dev->dev, DMA_BIT_MASK(dma_bit_mask)); + dma_set_coherent_mask(&gasket_dev->pci_dev->dev, + DMA_BIT_MASK(dma_bit_mask)); return 0; @@ -634,6 +642,7 @@ void gasket_disable_device(struct gasket_dev *gasket_dev) gasket_dev->internal_desc->driver_desc; int i; + dev_dbg(gasket_dev->dev, "disabling device\n"); /* Only delete the device if it has been successfully added. */ if (gasket_dev->dev_info.cdev_added) cdev_del(&gasket_dev->dev_info.cdev); @@ -652,13 +661,13 @@ void gasket_disable_device(struct gasket_dev *gasket_dev) EXPORT_SYMBOL(gasket_disable_device); /* - * Registered descriptor lookup. + * Registered driver descriptor lookup for PCI devices. * * Precondition: Called with g_mutex held (to avoid a race on return). * Returns NULL if no matching device was found. */ static struct gasket_internal_desc * -lookup_internal_desc(struct pci_dev *pci_dev) +lookup_pci_internal_desc(struct pci_dev *pci_dev) { int i; @@ -674,6 +683,25 @@ lookup_internal_desc(struct pci_dev *pci_dev) } /* + * Registered driver descriptor lookup for platform devices. + * Caller must hold g_mutex. + */ +static struct gasket_internal_desc * +lookup_platform_internal_desc(struct platform_device *pdev) +{ + int i; + + __must_hold(&g_mutex); + for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) { + if (g_descs[i].driver_desc && + strcmp(g_descs[i].driver_desc->name, pdev->name) == 0) + return &g_descs[i]; + } + + return NULL; +} + +/* * Verifies that the user has permissions to perform the requested mapping and * that the provided descriptor/range is of adequate size to hold the range to * be mapped. @@ -992,13 +1020,14 @@ static int gasket_mmap_coherent(struct gasket_dev *gasket_dev, } vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); - - ret = remap_pfn_range(vma, vma->vm_start, - (gasket_dev->coherent_buffer.phys_base) >> - PAGE_SHIFT, requested_length, vma->vm_page_prot); + vma->vm_pgoff = 0; + ret = dma_mmap_coherent(gasket_dev->dma_dev, vma, + gasket_dev->coherent_buffer.virt_base, + gasket_dev->coherent_buffer.phys_base, + requested_length); if (ret) { - dev_err(gasket_dev->dev, "Error remapping PFN range err=%d.\n", - ret); + dev_err(gasket_dev->dev, + "Error mmapping coherent buffer err=%d.\n", ret); trace_gasket_mmap_exit(ret); return ret; } @@ -1348,6 +1377,7 @@ static const struct file_operations gasket_file_ops = { .open = gasket_open, .release = gasket_release, .unlocked_ioctl = gasket_ioctl, + .compat_ioctl = gasket_ioctl, }; /* Perform final init and marks the device as active. */ @@ -1358,13 +1388,8 @@ int gasket_enable_device(struct gasket_dev *gasket_dev) const struct gasket_driver_desc *driver_desc = gasket_dev->internal_desc->driver_desc; - ret = gasket_interrupt_init(gasket_dev, driver_desc->name, - driver_desc->interrupt_type, - driver_desc->interrupts, - driver_desc->num_interrupts, - driver_desc->interrupt_pack_width, - driver_desc->interrupt_bar_index, - driver_desc->wire_interrupt_offsets); + dev_dbg(gasket_dev->dev, "enabling device\n"); + ret = gasket_interrupt_init(gasket_dev); if (ret) { dev_err(gasket_dev->dev, "Critical failure to allocate interrupts: %d\n", ret); @@ -1420,6 +1445,56 @@ int gasket_enable_device(struct gasket_dev *gasket_dev) } EXPORT_SYMBOL(gasket_enable_device); +static int __gasket_add_device(struct device *parent_dev, + struct gasket_internal_desc *internal_desc, + struct gasket_dev **gasket_devp) +{ + int ret; + struct gasket_dev *gasket_dev; + const struct gasket_driver_desc *driver_desc = + internal_desc->driver_desc; + + ret = gasket_alloc_dev(internal_desc, parent_dev, &gasket_dev); + if (ret) + return ret; + if (IS_ERR(gasket_dev->dev_info.device)) { + dev_err(parent_dev, "Cannot create %s device %s [ret = %ld]\n", + driver_desc->name, gasket_dev->dev_info.name, + PTR_ERR(gasket_dev->dev_info.device)); + ret = -ENODEV; + goto free_gasket_dev; + } + + ret = gasket_sysfs_create_mapping(gasket_dev->dev_info.device, + gasket_dev); + if (ret) + goto remove_device; + + ret = gasket_sysfs_create_entries(gasket_dev->dev_info.device, + gasket_sysfs_generic_attrs); + if (ret) + goto remove_sysfs_mapping; + + *gasket_devp = gasket_dev; + return 0; + +remove_sysfs_mapping: + gasket_sysfs_remove_mapping(gasket_dev->dev_info.device); +remove_device: + device_destroy(internal_desc->class, gasket_dev->dev_info.devt); +free_gasket_dev: + gasket_free_dev(gasket_dev); + return ret; +} + +static void __gasket_remove_device(struct gasket_internal_desc *internal_desc, + struct gasket_dev *gasket_dev) +{ + gasket_sysfs_remove_mapping(gasket_dev->dev_info.device); + device_destroy(internal_desc->class, gasket_dev->dev_info.devt); + gasket_free_dev(gasket_dev); +} + /* * Add PCI gasket device. * @@ -1432,16 +1507,14 @@ int gasket_pci_add_device(struct pci_dev *pci_dev, struct gasket_dev **gasket_devp) { int ret; - const char *kobj_name = dev_name(&pci_dev->dev); struct gasket_internal_desc *internal_desc; struct gasket_dev *gasket_dev; - const struct gasket_driver_desc *driver_desc; struct device *parent; - pr_debug("add PCI device %s\n", kobj_name); + dev_dbg(&pci_dev->dev, "add PCI gasket device\n"); mutex_lock(&g_mutex); - internal_desc = lookup_internal_desc(pci_dev); + internal_desc = lookup_pci_internal_desc(pci_dev); mutex_unlock(&g_mutex); if (!internal_desc) { dev_err(&pci_dev->dev, @@ -1449,29 +1522,15 @@ int gasket_pci_add_device(struct pci_dev *pci_dev, return -ENODEV; } - driver_desc = internal_desc->driver_desc; - parent = &pci_dev->dev; - ret = gasket_alloc_dev(internal_desc, parent, &gasket_dev, kobj_name); + ret = __gasket_add_device(parent, internal_desc, &gasket_dev); if (ret) return ret; - gasket_dev->pci_dev = pci_dev; - if (IS_ERR_OR_NULL(gasket_dev->dev_info.device)) { - pr_err("Cannot create %s device %s [ret = %ld]\n", - driver_desc->name, gasket_dev->dev_info.name, - PTR_ERR(gasket_dev->dev_info.device)); - ret = -ENODEV; - goto fail1; - } + gasket_dev->pci_dev = pci_dev; ret = gasket_setup_pci(pci_dev, gasket_dev); if (ret) - goto fail2; - - ret = gasket_sysfs_create_mapping(gasket_dev->dev_info.device, - gasket_dev); - if (ret) - goto fail3; + goto cleanup_pci; /* * Once we've created the mapping structures successfully, attempt to @@ -1482,24 +1541,15 @@ int gasket_pci_add_device(struct pci_dev *pci_dev, if (ret) { dev_err(gasket_dev->dev, "Cannot create sysfs pci link: %d\n", ret); - goto fail3; + goto cleanup_pci; } - ret = gasket_sysfs_create_entries(gasket_dev->dev_info.device, - gasket_sysfs_generic_attrs); - if (ret) - goto fail4; *gasket_devp = gasket_dev; return 0; -fail4: -fail3: - gasket_sysfs_remove_mapping(gasket_dev->dev_info.device); -fail2: +cleanup_pci: gasket_cleanup_pci(gasket_dev); - device_destroy(internal_desc->class, gasket_dev->dev_info.devt); -fail1: - gasket_free_dev(gasket_dev); + __gasket_remove_device(internal_desc, gasket_dev); return ret; } EXPORT_SYMBOL(gasket_pci_add_device); @@ -1510,18 +1560,15 @@ void gasket_pci_remove_device(struct pci_dev *pci_dev) int i; struct gasket_internal_desc *internal_desc; struct gasket_dev *gasket_dev = NULL; - const struct gasket_driver_desc *driver_desc; /* Find the device desc. */ mutex_lock(&g_mutex); - internal_desc = lookup_internal_desc(pci_dev); + internal_desc = lookup_pci_internal_desc(pci_dev); if (!internal_desc) { mutex_unlock(&g_mutex); return; } mutex_unlock(&g_mutex); - driver_desc = internal_desc->driver_desc; - /* Now find the specific device */ mutex_lock(&internal_desc->mutex); for (i = 0; i < GASKET_DEV_MAX; i++) { @@ -1540,13 +1587,84 @@ void gasket_pci_remove_device(struct pci_dev *pci_dev) internal_desc->driver_desc->name); gasket_cleanup_pci(gasket_dev); - - gasket_sysfs_remove_mapping(gasket_dev->dev_info.device); - device_destroy(internal_desc->class, gasket_dev->dev_info.devt); - gasket_free_dev(gasket_dev); + __gasket_remove_device(internal_desc, gasket_dev); } EXPORT_SYMBOL(gasket_pci_remove_device); +/* Add platform gasket device. Called by Gasket device probe function. */ +int gasket_platform_add_device(struct platform_device *pdev, + struct gasket_dev **gasket_devp) +{ + int ret; + struct gasket_internal_desc *internal_desc; + struct gasket_dev *gasket_dev; + struct device *parent; + + dev_dbg(&pdev->dev, "add platform gasket device\n"); + + mutex_lock(&g_mutex); + internal_desc = lookup_platform_internal_desc(pdev); + mutex_unlock(&g_mutex); + if (!internal_desc) { + dev_err(&pdev->dev, + "%s called for unknown driver type\n", __func__); + return -ENODEV; + } + + parent = &pdev->dev; + ret = __gasket_add_device(parent, internal_desc, &gasket_dev); + if (ret) + return ret; + + gasket_dev->platform_dev = pdev; + *gasket_devp = gasket_dev; + return 0; +} +EXPORT_SYMBOL(gasket_platform_add_device); + +/* Remove a platform gasket device. */ +void gasket_platform_remove_device(struct platform_device *pdev) +{ + int i; + struct gasket_internal_desc *internal_desc; + struct gasket_dev *gasket_dev = NULL; + + /* Find the device desc. */ + mutex_lock(&g_mutex); + internal_desc = lookup_platform_internal_desc(pdev); + mutex_unlock(&g_mutex); + if (!internal_desc) + return; + + /* Now find the specific device */ + mutex_lock(&internal_desc->mutex); + for (i = 0; i < GASKET_DEV_MAX; i++) { + if (internal_desc->devs[i] && + internal_desc->devs[i]->platform_dev == pdev) { + gasket_dev = internal_desc->devs[i]; + break; + } + } + mutex_unlock(&internal_desc->mutex); + + if (!gasket_dev) + return; + + dev_dbg(gasket_dev->dev, "remove %s platform gasket device\n", + internal_desc->driver_desc->name); + + __gasket_remove_device(internal_desc, gasket_dev); +} +EXPORT_SYMBOL(gasket_platform_remove_device); + +void gasket_set_dma_device(struct gasket_dev *gasket_dev, + struct device *dma_dev) +{ + put_device(gasket_dev->dma_dev); + gasket_dev->dma_dev = get_device(dma_dev); +} +EXPORT_SYMBOL(gasket_set_dma_device); + /** * Lookup a name by number in a num_name table. * @num: Number to lookup. @@ -1791,7 +1909,6 @@ static int __init gasket_init(void) { int i; - pr_debug("%s\n", __func__); mutex_lock(&g_mutex); for (i = 0; i < GASKET_FRAMEWORK_DESC_MAX; i++) { g_descs[i].driver_desc = NULL; @@ -1804,13 +1921,8 @@ static int __init gasket_init(void) return 0; } -static void __exit gasket_exit(void) -{ - pr_debug("%s\n", __func__); -} MODULE_DESCRIPTION("Google Gasket driver framework"); MODULE_VERSION(GASKET_FRAMEWORK_VERSION); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Rob Springer <rspringer@google.com>"); module_init(gasket_init); -module_exit(gasket_exit);
diff --git a/drivers/staging/gasket/gasket_core.h b/drivers/staging/gasket/gasket_core.h index 275fd0b..7405b6a 100644 --- a/drivers/staging/gasket/gasket_core.h +++ b/drivers/staging/gasket/gasket_core.h
@@ -14,6 +14,7 @@ #include <linux/init.h> #include <linux/module.h> #include <linux/pci.h> +#include <linux/platform_device.h> #include <linux/sched.h> #include <linux/slab.h> @@ -50,8 +51,7 @@ enum gasket_interrupt_packing { /* Type of the interrupt supported by the device. */ enum gasket_interrupt_type { PCI_MSIX = 0, - PCI_MSI = 1, - PLATFORM_WIRE = 2, + DEVICE_MANAGED = 1, /* Managed externally in device driver */ }; /* @@ -69,12 +69,6 @@ struct gasket_interrupt_desc { int packing; }; -/* Offsets to the wire interrupt handling registers */ -struct gasket_wire_interrupt_offsets { - u64 pending_bit_array; - u64 mask_array; -}; - /* * This enum is used to identify memory regions being part of the physical * memory that belongs to a device. @@ -266,9 +260,15 @@ struct gasket_dev { /* Device info */ struct device *dev; - /* PCI subsystem metadata. */ + /* DMA device to use, may be same as above or a parent */ + struct device *dma_dev; + + /* PCI device pointer for PCI devices */ struct pci_dev *pci_dev; + /* Platform device pointer for platform devices */ + struct platform_device *platform_dev; + /* This device's index into internal_desc->devs. */ int dev_idx; @@ -384,9 +384,6 @@ struct gasket_driver_desc { */ struct gasket_coherent_buffer_desc coherent_buffer_description; - /* Offset of wire interrupt registers. */ - const struct gasket_wire_interrupt_offsets *wire_interrupt_offsets; - /* Interrupt type. (One of gasket_interrupt_type). */ int interrupt_type; @@ -543,6 +540,17 @@ int gasket_pci_add_device(struct pci_dev *pci_dev, /* Remove a PCI gasket device. */ void gasket_pci_remove_device(struct pci_dev *pci_dev); +/* Add a platform gasket device. */ +int gasket_platform_add_device(struct platform_device *pdev, + struct gasket_dev **gasket_devp); + +/* Remove a platform gasket device. */ +void gasket_platform_remove_device(struct platform_device *pdev); + +/* Set DMA device to use (if different from PCI/platform device) */ +void gasket_set_dma_device(struct gasket_dev *gasket_dev, + struct device *dma_dev); + /* Enable a Gasket device. */ int gasket_enable_device(struct gasket_dev *gasket_dev); @@ -587,28 +595,28 @@ const char *gasket_num_name_lookup(uint num, const struct gasket_num_name *table); /* Handy inlines */ -static inline ulong gasket_dev_read_64(struct gasket_dev *gasket_dev, int bar, +static inline u64 gasket_dev_read_64(struct gasket_dev *gasket_dev, int bar, ulong location) { - return readq(&gasket_dev->bar_data[bar].virt_base[location]); + return readq_relaxed(&gasket_dev->bar_data[bar].virt_base[location]); } static inline void gasket_dev_write_64(struct gasket_dev *dev, u64 value, int bar, ulong location) { - writeq(value, &dev->bar_data[bar].virt_base[location]); + writeq_relaxed(value, &dev->bar_data[bar].virt_base[location]); } static inline void gasket_dev_write_32(struct gasket_dev *dev, u32 value, int bar, ulong location) { - writel(value, &dev->bar_data[bar].virt_base[location]); + writel_relaxed(value, &dev->bar_data[bar].virt_base[location]); } static inline u32 gasket_dev_read_32(struct gasket_dev *dev, int bar, ulong location) { - return readl(&dev->bar_data[bar].virt_base[location]); + return readl_relaxed(&dev->bar_data[bar].virt_base[location]); } static inline void gasket_read_modify_write_64(struct gasket_dev *dev, int bar,
diff --git a/drivers/staging/gasket/gasket_interrupt.c b/drivers/staging/gasket/gasket_interrupt.c index 1cfbc12..1e8b960 100644 --- a/drivers/staging/gasket/gasket_interrupt.c +++ b/drivers/staging/gasket/gasket_interrupt.c
@@ -9,6 +9,7 @@ #include <linux/device.h> #include <linux/interrupt.h> #include <linux/printk.h> +#include <linux/rwlock.h> #include <linux/version.h> #ifdef GASKET_KERNEL_TRACE_SUPPORT #define CREATE_TRACE_POINTS @@ -45,9 +46,6 @@ struct gasket_interrupt_data { /* The width of a single interrupt in a packed interrupt register. */ int pack_width; - /* offset of wire interrupt registers */ - const struct gasket_wire_interrupt_offsets *wire_interrupt_offsets; - /* * Design-wise, these elements should be bundled together, but * pci_enable_msix's interface requires that they be managed @@ -63,6 +61,9 @@ struct gasket_interrupt_data { /* The eventfd "callback" data for each interrupt. */ struct eventfd_ctx **eventfd_ctxs; + /* Spinlock to protect read/write races to eventfd_ctxs. */ + rwlock_t eventfd_ctx_lock; + /* The number of times each interrupt has been called. */ ulong *interrupt_counts; @@ -80,8 +81,8 @@ static void gasket_interrupt_setup(struct gasket_dev *gasket_dev) { int i; int pack_shift; - ulong mask; - ulong value; + u64 mask; + u64 value; struct gasket_interrupt_data *interrupt_data = gasket_dev->interrupt_data; @@ -92,18 +93,8 @@ static void gasket_interrupt_setup(struct gasket_dev *gasket_dev) dev_dbg(gasket_dev->dev, "Running interrupt setup\n"); - if (interrupt_data->type == PLATFORM_WIRE || - interrupt_data->type == PCI_MSI) { - /* Nothing needs to be done for platform or PCI devices. */ - return; - } - - if (interrupt_data->type != PCI_MSIX) { - dev_dbg(gasket_dev->dev, - "Cannot handle unsupported interrupt type %d\n", - interrupt_data->type); - return; - } + if (interrupt_data->type == DEVICE_MANAGED) + return; /* device driver handles setup */ /* Setup the MSIX table. */ @@ -157,9 +148,24 @@ static void gasket_interrupt_setup(struct gasket_dev *gasket_dev) } } -static irqreturn_t gasket_msix_interrupt_handler(int irq, void *dev_id) +void +gasket_handle_interrupt(struct gasket_interrupt_data *interrupt_data, + int interrupt_index) { struct eventfd_ctx *ctx; + + trace_gasket_interrupt_event(interrupt_data->name, interrupt_index); + read_lock(&interrupt_data->eventfd_ctx_lock); + ctx = interrupt_data->eventfd_ctxs[interrupt_index]; + if (ctx) + eventfd_signal(ctx, 1); + read_unlock(&interrupt_data->eventfd_ctx_lock); + + ++(interrupt_data->interrupt_counts[interrupt_index]); +} + +static irqreturn_t gasket_msix_interrupt_handler(int irq, void *dev_id) +{ struct gasket_interrupt_data *interrupt_data = dev_id; int interrupt = -1; int i; @@ -175,14 +181,7 @@ static irqreturn_t gasket_msix_interrupt_handler(int irq, void *dev_id) pr_err("Received unknown irq %d\n", irq); return IRQ_HANDLED; } - trace_gasket_interrupt_event(interrupt_data->name, interrupt); - - ctx = interrupt_data->eventfd_ctxs[interrupt]; - if (ctx) - eventfd_signal(ctx, 1); - - ++(interrupt_data->interrupt_counts[interrupt]); - + gasket_handle_interrupt(interrupt_data, interrupt); return IRQ_HANDLED; } @@ -192,6 +191,12 @@ gasket_interrupt_msix_init(struct gasket_interrupt_data *interrupt_data) int ret = 1; int i; + interrupt_data->msix_entries = + kcalloc(interrupt_data->num_interrupts, + sizeof(struct msix_entry), GFP_KERNEL); + if (!interrupt_data->msix_entries) + return -ENOMEM; + for (i = 0; i < interrupt_data->num_interrupts; i++) { interrupt_data->msix_entries[i].entry = i; interrupt_data->msix_entries[i].vector = 0; @@ -319,58 +324,45 @@ static struct gasket_sysfs_attribute interrupt_sysfs_attrs[] = { GASKET_END_OF_ATTR_ARRAY, }; -int gasket_interrupt_init(struct gasket_dev *gasket_dev, const char *name, - int type, - const struct gasket_interrupt_desc *interrupts, - int num_interrupts, int pack_width, int bar_index, - const struct gasket_wire_interrupt_offsets *wire_int_offsets) +int gasket_interrupt_init(struct gasket_dev *gasket_dev) { int ret; struct gasket_interrupt_data *interrupt_data; + const struct gasket_driver_desc *driver_desc = + gasket_get_driver_desc(gasket_dev); interrupt_data = kzalloc(sizeof(struct gasket_interrupt_data), GFP_KERNEL); if (!interrupt_data) return -ENOMEM; gasket_dev->interrupt_data = interrupt_data; - interrupt_data->name = name; - interrupt_data->type = type; + interrupt_data->name = driver_desc->name; + interrupt_data->type = driver_desc->interrupt_type; interrupt_data->pci_dev = gasket_dev->pci_dev; - interrupt_data->num_interrupts = num_interrupts; - interrupt_data->interrupts = interrupts; - interrupt_data->interrupt_bar_index = bar_index; - interrupt_data->pack_width = pack_width; - interrupt_data->num_configured = 0; - interrupt_data->wire_interrupt_offsets = wire_int_offsets; + interrupt_data->num_interrupts = driver_desc->num_interrupts; + interrupt_data->interrupts = driver_desc->interrupts; + interrupt_data->interrupt_bar_index = driver_desc->interrupt_bar_index; + interrupt_data->pack_width = driver_desc->interrupt_pack_width; - /* Allocate all dynamic structures. */ - interrupt_data->msix_entries = kcalloc(num_interrupts, - sizeof(struct msix_entry), - GFP_KERNEL); - if (!interrupt_data->msix_entries) { - kfree(interrupt_data); - return -ENOMEM; - } - - interrupt_data->eventfd_ctxs = kcalloc(num_interrupts, + interrupt_data->eventfd_ctxs = kcalloc(driver_desc->num_interrupts, sizeof(struct eventfd_ctx *), GFP_KERNEL); if (!interrupt_data->eventfd_ctxs) { - kfree(interrupt_data->msix_entries); kfree(interrupt_data); return -ENOMEM; } - interrupt_data->interrupt_counts = kcalloc(num_interrupts, + interrupt_data->interrupt_counts = kcalloc(driver_desc->num_interrupts, sizeof(ulong), GFP_KERNEL); if (!interrupt_data->interrupt_counts) { kfree(interrupt_data->eventfd_ctxs); - kfree(interrupt_data->msix_entries); kfree(interrupt_data); return -ENOMEM; } + rwlock_init(&interrupt_data->eventfd_ctx_lock); + switch (interrupt_data->type) { case PCI_MSIX: ret = gasket_interrupt_msix_init(interrupt_data); @@ -379,12 +371,12 @@ int gasket_interrupt_init(struct gasket_dev *gasket_dev, const char *name, force_msix_interrupt_unmasking(gasket_dev); break; - case PCI_MSI: - case PLATFORM_WIRE: + case DEVICE_MANAGED: /* Device driver manages IRQ init */ + interrupt_data->num_configured = interrupt_data->num_interrupts; + ret = 0; + break; + default: - dev_err(gasket_dev->dev, - "Cannot handle unsupported interrupt type %d\n", - interrupt_data->type); ret = -EINVAL; } @@ -409,14 +401,17 @@ gasket_interrupt_msix_cleanup(struct gasket_interrupt_data *interrupt_data) { int i; - for (i = 0; i < interrupt_data->num_configured; i++) + for (i = 0; i < interrupt_data->num_configured; i++) { + gasket_interrupt_clear_eventfd(interrupt_data, i); free_irq(interrupt_data->msix_entries[i].vector, interrupt_data); + } interrupt_data->num_configured = 0; if (interrupt_data->msix_configured) pci_disable_msix(interrupt_data->pci_dev); interrupt_data->msix_configured = 0; + kfree(interrupt_data->msix_entries); } int gasket_interrupt_reinit(struct gasket_dev *gasket_dev) @@ -438,20 +433,20 @@ int gasket_interrupt_reinit(struct gasket_dev *gasket_dev) force_msix_interrupt_unmasking(gasket_dev); break; - case PCI_MSI: - case PLATFORM_WIRE: + case DEVICE_MANAGED: /* Device driver manages IRQ reinit */ + ret = 0; + break; + default: - dev_dbg(gasket_dev->dev, - "Cannot handle unsupported interrupt type %d\n", - gasket_dev->interrupt_data->type); ret = -EINVAL; } if (ret) { - /* Failing to setup MSIx will cause the device + /* Failing to setup interrupts will cause the device * to report GASKET_STATUS_LAMED, but is not fatal. */ - dev_warn(gasket_dev->dev, "Couldn't init msix: %d\n", ret); + dev_warn(gasket_dev->dev, "Couldn't reinit interrupts: %d\n", + ret); return 0; } @@ -487,17 +482,15 @@ void gasket_interrupt_cleanup(struct gasket_dev *gasket_dev) gasket_interrupt_msix_cleanup(interrupt_data); break; - case PCI_MSI: - case PLATFORM_WIRE: + case DEVICE_MANAGED: /* Device driver manages IRQ cleanup */ + break; + default: - dev_dbg(gasket_dev->dev, - "Cannot handle unsupported interrupt type %d\n", - interrupt_data->type); + break; } kfree(interrupt_data->interrupt_counts); kfree(interrupt_data->eventfd_ctxs); - kfree(interrupt_data->msix_entries); kfree(interrupt_data); gasket_dev->interrupt_data = NULL; } @@ -509,11 +502,6 @@ int gasket_interrupt_system_status(struct gasket_dev *gasket_dev) return GASKET_STATUS_DEAD; } - if (!gasket_dev->interrupt_data->msix_configured) { - dev_dbg(gasket_dev->dev, "Interrupt not initialized\n"); - return GASKET_STATUS_LAMED; - } - if (gasket_dev->interrupt_data->num_configured != gasket_dev->interrupt_data->num_interrupts) { dev_dbg(gasket_dev->dev, @@ -527,24 +515,38 @@ int gasket_interrupt_system_status(struct gasket_dev *gasket_dev) int gasket_interrupt_set_eventfd(struct gasket_interrupt_data *interrupt_data, int interrupt, int event_fd) { - struct eventfd_ctx *ctx = eventfd_ctx_fdget(event_fd); - - if (IS_ERR(ctx)) - return PTR_ERR(ctx); + struct eventfd_ctx *ctx; + ulong flags; if (interrupt < 0 || interrupt >= interrupt_data->num_interrupts) return -EINVAL; + ctx = eventfd_ctx_fdget(event_fd); + if (IS_ERR(ctx)) + return PTR_ERR(ctx); + + /* Put the old eventfd ctx before setting, else we leak the ref. */ + write_lock_irqsave(&interrupt_data->eventfd_ctx_lock, flags); + if (interrupt_data->eventfd_ctxs[interrupt] != NULL) + eventfd_ctx_put(interrupt_data->eventfd_ctxs[interrupt]); interrupt_data->eventfd_ctxs[interrupt] = ctx; + write_unlock_irqrestore(&interrupt_data->eventfd_ctx_lock, flags); return 0; } int gasket_interrupt_clear_eventfd(struct gasket_interrupt_data *interrupt_data, int interrupt) { + ulong flags; + if (interrupt < 0 || interrupt >= interrupt_data->num_interrupts) return -EINVAL; + /* Put the old eventfd ctx before clearing, else we leak the ref. */ + write_lock_irqsave(&interrupt_data->eventfd_ctx_lock, flags); + if (interrupt_data->eventfd_ctxs[interrupt] != NULL) + eventfd_ctx_put(interrupt_data->eventfd_ctxs[interrupt]); interrupt_data->eventfd_ctxs[interrupt] = NULL; + write_unlock_irqrestore(&interrupt_data->eventfd_ctx_lock, flags); return 0; }
diff --git a/drivers/staging/gasket/gasket_interrupt.h b/drivers/staging/gasket/gasket_interrupt.h index 835af439..b17b723 100644 --- a/drivers/staging/gasket/gasket_interrupt.h +++ b/drivers/staging/gasket/gasket_interrupt.h
@@ -24,30 +24,8 @@ struct gasket_interrupt_data; /* * Initialize the interrupt module. * @gasket_dev: The Gasket device structure for the device to be initted. - * @type: Type of the interrupt. (See gasket_interrupt_type). - * @name: The name to associate with these interrupts. - * @interrupts: An array of all interrupt descriptions for this device. - * @num_interrupts: The length of the @interrupts array. - * @pack_width: The width, in bits, of a single field in a packed interrupt reg. - * @bar_index: The bar containing all interrupt registers. - * - * Allocates and initializes data to track interrupt state for a device. - * After this call, no interrupts will be configured/delivered; call - * gasket_interrupt_set_vector[_packed] to associate each interrupt with an - * __iomem location, then gasket_interrupt_set_eventfd to associate an eventfd - * with an interrupt. - * - * If num_interrupts interrupts are not available, this call will return a - * negative error code. In that case, gasket_interrupt_cleanup should still be - * called. Returns 0 on success (which can include a device where interrupts - * are not possible to set up, but is otherwise OK; that device will report - * status LAMED.) */ -int gasket_interrupt_init(struct gasket_dev *gasket_dev, const char *name, - int type, - const struct gasket_interrupt_desc *interrupts, - int num_interrupts, int pack_width, int bar_index, - const struct gasket_wire_interrupt_offsets *wire_int_offsets); +int gasket_interrupt_init(struct gasket_dev *gasket_dev); /* * Clean up a device's interrupt structure. @@ -67,6 +45,11 @@ void gasket_interrupt_cleanup(struct gasket_dev *gasket_dev); */ int gasket_interrupt_reinit(struct gasket_dev *gasket_dev); +/* Handle gasket interrupt processing, called from an external handler. */ +void +gasket_handle_interrupt(struct gasket_interrupt_data *interrupt_data, + int interrupt_index); + /* * Reset the counts stored in the interrupt subsystem. * @gasket_dev: The Gasket information structure for this device.
diff --git a/drivers/staging/gasket/gasket_ioctl.c b/drivers/staging/gasket/gasket_ioctl.c index 0ca48e6..d36861d 100644 --- a/drivers/staging/gasket/gasket_ioctl.c +++ b/drivers/staging/gasket/gasket_ioctl.c
@@ -20,6 +20,7 @@ #define trace_gasket_ioctl_integer_data(x) #define trace_gasket_ioctl_eventfd_data(x, ...) #define trace_gasket_ioctl_page_table_data(x, ...) +#define trace_gasket_ioctl_page_table_flags_data(x, ...) #define trace_gasket_ioctl_config_coherent_allocator(x, ...) #endif @@ -130,29 +131,59 @@ static int gasket_partition_page_table( } /* Map a userspace buffer to a device virtual address. */ +static int gasket_map_buffers_common(struct gasket_dev *gasket_dev, + struct gasket_page_table_ioctl_flags + *pibuf) +{ + if (pibuf->base.page_table_index >= gasket_dev->num_page_tables) + return -EFAULT; + + if (gasket_page_table_are_addrs_bad(gasket_dev->page_table[pibuf->base.page_table_index], + pibuf->base.host_address, + pibuf->base.device_address, + pibuf->base.size)) + return -EINVAL; + + return gasket_page_table_map(gasket_dev->page_table[pibuf->base.page_table_index], + pibuf->base.host_address, + pibuf->base.device_address, + pibuf->base.size / PAGE_SIZE, + pibuf->flags); +} + static int gasket_map_buffers(struct gasket_dev *gasket_dev, struct gasket_page_table_ioctl __user *argp) { - struct gasket_page_table_ioctl ibuf; + struct gasket_page_table_ioctl_flags ibuf; - if (copy_from_user(&ibuf, argp, sizeof(struct gasket_page_table_ioctl))) + if (copy_from_user(&ibuf.base, argp, sizeof(struct gasket_page_table_ioctl))) return -EFAULT; - trace_gasket_ioctl_page_table_data(ibuf.page_table_index, ibuf.size, - ibuf.host_address, - ibuf.device_address); + ibuf.flags = 0; - if (ibuf.page_table_index >= gasket_dev->num_page_tables) + trace_gasket_ioctl_page_table_data(ibuf.base.page_table_index, + ibuf.base.size, + ibuf.base.host_address, + ibuf.base.device_address); + + return gasket_map_buffers_common(gasket_dev, &ibuf); +} + +static int gasket_map_buffers_flags(struct gasket_dev *gasket_dev, + struct gasket_page_table_ioctl_flags __user *argp) +{ + struct gasket_page_table_ioctl_flags ibuf; + + if (copy_from_user(&ibuf, argp, sizeof(struct gasket_page_table_ioctl_flags))) return -EFAULT; - if (gasket_page_table_are_addrs_bad(gasket_dev->page_table[ibuf.page_table_index], - ibuf.host_address, - ibuf.device_address, ibuf.size)) - return -EINVAL; + trace_gasket_ioctl_page_table_flags_data(ibuf.base.page_table_index, + ibuf.base.size, + ibuf.base.host_address, + ibuf.base.device_address, + ibuf.flags); - return gasket_page_table_map(gasket_dev->page_table[ibuf.page_table_index], - ibuf.host_address, ibuf.device_address, - ibuf.size / PAGE_SIZE); + return gasket_map_buffers_common(gasket_dev, &ibuf); } /* Unmap a userspace buffer from a device virtual address. */ @@ -191,6 +222,7 @@ static int gasket_config_coherent_allocator( { int ret; struct gasket_coherent_alloc_config_ioctl ibuf; + dma_addr_t dma_address; if (copy_from_user(&ibuf, argp, sizeof(struct gasket_coherent_alloc_config_ioctl))) @@ -206,16 +238,21 @@ static int gasket_config_coherent_allocator( return -ENOMEM; if (ibuf.enable == 0) { + dma_address = ibuf.dma_address; ret = gasket_free_coherent_memory(gasket_dev, ibuf.size, - ibuf.dma_address, + dma_address, ibuf.page_table_index); } else { ret = gasket_alloc_coherent_memory(gasket_dev, ibuf.size, - &ibuf.dma_address, + &dma_address, ibuf.page_table_index); } if (ret) return ret; + + if (ibuf.enable != 0) + ibuf.dma_address = dma_address; + if (copy_to_user(argp, &ibuf, sizeof(ibuf))) return -EFAULT; @@ -252,6 +289,7 @@ static bool gasket_ioctl_check_permissions(struct file *filp, uint cmd) return alive && write; case GASKET_IOCTL_MAP_BUFFER: + case GASKET_IOCTL_MAP_BUFFER_FLAGS: case GASKET_IOCTL_UNMAP_BUFFER: return alive && write; @@ -336,6 +374,9 @@ long gasket_handle_ioctl(struct file *filp, uint cmd, void __user *argp) case GASKET_IOCTL_MAP_BUFFER: retval = gasket_map_buffers(gasket_dev, argp); break; + case GASKET_IOCTL_MAP_BUFFER_FLAGS: + retval = gasket_map_buffers_flags(gasket_dev, argp); + break; case GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR: retval = gasket_config_coherent_allocator(gasket_dev, argp); break; @@ -381,6 +422,7 @@ long gasket_is_supported_ioctl(uint cmd) case GASKET_IOCTL_PAGE_TABLE_SIZE: case GASKET_IOCTL_SIMPLE_PAGE_TABLE_SIZE: case GASKET_IOCTL_MAP_BUFFER: + case GASKET_IOCTL_MAP_BUFFER_FLAGS: case GASKET_IOCTL_UNMAP_BUFFER: case GASKET_IOCTL_CLEAR_INTERRUPT_COUNTS: case GASKET_IOCTL_CONFIG_COHERENT_ALLOCATOR:
diff --git a/drivers/staging/gasket/gasket_page_table.c b/drivers/staging/gasket/gasket_page_table.c index d4c5f8a..b5438da 100644 --- a/drivers/staging/gasket/gasket_page_table.c +++ b/drivers/staging/gasket/gasket_page_table.c
@@ -10,10 +10,18 @@ * * This file assumes 4kB pages throughout; can be factored out when necessary. * - * Address format is as follows: + * There is a configurable number of page table entries, as well as a + * configurable bit index for the extended address flag. Both of these are + * specified in gasket_page_table_init through the page_table_config parameter. + * + * The following example assumes: + * page_table_config->total_entries = 8192 + * page_table_config->extended_bit = 63 + * + * Address format: * Simple addresses - those whose containing pages are directly placed in the * device's address translation registers - are laid out as: - * [ 63 - 40: Unused | 39 - 28: 0 | 27 - 12: page index | 11 - 0: page offset ] + * [ 63 - 25: 0 | 24 - 12: page index | 11 - 0: page offset ] * page index: The index of the containing page in the device's address * translation registers. * page offset: The index of the address into the containing page. @@ -21,7 +29,7 @@ * Extended address - those whose containing pages are contained in a second- * level page table whose address is present in the device's address translation * registers - are laid out as: - * [ 63 - 40: Unused | 39: flag | 38 - 37: 0 | 36 - 21: dev/level 0 index | + * [ 63: flag | 62 - 34: 0 | 33 - 21: dev/level 0 index | * 20 - 12: host/level 1 index | 11 - 0: page offset ] * flag: Marker indicating that this is an extended address. Always 1. * dev index: The index of the first-level page in the device's extended @@ -79,6 +87,19 @@ */ #define GASKET_EXTENDED_LVL1_SHIFT 12 +/* + * Utilities for accessing flags bitfields. + */ +#define MASK(field) (((1u << field##_WIDTH) - 1) << field##_SHIFT) +#define GET(field, flags) (((flags) & MASK(field)) >> field##_SHIFT) +#define SET(field, flags, val) (((flags) & ~MASK(field)) | ((val) << field##_SHIFT)) + +#define FLAGS_STATUS_SHIFT 0 +#define FLAGS_STATUS_WIDTH 1 + +#define FLAGS_DMA_DIRECTION_SHIFT 1 +#define FLAGS_DMA_DIRECTION_WIDTH 2 + /* Type declarations */ /* Valid states for a struct gasket_page_table_entry. */ enum pte_status { @@ -100,14 +121,12 @@ enum pte_status { * to the actual page mapped and described by this structure. */ struct gasket_page_table_entry { - /* The status of this entry/slot: free or in use. */ - enum pte_status status; - - /* Address of the page in DMA space. */ - dma_addr_t dma_addr; - - /* Linux page descriptor for the page described by this structure. */ - struct page *page; + /* + * Internal structure matches gasket_page_table_ioctl_flags.flags. + * NOTE: All fields should have a default value of 0. This ensures that + * the kernel will be backwards compatible with old drivers. + */ + u32 flags; /* * Index for alignment into host vaddrs. @@ -119,6 +138,12 @@ struct gasket_page_table_entry { */ int offset; + /* Address of the page in DMA space. */ + dma_addr_t dma_addr; + + /* Linux page descriptor for the page described by this structure. */ + struct page *page; + /* * If this is an extended and first-level entry, sublevel points * to the second-level entries underneath this entry. @@ -143,7 +168,7 @@ struct gasket_coherent_page_entry { u64 user_virt; /* User virtual address that was mapped by the mmap kernel subsystem */ - u64 kernel_virt; + dma_addr_t kernel_virt; /* * Whether this page has been mapped into a user land process virtual @@ -297,7 +322,7 @@ static bool gasket_is_pte_range_free(struct gasket_page_table_entry *ptes, int i; for (i = 0; i < num_entries; i++) { - if (ptes[i].status != PTE_FREE) + if (GET(FLAGS_STATUS, ptes[i].flags) != PTE_FREE) return false; } @@ -313,16 +338,14 @@ static void gasket_free_extended_subtable(struct gasket_page_table *pg_tbl, u64 __iomem *slot) { /* Release the page table from the driver */ - pte->status = PTE_FREE; + pte->flags = SET(FLAGS_STATUS, pte->flags, PTE_FREE); /* Release the page table from the device */ writeq(0, slot); - /* Force sync around the address release. */ - mb(); if (pte->dma_addr) dma_unmap_page(pg_tbl->device, pte->dma_addr, PAGE_SIZE, - DMA_BIDIRECTIONAL); + DMA_TO_DEVICE); vfree(pte->sublevel); @@ -349,7 +372,7 @@ gasket_page_table_garbage_collect_nolock(struct gasket_page_table *pg_tbl) slot = pg_tbl->base_slot + pg_tbl->num_simple_entries; pte < pg_tbl->entries + pg_tbl->config.total_entries; pte++, slot++) { - if (pte->status == PTE_INUSE) { + if (GET(FLAGS_STATUS, pte->flags) == PTE_INUSE) { if (gasket_is_pte_range_free(pte->sublevel, GASKET_PAGES_PER_SUBTABLE)) gasket_free_extended_subtable(pg_tbl, pte, @@ -398,7 +421,7 @@ int gasket_page_table_partition(struct gasket_page_table *pg_tbl, start = min(pg_tbl->num_simple_entries, num_simple_entries); for (i = start; i < pg_tbl->config.total_entries; i++) { - if (pg_tbl->entries[i].status != PTE_FREE) { + if (GET(FLAGS_STATUS, pg_tbl->entries[i].flags) != PTE_FREE) { dev_err(pg_tbl->device, "entry %d is not free\n", i); mutex_unlock(&pg_tbl->mutex); return -EBUSY; @@ -435,6 +458,19 @@ static int is_coherent(struct gasket_page_table *pg_tbl, ulong host_addr) return min <= host_addr && host_addr < max; } +/* Safely return a page to the OS. */ +static bool gasket_release_page(struct page *page) +{ + if (!page) + return false; + + if (!PageReserved(page)) + SetPageDirty(page); + put_page(page); + + return true; +} + /* * Get and map last level page table buffers. * @@ -446,7 +482,8 @@ static int is_coherent(struct gasket_page_table *pg_tbl, ulong host_addr) static int gasket_perform_mapping(struct gasket_page_table *pg_tbl, struct gasket_page_table_entry *ptes, u64 __iomem *slots, ulong host_addr, - uint num_pages, int is_simple_mapping) + uint num_pages, u32 flags, + int is_simple_mapping) { int ret; ulong offset; @@ -455,6 +492,12 @@ static int gasket_perform_mapping(struct gasket_page_table *pg_tbl, ulong page_addr; int i; + if (GET(FLAGS_DMA_DIRECTION, flags) == DMA_NONE) { + dev_err(pg_tbl->device, "invalid DMA direction flags=0x%lx\n", + (unsigned long)flags); + return -EINVAL; + } + for (i = 0; i < num_pages; i++) { page_addr = host_addr + i * PAGE_SIZE; offset = page_addr & (PAGE_SIZE - 1); @@ -484,44 +527,43 @@ static int gasket_perform_mapping(struct gasket_page_table *pg_tbl, ptes[i].offset = offset; /* Map the page into DMA space. */ - ptes[i].dma_addr = - dma_map_page(pg_tbl->device, page, 0, PAGE_SIZE, - DMA_BIDIRECTIONAL); + ptes[i].dma_addr = dma_map_page(pg_tbl->device, page, 0, PAGE_SIZE, + GET(FLAGS_DMA_DIRECTION, flags)); dev_dbg(pg_tbl->device, "%s i %d pte %p pfn %p -> mapped %llx\n", __func__, i, &ptes[i], (void *)page_to_pfn(page), (unsigned long long)ptes[i].dma_addr); - if (ptes[i].dma_addr == -1) { + if (dma_mapping_error(pg_tbl->device, + ptes[i].dma_addr)) { dev_dbg(pg_tbl->device, "%s i %d -> fail to map page %llx " - "[pfn %p ohys %p]\n", + "[pfn %p phys %p]\n", __func__, i, (unsigned long long)ptes[i].dma_addr, (void *)page_to_pfn(page), (void *)page_to_phys(page)); - return -1; + + /* clean up */ + if (gasket_release_page(ptes[i].page)) + --pg_tbl->num_active_pages; + + memset(&ptes[i], 0, sizeof(struct gasket_page_table_entry)); + return -EINVAL; } - /* Wait until the page is mapped. */ - mb(); } /* Make the DMA-space address available to the device. */ dma_addr = (ptes[i].dma_addr + offset) | GASKET_VALID_SLOT_FLAG; - if (is_simple_mapping) { + if (is_simple_mapping) writeq(dma_addr, &slots[i]); - } else { + else ((u64 __force *)slots)[i] = dma_addr; - /* Extended page table vectors are in DRAM, - * and so need to be synced each time they are updated. - */ - dma_map_single(pg_tbl->device, - (void *)&((u64 __force *)slots)[i], - sizeof(u64), DMA_TO_DEVICE); - } - ptes[i].status = PTE_INUSE; + + /* Set PTE flags equal to flags param with STATUS=PTE_INUSE. */ + ptes[i].flags = SET(FLAGS_STATUS, flags, PTE_INUSE); } return 0; } @@ -531,7 +573,7 @@ static int gasket_perform_mapping(struct gasket_page_table *pg_tbl, * Does not perform validity checking. */ static int gasket_simple_page_idx(struct gasket_page_table *pg_tbl, - ulong dev_addr) + u64 dev_addr) { return (dev_addr >> GASKET_SIMPLE_PAGE_SHIFT) & (pg_tbl->config.total_entries - 1); @@ -542,10 +584,10 @@ static int gasket_simple_page_idx(struct gasket_page_table *pg_tbl, * Does not perform validity checking. */ static ulong gasket_extended_lvl0_page_idx(struct gasket_page_table *pg_tbl, - ulong dev_addr) + u64 dev_addr) { return (dev_addr >> GASKET_EXTENDED_LVL0_SHIFT) & - ((1 << GASKET_EXTENDED_LVL0_WIDTH) - 1); + (pg_tbl->config.total_entries - 1); } /* @@ -553,7 +595,7 @@ static ulong gasket_extended_lvl0_page_idx(struct gasket_page_table *pg_tbl, * Does not perform validity checking. */ static ulong gasket_extended_lvl1_page_idx(struct gasket_page_table *pg_tbl, - ulong dev_addr) + u64 dev_addr) { return (dev_addr >> GASKET_EXTENDED_LVL1_SHIFT) & (GASKET_PAGES_PER_SUBTABLE - 1); @@ -564,7 +606,7 @@ static ulong gasket_extended_lvl1_page_idx(struct gasket_page_table *pg_tbl, * The page table mutex must be held by the caller. */ static int gasket_alloc_simple_entries(struct gasket_page_table *pg_tbl, - ulong dev_addr, uint num_pages) + u64 dev_addr, uint num_pages) { if (!gasket_is_pte_range_free(pg_tbl->entries + gasket_simple_page_idx(pg_tbl, dev_addr), @@ -574,19 +616,6 @@ static int gasket_alloc_simple_entries(struct gasket_page_table *pg_tbl, return 0; } -/* Safely return a page to the OS. */ -static bool gasket_release_page(struct page *page) -{ - if (!page) - return false; - - if (!PageReserved(page)) - SetPageDirty(page); - put_page(page); - - return true; -} - /* * Unmap and release mapped pages. * The page table mutex must be held by the caller. @@ -603,23 +632,20 @@ static void gasket_perform_unmapping(struct gasket_page_table *pg_tbl, */ for (i = 0; i < num_pages; i++) { /* release the address from the device, */ - if (is_simple_mapping || ptes[i].status == PTE_INUSE) + if (is_simple_mapping) writeq(0, &slots[i]); else ((u64 __force *)slots)[i] = 0; - /* Force sync around the address release. */ - mb(); /* release the address from the driver, */ - if (ptes[i].status == PTE_INUSE) { - if (ptes[i].dma_addr) { - dma_unmap_page(pg_tbl->device, ptes[i].dma_addr, - PAGE_SIZE, DMA_FROM_DEVICE); + if (GET(FLAGS_STATUS, ptes[i].flags) == PTE_INUSE) { + if (ptes[i].page && ptes[i].dma_addr) { + dma_unmap_page(pg_tbl->device, ptes[i].dma_addr, PAGE_SIZE, + GET(FLAGS_DMA_DIRECTION, ptes[i].flags)); } if (gasket_release_page(ptes[i].page)) --pg_tbl->num_active_pages; } - ptes[i].status = PTE_FREE; /* and clear the PTE. */ memset(&ptes[i], 0, sizeof(struct gasket_page_table_entry)); @@ -631,7 +657,7 @@ static void gasket_perform_unmapping(struct gasket_page_table *pg_tbl, * The page table mutex must be held by the caller. */ static void gasket_unmap_simple_pages(struct gasket_page_table *pg_tbl, - ulong dev_addr, uint num_pages) + u64 dev_addr, uint num_pages) { uint slot = gasket_simple_page_idx(pg_tbl, dev_addr); @@ -644,7 +670,7 @@ static void gasket_unmap_simple_pages(struct gasket_page_table *pg_tbl, * The page table mutex must be held by the caller. */ static void gasket_unmap_extended_pages(struct gasket_page_table *pg_tbl, - ulong dev_addr, uint num_pages) + u64 dev_addr, uint num_pages) { uint slot_idx, remain, len; struct gasket_page_table_entry *pte; @@ -659,12 +685,19 @@ static void gasket_unmap_extended_pages(struct gasket_page_table *pg_tbl, /* TODO: Add check to ensure pte remains valid? */ len = min(remain, GASKET_PAGES_PER_SUBTABLE - slot_idx); - if (pte->status == PTE_INUSE) { + if (GET(FLAGS_STATUS, pte->flags) == PTE_INUSE) { slot_base = (u64 __iomem *)(page_address(pte->page) + pte->offset); gasket_perform_unmapping(pg_tbl, pte->sublevel + slot_idx, slot_base + slot_idx, len, 0); + /* + * Extended page tables are in DRAM so they need to be + * synced each time they are updated. + */ + dma_sync_single_for_device(pg_tbl->device, + pte->dma_addr + slot_idx * sizeof(u64), + len * sizeof(u64), DMA_TO_DEVICE); } remain -= len; @@ -675,7 +708,7 @@ static void gasket_unmap_extended_pages(struct gasket_page_table *pg_tbl, /* Evaluates to nonzero if the specified virtual address is simple. */ static inline bool gasket_addr_is_simple(struct gasket_page_table *pg_tbl, - ulong addr) + u64 addr) { return !((addr) & (pg_tbl)->extended_flag); } @@ -684,38 +717,21 @@ static inline bool gasket_addr_is_simple(struct gasket_page_table *pg_tbl, * Convert (simple, page, offset) into a device address. * Examples: * Simple page 0, offset 32: - * Input (0, 0, 32), Output 0x20 + * Input (1, 0, 32), Output 0x20 * Simple page 1000, offset 511: - * Input (0, 1000, 512), Output 0x3E81FF + * Input (1, 1000, 511), Output 0x3E81FF * Extended page 0, offset 32: * Input (0, 0, 32), Output 0x8000000020 * Extended page 1000, offset 511: - * Input (1, 1000, 512), Output 0x8003E81FF + * Input (0, 1000, 511), Output 0x8003E81FF */ -static ulong gasket_components_to_dev_address(struct gasket_page_table *pg_tbl, +static u64 gasket_components_to_dev_address(struct gasket_page_table *pg_tbl, int is_simple, uint page_index, uint offset) { - ulong lvl0_index, lvl1_index; + u64 dev_addr = (page_index << GASKET_SIMPLE_PAGE_SHIFT) | offset; - if (is_simple) { - /* Return simple addresses directly. */ - lvl0_index = page_index & (pg_tbl->config.total_entries - 1); - return (lvl0_index << GASKET_SIMPLE_PAGE_SHIFT) | offset; - } - - /* - * This could be compressed into fewer statements, but - * A) the compiler should optimize it - * B) this is not slow - * C) this is an uncommon operation - * D) this is actually readable this way. - */ - lvl0_index = page_index / GASKET_PAGES_PER_SUBTABLE; - lvl1_index = page_index & (GASKET_PAGES_PER_SUBTABLE - 1); - return (pg_tbl)->extended_flag | - (lvl0_index << GASKET_EXTENDED_LVL0_SHIFT) | - (lvl1_index << GASKET_EXTENDED_LVL1_SHIFT) | offset; + return is_simple ? dev_addr : (pg_tbl->extended_flag | dev_addr); } /* @@ -726,7 +742,7 @@ static ulong gasket_components_to_dev_address(struct gasket_page_table *pg_tbl, * currently-partitioned simple pages. */ static bool gasket_is_simple_dev_addr_bad(struct gasket_page_table *pg_tbl, - ulong dev_addr, uint num_pages) + u64 dev_addr, uint num_pages) { ulong page_offset = dev_addr & (PAGE_SIZE - 1); ulong page_index = @@ -734,7 +750,7 @@ static bool gasket_is_simple_dev_addr_bad(struct gasket_page_table *pg_tbl, if (gasket_components_to_dev_address(pg_tbl, 1, page_index, page_offset) != dev_addr) { - dev_err(pg_tbl->device, "address is invalid, 0x%lX\n", + dev_err(pg_tbl->device, "address is invalid, 0x%llX\n", dev_addr); return true; } @@ -764,19 +780,19 @@ static bool gasket_is_simple_dev_addr_bad(struct gasket_page_table *pg_tbl, * currently-partitioned extended pages. */ static bool gasket_is_extended_dev_addr_bad(struct gasket_page_table *pg_tbl, - ulong dev_addr, uint num_pages) + u64 dev_addr, uint num_pages) { /* Starting byte index of dev_addr into the first mapped page */ ulong page_offset = dev_addr & (PAGE_SIZE - 1); ulong page_global_idx, page_lvl0_idx; ulong num_lvl0_pages; - ulong addr; + u64 addr; /* check if the device address is out of bound */ addr = dev_addr & ~((pg_tbl)->extended_flag); if (addr >> (GASKET_EXTENDED_LVL0_WIDTH + GASKET_EXTENDED_LVL0_SHIFT)) { - dev_err(pg_tbl->device, "device address out of bounds: 0x%lx\n", - dev_addr); + dev_err(pg_tbl->device, + "device address out of bounds: 0x%llx\n", dev_addr); return true; } @@ -793,7 +809,7 @@ static bool gasket_is_extended_dev_addr_bad(struct gasket_page_table *pg_tbl, if (gasket_components_to_dev_address(pg_tbl, 0, page_global_idx, page_offset) != dev_addr) { - dev_err(pg_tbl->device, "address is invalid: 0x%lx\n", + dev_err(pg_tbl->device, "address is invalid: 0x%llx\n", dev_addr); return true; } @@ -821,7 +837,7 @@ static bool gasket_is_extended_dev_addr_bad(struct gasket_page_table *pg_tbl, * The page table mutex must be held by the caller. */ static void gasket_page_table_unmap_nolock(struct gasket_page_table *pg_tbl, - ulong dev_addr, uint num_pages) + u64 dev_addr, uint num_pages) { if (!num_pages) return; @@ -837,8 +853,8 @@ static void gasket_page_table_unmap_nolock(struct gasket_page_table *pg_tbl, * If there is an error, no pages are mapped. */ static int gasket_map_simple_pages(struct gasket_page_table *pg_tbl, - ulong host_addr, ulong dev_addr, - uint num_pages) + ulong host_addr, u64 dev_addr, + uint num_pages, u32 flags) { int ret; uint slot_idx = gasket_simple_page_idx(pg_tbl, dev_addr); @@ -846,14 +862,15 @@ static int gasket_map_simple_pages(struct gasket_page_table *pg_tbl, ret = gasket_alloc_simple_entries(pg_tbl, dev_addr, num_pages); if (ret) { dev_err(pg_tbl->device, - "page table slots %u (@ 0x%lx) to %u are not available\n", - slot_idx, dev_addr, slot_idx + num_pages - 1); + "page table slots %u (@ 0x%llx) to %u are not available\n", + slot_idx, (long long unsigned int)dev_addr, + slot_idx + num_pages - 1); return ret; } ret = gasket_perform_mapping(pg_tbl, pg_tbl->entries + slot_idx, pg_tbl->base_slot + slot_idx, host_addr, - num_pages, 1); + num_pages, flags, 1); if (ret) { gasket_page_table_unmap_nolock(pg_tbl, dev_addr, num_pages); @@ -896,15 +913,29 @@ static int gasket_alloc_extended_subtable(struct gasket_page_table *pg_tbl, /* Map the page into DMA space. */ pte->dma_addr = dma_map_page(pg_tbl->device, pte->page, 0, PAGE_SIZE, - DMA_BIDIRECTIONAL); - /* Wait until the page is mapped. */ - mb(); + DMA_TO_DEVICE); + if (dma_mapping_error(pg_tbl->device, pte->dma_addr)) { + dev_dbg(pg_tbl->device, + "%s -> fail to map page %llx " + "[pfn %p phys %p]\n", + __func__, + (unsigned long long)pte->dma_addr, + (void *)page_to_pfn(pte->page), + (void *)page_to_phys(pte->page)); + + /* clean up */ + free_page(page_addr); + vfree(pte->sublevel); + memset(pte, 0, sizeof(struct gasket_page_table_entry)); + + return -ENOMEM; + } /* make the addresses available to the device */ dma_addr = (pte->dma_addr + pte->offset) | GASKET_VALID_SLOT_FLAG; writeq(dma_addr, slot); - pte->status = PTE_INUSE; + pte->flags = SET(FLAGS_STATUS, pte->flags, PTE_INUSE); return 0; } @@ -924,7 +955,7 @@ static int gasket_alloc_extended_subtable(struct gasket_page_table *pg_tbl, * The page table mutex must be held by the caller. */ static int gasket_alloc_extended_entries(struct gasket_page_table *pg_tbl, - ulong dev_addr, uint num_entries) + u64 dev_addr, uint num_entries) { int ret = 0; uint remain, subtable_slot_idx, len; @@ -942,7 +973,7 @@ static int gasket_alloc_extended_entries(struct gasket_page_table *pg_tbl, len = min(remain, GASKET_PAGES_PER_SUBTABLE - subtable_slot_idx); - if (pte->status == PTE_FREE) { + if (GET(FLAGS_STATUS, pte->flags) == PTE_FREE) { ret = gasket_alloc_extended_subtable(pg_tbl, pte, slot); if (ret) { dev_err(pg_tbl->device, @@ -969,11 +1000,11 @@ static int gasket_alloc_extended_entries(struct gasket_page_table *pg_tbl, * If there is an error, no pages are mapped. */ static int gasket_map_extended_pages(struct gasket_page_table *pg_tbl, - ulong host_addr, ulong dev_addr, - uint num_pages) + ulong host_addr, u64 dev_addr, + uint num_pages, u32 flags) { int ret; - ulong dev_addr_end; + u64 dev_addr_end; uint slot_idx, remain, len; struct gasket_page_table_entry *pte; u64 __iomem *slot_base; @@ -982,11 +1013,11 @@ static int gasket_map_extended_pages(struct gasket_page_table *pg_tbl, if (ret) { dev_addr_end = dev_addr + (num_pages / PAGE_SIZE) - 1; dev_err(pg_tbl->device, - "page table slots (%lu,%lu) (@ 0x%lx) to (%lu,%lu) are " - "not available\n", + "page table slots (%lu,%lu) (@ 0x%llx) to (%lu,%lu) " + "are not available\n", gasket_extended_lvl0_page_idx(pg_tbl, dev_addr), - dev_addr, gasket_extended_lvl1_page_idx(pg_tbl, dev_addr), + (long long unsigned int)dev_addr, gasket_extended_lvl0_page_idx(pg_tbl, dev_addr_end), gasket_extended_lvl1_page_idx(pg_tbl, dev_addr_end)); return ret; @@ -1004,13 +1035,21 @@ static int gasket_map_extended_pages(struct gasket_page_table *pg_tbl, (u64 __iomem *)(page_address(pte->page) + pte->offset); ret = gasket_perform_mapping(pg_tbl, pte->sublevel + slot_idx, slot_base + slot_idx, host_addr, - len, 0); + len, flags, 0); if (ret) { gasket_page_table_unmap_nolock(pg_tbl, dev_addr, num_pages); return ret; } + /* + * Extended page tables are in DRAM so they need to be synced + * each time they are updated. + */ + dma_sync_single_for_device(pg_tbl->device, + pte->dma_addr + slot_idx * sizeof(u64), + len * sizeof(u64), DMA_TO_DEVICE); + remain -= len; slot_idx = 0; pte++; @@ -1029,7 +1068,7 @@ static int gasket_map_extended_pages(struct gasket_page_table *pg_tbl, * The page table mutex is held for the entire operation. */ int gasket_page_table_map(struct gasket_page_table *pg_tbl, ulong host_addr, - ulong dev_addr, uint num_pages) + u64 dev_addr, uint num_pages, u32 flags) { int ret; @@ -1040,18 +1079,18 @@ int gasket_page_table_map(struct gasket_page_table *pg_tbl, ulong host_addr, if (gasket_addr_is_simple(pg_tbl, dev_addr)) { ret = gasket_map_simple_pages(pg_tbl, host_addr, dev_addr, - num_pages); + num_pages, flags); } else { ret = gasket_map_extended_pages(pg_tbl, host_addr, dev_addr, - num_pages); + num_pages, flags); } mutex_unlock(&pg_tbl->mutex); dev_dbg(pg_tbl->device, - "%s done: ha %llx daddr %llx num %d, ret %d\n", + "%s done: ha %llx daddr %llx num %d, flags %x ret %d\n", __func__, (unsigned long long)host_addr, - (unsigned long long)dev_addr, num_pages, ret); + (unsigned long long)dev_addr, num_pages, flags, ret); return ret; } EXPORT_SYMBOL(gasket_page_table_map); @@ -1065,7 +1104,7 @@ EXPORT_SYMBOL(gasket_page_table_map); * * The page table mutex is held for the entire operation. */ -void gasket_page_table_unmap(struct gasket_page_table *pg_tbl, ulong dev_addr, +void gasket_page_table_unmap(struct gasket_page_table *pg_tbl, u64 dev_addr, uint num_pages) { if (!num_pages) @@ -1110,7 +1149,7 @@ void gasket_page_table_reset(struct gasket_page_table *pg_tbl) /* See gasket_page_table.h for description. */ int gasket_page_table_lookup_page( - struct gasket_page_table *pg_tbl, ulong dev_addr, struct page **ppage, + struct gasket_page_table *pg_tbl, u64 dev_addr, struct page **ppage, ulong *poffset) { uint page_num; @@ -1123,7 +1162,7 @@ int gasket_page_table_lookup_page( goto fail; pte = pg_tbl->entries + page_num; - if (pte->status != PTE_INUSE) + if (GET(FLAGS_STATUS, pte->flags) != PTE_INUSE) goto fail; } else { /* Find the level 0 entry, */ @@ -1132,13 +1171,13 @@ int gasket_page_table_lookup_page( goto fail; pte = pg_tbl->entries + pg_tbl->num_simple_entries + page_num; - if (pte->status != PTE_INUSE) + if (GET(FLAGS_STATUS, pte->flags) != PTE_INUSE) goto fail; /* and its contained level 1 entry. */ page_num = gasket_extended_lvl1_page_idx(pg_tbl, dev_addr); pte = pte->sublevel + page_num; - if (pte->status != PTE_INUSE) + if (GET(FLAGS_STATUS, pte->flags) != PTE_INUSE) goto fail; } @@ -1151,12 +1190,12 @@ int gasket_page_table_lookup_page( *ppage = NULL; *poffset = 0; mutex_unlock(&pg_tbl->mutex); - return -1; + return -EINVAL; } /* See gasket_page_table.h for description. */ bool gasket_page_table_are_addrs_bad( - struct gasket_page_table *pg_tbl, ulong host_addr, ulong dev_addr, + struct gasket_page_table *pg_tbl, ulong host_addr, u64 dev_addr, ulong bytes) { if (host_addr & (PAGE_SIZE - 1)) { @@ -1172,7 +1211,7 @@ EXPORT_SYMBOL(gasket_page_table_are_addrs_bad); /* See gasket_page_table.h for description. */ bool gasket_page_table_is_dev_addr_bad( - struct gasket_page_table *pg_tbl, ulong dev_addr, ulong bytes) + struct gasket_page_table *pg_tbl, u64 dev_addr, ulong bytes) { uint num_pages = bytes / PAGE_SIZE; @@ -1291,7 +1330,7 @@ int gasket_alloc_coherent_memory(struct gasket_dev *gasket_dev, u64 size, return -EINVAL; mem = dma_alloc_coherent(gasket_get_device(gasket_dev), - num_pages * PAGE_SIZE, &handle, 0); + num_pages * PAGE_SIZE, &handle, GFP_KERNEL); if (!mem) goto nomem; @@ -1303,7 +1342,6 @@ int gasket_alloc_coherent_memory(struct gasket_dev *gasket_dev, u64 size, GFP_KERNEL); if (!gasket_dev->page_table[index]->coherent_pages) goto nomem; - *dma_address = 0; gasket_dev->coherent_buffer.length_bytes = PAGE_SIZE * (num_pages); @@ -1315,23 +1353,22 @@ int gasket_alloc_coherent_memory(struct gasket_dev *gasket_dev, u64 size, gasket_dev->page_table[index]->coherent_pages[j].paddr = handle + j * PAGE_SIZE; gasket_dev->page_table[index]->coherent_pages[j].kernel_virt = - (u64)mem + j * PAGE_SIZE; + (ulong)mem + j * PAGE_SIZE; } - if (*dma_address == 0) - goto nomem; return 0; nomem: if (mem) { dma_free_coherent(gasket_get_device(gasket_dev), num_pages * PAGE_SIZE, mem, handle); + gasket_dev->coherent_buffer.length_bytes = 0; + gasket_dev->coherent_buffer.virt_base = NULL; + gasket_dev->coherent_buffer.phys_base = 0; } - if (gasket_dev->page_table[index]->coherent_pages) { - kfree(gasket_dev->page_table[index]->coherent_pages); - gasket_dev->page_table[index]->coherent_pages = NULL; - } + kfree(gasket_dev->page_table[index]->coherent_pages); + gasket_dev->page_table[index]->coherent_pages = NULL; gasket_dev->page_table[index]->num_coherent_pages = 0; return -ENOMEM; } @@ -1350,15 +1387,8 @@ int gasket_free_coherent_memory(struct gasket_dev *gasket_dev, u64 size, if (driver_desc->coherent_buffer_description.base != dma_address) return -EADDRNOTAVAIL; - if (gasket_dev->coherent_buffer.length_bytes) { - dma_free_coherent(gasket_get_device(gasket_dev), - gasket_dev->coherent_buffer.length_bytes, - gasket_dev->coherent_buffer.virt_base, - gasket_dev->coherent_buffer.phys_base); - gasket_dev->coherent_buffer.length_bytes = 0; - gasket_dev->coherent_buffer.virt_base = NULL; - gasket_dev->coherent_buffer.phys_base = 0; - } + gasket_free_coherent_memory_all(gasket_dev, index); + return 0; } @@ -1378,4 +1408,8 @@ void gasket_free_coherent_memory_all( gasket_dev->coherent_buffer.virt_base = NULL; gasket_dev->coherent_buffer.phys_base = 0; } + + kfree(gasket_dev->page_table[index]->coherent_pages); + gasket_dev->page_table[index]->coherent_pages = NULL; + gasket_dev->page_table[index]->num_coherent_pages = 0; }
diff --git a/drivers/staging/gasket/gasket_page_table.h b/drivers/staging/gasket/gasket_page_table.h index 7b01b73..609e1d9 100644 --- a/drivers/staging/gasket/gasket_page_table.h +++ b/drivers/staging/gasket/gasket_page_table.h
@@ -85,6 +85,8 @@ int gasket_page_table_partition(struct gasket_page_table *page_table, * @host_addr: Starting host virtual memory address of the pages. * @dev_addr: Starting device address of the pages. * @num_pages: Number of [4kB] pages to map. + * @flags: Specifies attributes to apply to the pages. + * Internal structure matches gasket_page_table_ioctl_flags.flags. * * Description: Maps the "num_pages" pages of host memory pointed to by * host_addr to the address "dev_addr" in device memory. @@ -95,7 +97,7 @@ int gasket_page_table_partition(struct gasket_page_table *page_table, * If there is an error, no pages are mapped. */ int gasket_page_table_map(struct gasket_page_table *page_table, ulong host_addr, - ulong dev_addr, uint num_pages); + u64 dev_addr, uint num_pages, u32 flags); /* * Un-map host pages from device memory. @@ -106,7 +108,7 @@ int gasket_page_table_map(struct gasket_page_table *page_table, ulong host_addr, * Description: The inverse of gasket_map_pages. Unmaps pages from the device. */ void gasket_page_table_unmap(struct gasket_page_table *page_table, - ulong dev_addr, uint num_pages); + u64 dev_addr, uint num_pages); /* * Unmap ALL host pages from device memory. @@ -146,7 +148,7 @@ void gasket_page_table_garbage_collect(struct gasket_page_table *page_table); * and offset are returned through the pointers, if successful. */ int gasket_page_table_lookup_page(struct gasket_page_table *page_table, - ulong dev_addr, struct page **page, + u64 dev_addr, struct page **page, ulong *poffset); /* @@ -163,7 +165,7 @@ int gasket_page_table_lookup_page(struct gasket_page_table *page_table, * Returns true if the mapping is bad, false otherwise. */ bool gasket_page_table_are_addrs_bad(struct gasket_page_table *page_table, - ulong host_addr, ulong dev_addr, + ulong host_addr, u64 dev_addr, ulong bytes); /* @@ -179,7 +181,7 @@ bool gasket_page_table_are_addrs_bad(struct gasket_page_table *page_table, * Returns true if the address is bad, false otherwise. */ bool gasket_page_table_is_dev_addr_bad(struct gasket_page_table *page_table, - ulong dev_addr, ulong bytes); + u64 dev_addr, ulong bytes); /* * Gets maximum size for the given page table.
diff --git a/drivers/staging/gasket/gasket_sysfs.h b/drivers/staging/gasket/gasket_sysfs.h index f32eaf8..e8f29a3 100644 --- a/drivers/staging/gasket/gasket_sysfs.h +++ b/drivers/staging/gasket/gasket_sysfs.h
@@ -79,6 +79,13 @@ struct gasket_sysfs_attribute { .data.attr_type = _attr_type \ } +#define GASKET_SYSFS_RW(_name, _show_function, _store_function, _attr_type) \ + { \ + .attr = __ATTR(_name, S_IWUSR | S_IWGRP | S_IRUGO, \ + _show_function, _store_function), \ + .data.attr_type = _attr_type \ + } + /* Initializes the Gasket sysfs subsystem. * * Description: Performs one-time initialization. Must be called before usage @@ -152,8 +159,8 @@ void gasket_sysfs_put_device_data(struct device *device, * Returns the Gasket sysfs attribute associated with the kernel device * attribute and device structure itself. Upon success, this call will take a * reference to internal sysfs data that must be released with a call to - * gasket_sysfs_get_device_data. While this reference is held, the underlying - * device sysfs information/structure will remain valid/will not be deleted. + * gasket_sysfs_put_attr. While this reference is held, the underlying device + * sysfs information/structure will remain valid/will not be deleted. */ struct gasket_sysfs_attribute * gasket_sysfs_get_attr(struct device *device, struct device_attribute *attr);