--- a/usr/src/uts/i86pc/io/amd_iommu/amd_iommu.c Thu Sep 17 10:25:33 2009 +0800
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,444 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-
-/*
- * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
- * Use is subject to license terms.
- */
-
-#include <sys/types.h>
-#include <sys/file.h>
-#include <sys/errno.h>
-#include <sys/open.h>
-#include <sys/stat.h>
-#include <sys/cred.h>
-#include <sys/modctl.h>
-#include <sys/conf.h>
-#include <sys/devops.h>
-#include <sys/ddi.h>
-#include <sys/x86_archext.h>
-
-#include <sys/amd_iommu.h>
-#include "amd_iommu_impl.h"
-#include "amd_iommu_acpi.h"
-
-
-#define AMD_IOMMU_MINOR2INST(x) (x)
-#define AMD_IOMMU_INST2MINOR(x) (x)
-#define AMD_IOMMU_NODETYPE "ddi_iommu"
-#define AMD_IOMMU_MINOR_NAME "amd-iommu"
-
-static int amd_iommu_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg,
- void **result);
-static int amd_iommu_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
-static int amd_iommu_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
-static int amd_iommu_open(dev_t *devp, int flag, int otyp, cred_t *credp);
-static int amd_iommu_close(dev_t dev, int flag, int otyp, cred_t *credp);
-static int amd_iommu_ioctl(dev_t dev, int cmd, intptr_t arg, int mode,
- cred_t *credp, int *rvalp);
-
-static struct cb_ops amd_iommu_cb_ops = {
- amd_iommu_open, /* cb_open */
- amd_iommu_close, /* cb_close */
- nodev, /* cb_strategy */
- nodev, /* cb_print */
- nodev, /* cb_dump */
- nodev, /* cb_read */
- nodev, /* cb_write */
- amd_iommu_ioctl, /* cb_ioctl */
- nodev, /* cb_devmap */
- nodev, /* cb_mmap */
- nodev, /* cb_segmap */
- nochpoll, /* cb_chpoll */
- ddi_prop_op, /* cb_prop_op */
- NULL, /* cb_str */
- D_NEW | D_MP, /* cb_flag */
- CB_REV, /* cb_rev */
- nodev, /* cb_aread */
- nodev /* cb_awrite */
-};
-
-static struct dev_ops amd_iommu_dev_ops = {
- DEVO_REV, /* devo_rev */
- 0, /* devo_refcnt */
- amd_iommu_getinfo, /* devo_getinfo */
- nulldev, /* devo_identify */
- nulldev, /* devo_probe */
- amd_iommu_attach, /* devo_attach */
- amd_iommu_detach, /* devo_detach */
- nodev, /* devo_reset */
- &amd_iommu_cb_ops, /* devo_cb_ops */
- NULL, /* devo_bus_ops */
- nulldev /* devo_power */
-};
-
-static struct modldrv modldrv = {
- &mod_driverops,
- "AMD IOMMU 0.1",
- &amd_iommu_dev_ops
-};
-
-static struct modlinkage modlinkage = {
- MODREV_1,
- (void *)&modldrv,
- NULL
-};
-
-amd_iommu_debug_t amd_iommu_debug;
-kmutex_t amd_iommu_global_lock;
-const char *amd_iommu_modname = "amd_iommu";
-amd_iommu_alias_t **amd_iommu_alias;
-amd_iommu_page_table_hash_t amd_iommu_page_table_hash;
-static void *amd_iommu_statep;
-int amd_iommu_64bit_bug;
-int amd_iommu_unity_map;
-int amd_iommu_no_RW_perms;
-int amd_iommu_no_unmap;
-int amd_iommu_pageva_inval_all;
-int amd_iommu_disable; /* disable IOMMU */
-char *amd_iommu_disable_list; /* list of drivers bypassing IOMMU */
-
-int
-_init(void)
-{
- int error = ENOTSUP;
-
-#if defined(__amd64) && !defined(__xpv)
-
- if (get_hwenv() != HW_NATIVE)
- return (ENOTSUP);
-
- error = ddi_soft_state_init(&amd_iommu_statep,
- sizeof (struct amd_iommu_state), 1);
- if (error) {
- cmn_err(CE_WARN, "%s: _init: failed to init soft state.",
- amd_iommu_modname);
- return (error);
- }
-
- if (amd_iommu_acpi_init() != DDI_SUCCESS) {
- if (amd_iommu_debug) {
- cmn_err(CE_WARN, "%s: _init: ACPI init failed.",
- amd_iommu_modname);
- }
- ddi_soft_state_fini(&amd_iommu_statep);
- return (ENOTSUP);
- }
-
- amd_iommu_read_boot_props();
-
- if (amd_iommu_page_table_hash_init(&amd_iommu_page_table_hash)
- != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: _init: Page table hash init failed.",
- amd_iommu_modname);
- if (amd_iommu_disable_list) {
- kmem_free(amd_iommu_disable_list,
- strlen(amd_iommu_disable_list) + 1);
- amd_iommu_disable_list = NULL;
- }
- amd_iommu_acpi_fini();
- ddi_soft_state_fini(&amd_iommu_statep);
- amd_iommu_statep = NULL;
- return (EFAULT);
- }
-
- error = mod_install(&modlinkage);
- if (error) {
- cmn_err(CE_WARN, "%s: _init: mod_install failed.",
- amd_iommu_modname);
- amd_iommu_page_table_hash_fini(&amd_iommu_page_table_hash);
- if (amd_iommu_disable_list) {
- kmem_free(amd_iommu_disable_list,
- strlen(amd_iommu_disable_list) + 1);
- amd_iommu_disable_list = NULL;
- }
- amd_iommu_acpi_fini();
- ddi_soft_state_fini(&amd_iommu_statep);
- amd_iommu_statep = NULL;
- return (error);
- }
- error = 0;
-#endif
-
- return (error);
-}
-
-int
-_info(struct modinfo *modinfop)
-{
- return (mod_info(&modlinkage, modinfop));
-}
-
-int
-_fini(void)
-{
- int error;
-
- error = mod_remove(&modlinkage);
- if (error)
- return (error);
-
- amd_iommu_page_table_hash_fini(&amd_iommu_page_table_hash);
- if (amd_iommu_disable_list) {
- kmem_free(amd_iommu_disable_list,
- strlen(amd_iommu_disable_list) + 1);
- amd_iommu_disable_list = NULL;
- }
- amd_iommu_acpi_fini();
- ddi_soft_state_fini(&amd_iommu_statep);
- amd_iommu_statep = NULL;
-
- return (0);
-}
-
-/*ARGSUSED*/
-static int
-amd_iommu_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
-{
- struct amd_iommu_state *statep;
-
- ASSERT(result);
-
- *result = NULL;
-
- switch (cmd) {
- case DDI_INFO_DEVT2DEVINFO:
- statep = ddi_get_soft_state(amd_iommu_statep,
- AMD_IOMMU_MINOR2INST(getminor((dev_t)arg)));
- if (statep) {
- *result = statep->aioms_devi;
- return (DDI_SUCCESS);
- }
- break;
- case DDI_INFO_DEVT2INSTANCE:
- *result = (void *)(uintptr_t)
- AMD_IOMMU_MINOR2INST(getminor((dev_t)arg));
- return (DDI_SUCCESS);
- }
-
- return (DDI_FAILURE);
-}
-
-static int
-amd_iommu_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
-{
- int instance = ddi_get_instance(dip);
- const char *driver = ddi_driver_name(dip);
- struct amd_iommu_state *statep;
-
- ASSERT(instance >= 0);
- ASSERT(driver);
-
- switch (cmd) {
- case DDI_ATTACH:
- if (ddi_soft_state_zalloc(amd_iommu_statep, instance)
- != DDI_SUCCESS) {
- cmn_err(CE_WARN, "Unable to allocate soft state for "
- "%s%d", driver, instance);
- return (DDI_FAILURE);
- }
-
- statep = ddi_get_soft_state(amd_iommu_statep, instance);
- if (statep == NULL) {
- cmn_err(CE_WARN, "Unable to get soft state for "
- "%s%d", driver, instance);
- ddi_soft_state_free(amd_iommu_statep, instance);
- return (DDI_FAILURE);
- }
-
- if (ddi_create_minor_node(dip, AMD_IOMMU_MINOR_NAME, S_IFCHR,
- AMD_IOMMU_INST2MINOR(instance), AMD_IOMMU_NODETYPE,
- 0) != DDI_SUCCESS) {
- cmn_err(CE_WARN, "Unable to create minor node for "
- "%s%d", driver, instance);
- ddi_remove_minor_node(dip, NULL);
- ddi_soft_state_free(amd_iommu_statep, instance);
- return (DDI_FAILURE);
- }
-
- statep->aioms_devi = dip;
- statep->aioms_instance = instance;
- statep->aioms_iommu_start = NULL;
- statep->aioms_iommu_end = NULL;
-
- amd_iommu_lookup_conf_props(dip);
-
- if (amd_iommu_disable_list) {
- cmn_err(CE_NOTE, "AMD IOMMU disabled for the following"
- " drivers:\n%s", amd_iommu_disable_list);
- }
-
- if (amd_iommu_disable) {
- cmn_err(CE_NOTE, "AMD IOMMU disabled by user");
- } else if (amd_iommu_setup(dip, statep) != DDI_SUCCESS) {
- cmn_err(CE_WARN, "Unable to initialize AMD IOMMU "
- "%s%d", driver, instance);
- ddi_remove_minor_node(dip, NULL);
- ddi_soft_state_free(amd_iommu_statep, instance);
- return (DDI_FAILURE);
- }
-
- ddi_report_dev(dip);
-
- return (DDI_SUCCESS);
-
- case DDI_RESUME:
- return (DDI_SUCCESS);
- default:
- return (DDI_FAILURE);
- }
-}
-
-static int
-amd_iommu_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
-{
- int instance = ddi_get_instance(dip);
- const char *driver = ddi_driver_name(dip);
- struct amd_iommu_state *statep;
-
- ASSERT(instance >= 0);
- ASSERT(driver);
-
- switch (cmd) {
- case DDI_DETACH:
- statep = ddi_get_soft_state(amd_iommu_statep, instance);
- if (statep == NULL) {
- cmn_err(CE_WARN, "%s%d: Cannot get soft state",
- driver, instance);
- return (DDI_FAILURE);
- }
- return (DDI_FAILURE);
- case DDI_SUSPEND:
- return (DDI_SUCCESS);
- default:
- return (DDI_FAILURE);
- }
-}
-
-/*ARGSUSED*/
-static int
-amd_iommu_open(dev_t *devp, int flag, int otyp, cred_t *credp)
-{
- int instance = AMD_IOMMU_MINOR2INST(getminor(*devp));
- struct amd_iommu_state *statep;
- const char *f = "amd_iommu_open";
-
- if (instance < 0) {
- cmn_err(CE_WARN, "%s: invalid instance %d",
- f, instance);
- return (ENXIO);
- }
-
- if (!(flag & (FREAD|FWRITE))) {
- cmn_err(CE_WARN, "%s: invalid flags %d", f, flag);
- return (EINVAL);
- }
-
- if (otyp != OTYP_CHR) {
- cmn_err(CE_WARN, "%s: invalid otyp %d", f, otyp);
- return (EINVAL);
- }
-
- statep = ddi_get_soft_state(amd_iommu_statep, instance);
- if (statep == NULL) {
- cmn_err(CE_WARN, "%s: cannot get soft state: instance %d",
- f, instance);
- return (ENXIO);
- }
-
- ASSERT(statep->aioms_instance == instance);
-
- return (0);
-}
-
-/*ARGSUSED*/
-static int
-amd_iommu_close(dev_t dev, int flag, int otyp, cred_t *credp)
-{
- int instance = AMD_IOMMU_MINOR2INST(getminor(dev));
- struct amd_iommu_state *statep;
- const char *f = "amd_iommu_close";
-
- if (instance < 0) {
- cmn_err(CE_WARN, "%s: invalid instance %d", f, instance);
- return (ENXIO);
- }
-
- if (!(flag & (FREAD|FWRITE))) {
- cmn_err(CE_WARN, "%s: invalid flags %d", f, flag);
- return (EINVAL);
- }
-
- if (otyp != OTYP_CHR) {
- cmn_err(CE_WARN, "%s: invalid otyp %d", f, otyp);
- return (EINVAL);
- }
-
- statep = ddi_get_soft_state(amd_iommu_statep, instance);
- if (statep == NULL) {
- cmn_err(CE_WARN, "%s: cannot get soft state: instance %d",
- f, instance);
- return (ENXIO);
- }
-
- ASSERT(statep->aioms_instance == instance);
- return (0);
-
-}
-
-/*ARGSUSED*/
-static int
-amd_iommu_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp,
- int *rvalp)
-{
- int instance = AMD_IOMMU_MINOR2INST(getminor(dev));
- struct amd_iommu_state *statep;
- const char *f = "amd_iommu_ioctl";
-
- ASSERT(*rvalp);
-
- if (instance < 0) {
- cmn_err(CE_WARN, "%s: invalid instance %d", f, instance);
- return (ENXIO);
- }
-
-
- if (!(mode & (FREAD|FWRITE))) {
- cmn_err(CE_WARN, "%s: invalid mode %d", f, mode);
- return (EINVAL);
- }
-
- if (mode & FKIOCTL) {
- cmn_err(CE_WARN, "%s: FKIOCTL unsupported mode %d", f, mode);
- return (EINVAL);
- }
-
- statep = ddi_get_soft_state(amd_iommu_statep, instance);
- if (statep == NULL) {
- cmn_err(CE_WARN, "%s: cannot get soft state: instance %d",
- f, instance);
- return (ENXIO);
- }
-
- ASSERT(statep->aioms_instance == instance);
-
- return (ENOTTY);
-}
--- a/usr/src/uts/i86pc/io/amd_iommu/amd_iommu_impl.c Thu Sep 17 10:25:33 2009 +0800
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,1880 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-
-/*
- * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
- * Use is subject to license terms.
- */
-
-#include <sys/sunddi.h>
-#include <sys/iommulib.h>
-#include <sys/amd_iommu.h>
-#include <sys/pci_cap.h>
-#include <sys/bootconf.h>
-#include <sys/ddidmareq.h>
-
-#include "amd_iommu_impl.h"
-#include "amd_iommu_acpi.h"
-#include "amd_iommu_page_tables.h"
-
-static int amd_iommu_fini(amd_iommu_t *iommu);
-static void amd_iommu_teardown_interrupts(amd_iommu_t *iommu);
-static void amd_iommu_stop(amd_iommu_t *iommu);
-
-static int amd_iommu_probe(iommulib_handle_t handle, dev_info_t *rdip);
-static int amd_iommu_allochdl(iommulib_handle_t handle,
- dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
- int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *dma_handlep);
-static int amd_iommu_freehdl(iommulib_handle_t handle,
- dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle);
-static int amd_iommu_bindhdl(iommulib_handle_t handle, dev_info_t *dip,
- dev_info_t *rdip, ddi_dma_handle_t dma_handle,
- struct ddi_dma_req *dmareq, ddi_dma_cookie_t *cookiep,
- uint_t *ccountp);
-static int amd_iommu_unbindhdl(iommulib_handle_t handle,
- dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle);
-static int amd_iommu_sync(iommulib_handle_t handle, dev_info_t *dip,
- dev_info_t *rdip, ddi_dma_handle_t dma_handle, off_t off,
- size_t len, uint_t cache_flags);
-static int amd_iommu_win(iommulib_handle_t handle, dev_info_t *dip,
- dev_info_t *rdip, ddi_dma_handle_t dma_handle, uint_t win,
- off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
- uint_t *ccountp);
-static int amd_iommu_map(iommulib_handle_t handle, dev_info_t *dip,
- dev_info_t *rdip, struct ddi_dma_req *dmareq,
- ddi_dma_handle_t *dma_handle);
-static int amd_iommu_mctl(iommulib_handle_t handle, dev_info_t *dip,
- dev_info_t *rdip, ddi_dma_handle_t dma_handle,
- enum ddi_dma_ctlops request, off_t *offp, size_t *lenp,
- caddr_t *objpp, uint_t cache_flags);
-
-static int unmap_current_window(amd_iommu_t *iommu, dev_info_t *rdip,
- ddi_dma_cookie_t *cookie_array, uint_t ccount, int ncookies, int locked);
-
-extern void *device_arena_alloc(size_t size, int vm_flag);
-extern void device_arena_free(void * vaddr, size_t size);
-
-ddi_dma_attr_t amd_iommu_dma_attr = {
- DMA_ATTR_V0,
- 0U, /* dma_attr_addr_lo */
- 0xffffffffffffffffULL, /* dma_attr_addr_hi */
- 0xffffffffU, /* dma_attr_count_max */
- (uint64_t)4096, /* dma_attr_align */
- 1, /* dma_attr_burstsizes */
- 64, /* dma_attr_minxfer */
- 0xffffffffU, /* dma_attr_maxxfer */
- 0xffffffffU, /* dma_attr_seg */
- 1, /* dma_attr_sgllen, variable */
- 64, /* dma_attr_granular */
- 0 /* dma_attr_flags */
-};
-
-ddi_device_acc_attr_t amd_iommu_devacc = {
- DDI_DEVICE_ATTR_V0,
- DDI_NEVERSWAP_ACC,
- DDI_STRICTORDER_ACC
-};
-
-struct iommulib_ops amd_iommulib_ops = {
- IOMMU_OPS_VERSION,
- AMD_IOMMU,
- "AMD IOMMU Vers. 1",
- NULL,
- amd_iommu_probe,
- amd_iommu_allochdl,
- amd_iommu_freehdl,
- amd_iommu_bindhdl,
- amd_iommu_unbindhdl,
- amd_iommu_sync,
- amd_iommu_win,
- amd_iommu_map,
- amd_iommu_mctl
-};
-
-static kmutex_t amd_iommu_pgtable_lock;
-
-static int
-amd_iommu_register(amd_iommu_t *iommu)
-{
- dev_info_t *dip = iommu->aiomt_dip;
- const char *driver = ddi_driver_name(dip);
- int instance = ddi_get_instance(dip);
- iommulib_ops_t *iommulib_ops;
- iommulib_handle_t handle;
- const char *f = "amd_iommu_register";
-
- iommulib_ops = kmem_zalloc(sizeof (iommulib_ops_t), KM_SLEEP);
-
- *iommulib_ops = amd_iommulib_ops;
-
- iommulib_ops->ilops_data = (void *)iommu;
- iommu->aiomt_iommulib_ops = iommulib_ops;
-
- if (iommulib_iommu_register(dip, iommulib_ops, &handle)
- != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: Register with iommulib "
- "failed idx=%d", f, driver, instance, iommu->aiomt_idx);
- kmem_free(iommulib_ops, sizeof (iommulib_ops_t));
- return (DDI_FAILURE);
- }
-
- iommu->aiomt_iommulib_handle = handle;
-
- return (DDI_SUCCESS);
-}
-
-static int
-amd_iommu_unregister(amd_iommu_t *iommu)
-{
- if (iommu->aiomt_iommulib_handle == NULL) {
- /* we never registered */
- return (DDI_SUCCESS);
- }
-
- if (iommulib_iommu_unregister(iommu->aiomt_iommulib_handle)
- != DDI_SUCCESS) {
- return (DDI_FAILURE);
- }
-
- kmem_free(iommu->aiomt_iommulib_ops, sizeof (iommulib_ops_t));
- iommu->aiomt_iommulib_ops = NULL;
- iommu->aiomt_iommulib_handle = NULL;
-
- return (DDI_SUCCESS);
-}
-
-static int
-amd_iommu_setup_passthru(amd_iommu_t *iommu)
-{
- gfx_entry_t *gfxp;
- dev_info_t *dip;
-
- /*
- * Setup passthru mapping for "special" devices
- */
- amd_iommu_set_passthru(iommu, NULL);
-
- for (gfxp = gfx_devinfo_list; gfxp; gfxp = gfxp->g_next) {
- gfxp->g_ref++;
- dip = gfxp->g_dip;
- if (dip) {
- amd_iommu_set_passthru(iommu, dip);
- }
- gfxp->g_ref--;
- }
-
- return (DDI_SUCCESS);
-}
-
-static int
-amd_iommu_start(amd_iommu_t *iommu)
-{
- dev_info_t *dip = iommu->aiomt_dip;
- int instance = ddi_get_instance(dip);
- const char *driver = ddi_driver_name(dip);
- amd_iommu_acpi_ivhd_t *hinfop;
- const char *f = "amd_iommu_start";
-
- hinfop = amd_iommu_lookup_all_ivhd();
-
- /*
- * Disable HT tunnel translation.
- * XXX use ACPI
- */
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
- AMD_IOMMU_HT_TUN_ENABLE, 0);
-
- if (hinfop) {
- if (amd_iommu_debug) {
- cmn_err(CE_NOTE,
- "amd_iommu: using ACPI for CTRL registers");
- }
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
- AMD_IOMMU_ISOC, hinfop->ach_Isoc);
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
- AMD_IOMMU_RESPASSPW, hinfop->ach_ResPassPW);
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
- AMD_IOMMU_PASSPW, hinfop->ach_PassPW);
- }
-
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
- AMD_IOMMU_INVTO, 5);
-
-
- /*
- * The Device table entry bit 0 (V) controls whether the device
- * table entry is valid for address translation and Device table
- * entry bit 128 (IV) controls whether interrupt remapping is valid.
- * By setting both to zero we are essentially doing pass-thru. Since
- * this table is zeroed on allocation, essentially we will have
- * pass-thru when IOMMU is enabled.
- */
-
- /* Finally enable the IOMMU ... */
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
- AMD_IOMMU_ENABLE, 1);
-
- if (amd_iommu_debug) {
- cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. "
- "Successfully started AMD IOMMU", f, driver, instance,
- iommu->aiomt_idx);
- }
- cmn_err(CE_NOTE, "AMD IOMMU (%d,%d) enabled",
- instance, iommu->aiomt_idx);
-
- return (DDI_SUCCESS);
-}
-
-static void
-amd_iommu_stop(amd_iommu_t *iommu)
-{
- dev_info_t *dip = iommu->aiomt_dip;
- int instance = ddi_get_instance(dip);
- const char *driver = ddi_driver_name(dip);
- const char *f = "amd_iommu_stop";
-
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
- AMD_IOMMU_ENABLE, 0);
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
- AMD_IOMMU_EVENTINT_ENABLE, 0);
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
- AMD_IOMMU_COMWAITINT_ENABLE, 0);
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
- AMD_IOMMU_EVENTLOG_ENABLE, 0);
-
- /*
- * Disable translation on HT tunnel traffic
- */
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
- AMD_IOMMU_HT_TUN_ENABLE, 0);
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
- AMD_IOMMU_CMDBUF_ENABLE, 0);
-
- cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMYU idx=%d. "
- "Successfully stopped AMD IOMMU", f, driver, instance,
- iommu->aiomt_idx);
-}
-
-static int
-amd_iommu_setup_tables_and_buffers(amd_iommu_t *iommu)
-{
- dev_info_t *dip = iommu->aiomt_dip;
- int instance = ddi_get_instance(dip);
- const char *driver = ddi_driver_name(dip);
- uint32_t dma_bufsz;
- caddr_t addr;
- uint32_t sz;
- uint32_t p2sz;
- int i;
- uint64_t *dentry;
- int err;
- const char *f = "amd_iommu_setup_tables_and_buffers";
-
- /*
- * We will put the Device Table, Command Buffer and
- * Event Log in contiguous memory. Allocate the maximum
- * size allowed for such structures
- * Device Table: 256b * 64K = 32B * 64K
- * Command Buffer: 128b * 32K = 16B * 32K
- * Event Log: 128b * 32K = 16B * 32K
- */
- iommu->aiomt_devtbl_sz = (1<<AMD_IOMMU_DEVTBL_SZ) * AMD_IOMMU_DEVENT_SZ;
- iommu->aiomt_cmdbuf_sz = (1<<AMD_IOMMU_CMDBUF_SZ) * AMD_IOMMU_CMD_SZ;
- iommu->aiomt_eventlog_sz =
- (1<<AMD_IOMMU_EVENTLOG_SZ) * AMD_IOMMU_EVENT_SZ;
-
- dma_bufsz = iommu->aiomt_devtbl_sz + iommu->aiomt_cmdbuf_sz
- + iommu->aiomt_eventlog_sz;
-
- /*
- * Alloc a DMA handle.
- */
- err = ddi_dma_alloc_handle(dip, &amd_iommu_dma_attr,
- DDI_DMA_SLEEP, NULL, &iommu->aiomt_dmahdl);
- if (err != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: Cannot alloc DMA handle for "
- "AMD IOMMU tables and buffers", f, driver, instance);
- return (DDI_FAILURE);
- }
-
- /*
- * Alloc memory for tables and buffers
- * XXX remove cast to size_t
- */
- err = ddi_dma_mem_alloc(iommu->aiomt_dmahdl, dma_bufsz,
- &amd_iommu_devacc, DDI_DMA_CONSISTENT|IOMEM_DATA_UNCACHED,
- DDI_DMA_SLEEP, NULL, (caddr_t *)&iommu->aiomt_dma_bufva,
- (size_t *)&iommu->aiomt_dma_mem_realsz, &iommu->aiomt_dma_mem_hdl);
- if (err != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: Cannot alloc memory for DMA "
- "to AMD IOMMU tables and buffers", f, driver, instance);
- iommu->aiomt_dma_bufva = NULL;
- iommu->aiomt_dma_mem_realsz = 0;
- ddi_dma_free_handle(&iommu->aiomt_dmahdl);
- iommu->aiomt_dmahdl = NULL;
- return (DDI_FAILURE);
- }
-
- /*
- * The VA must be 4K aligned and >= table size
- */
- ASSERT(((uintptr_t)iommu->aiomt_dma_bufva &
- AMD_IOMMU_TABLE_ALIGN) == 0);
- ASSERT(iommu->aiomt_dma_mem_realsz >= dma_bufsz);
-
- /*
- * Now bind the handle
- */
- err = ddi_dma_addr_bind_handle(iommu->aiomt_dmahdl, NULL,
- iommu->aiomt_dma_bufva, iommu->aiomt_dma_mem_realsz,
- DDI_DMA_RDWR | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP,
- NULL, &iommu->aiomt_buf_dma_cookie, &iommu->aiomt_buf_dma_ncookie);
- if (err != DDI_DMA_MAPPED) {
- cmn_err(CE_WARN, "%s: %s%d: Cannot bind memory for DMA "
- "to AMD IOMMU tables and buffers. bufrealsz=%p",
- f, driver, instance,
- (void *)(uintptr_t)iommu->aiomt_dma_mem_realsz);
- iommu->aiomt_buf_dma_cookie.dmac_laddress = 0;
- iommu->aiomt_buf_dma_cookie.dmac_size = 0;
- iommu->aiomt_buf_dma_cookie.dmac_type = 0;
- iommu->aiomt_buf_dma_ncookie = 0;
- ddi_dma_mem_free(&iommu->aiomt_dma_mem_hdl);
- iommu->aiomt_dma_mem_hdl = NULL;
- iommu->aiomt_dma_bufva = NULL;
- iommu->aiomt_dma_mem_realsz = 0;
- ddi_dma_free_handle(&iommu->aiomt_dmahdl);
- iommu->aiomt_dmahdl = NULL;
- return (DDI_FAILURE);
- }
-
- /*
- * We assume the DMA engine on the IOMMU is capable of handling the
- * whole table buffer in a single cookie. If not and multiple cookies
- * are needed we fail.
- */
- if (iommu->aiomt_buf_dma_ncookie != 1) {
- cmn_err(CE_WARN, "%s: %s%d: Cannot handle multiple "
- "cookies for DMA to AMD IOMMU tables and buffers. "
- "#cookies=%u", f, driver, instance,
- iommu->aiomt_buf_dma_ncookie);
- (void) ddi_dma_unbind_handle(iommu->aiomt_dmahdl);
- iommu->aiomt_buf_dma_cookie.dmac_laddress = 0;
- iommu->aiomt_buf_dma_cookie.dmac_size = 0;
- iommu->aiomt_buf_dma_cookie.dmac_type = 0;
- iommu->aiomt_buf_dma_ncookie = 0;
- ddi_dma_mem_free(&iommu->aiomt_dma_mem_hdl);
- iommu->aiomt_dma_mem_hdl = NULL;
- iommu->aiomt_dma_bufva = NULL;
- iommu->aiomt_dma_mem_realsz = 0;
- ddi_dma_free_handle(&iommu->aiomt_dmahdl);
- iommu->aiomt_dmahdl = NULL;
- return (DDI_FAILURE);
- }
-
- /*
- * The address in the cookie must be 4K aligned and >= table size
- */
- ASSERT((iommu->aiomt_buf_dma_cookie.dmac_cookie_addr
- & AMD_IOMMU_TABLE_ALIGN) == 0);
- ASSERT(iommu->aiomt_buf_dma_cookie.dmac_size
- <= iommu->aiomt_dma_mem_realsz);
- ASSERT(iommu->aiomt_buf_dma_cookie.dmac_size >= dma_bufsz);
-
- /*
- * Setup the device table pointers in the iommu struct as
- * well as the IOMMU device table register
- */
- iommu->aiomt_devtbl = iommu->aiomt_dma_bufva;
- bzero(iommu->aiomt_devtbl, iommu->aiomt_devtbl_sz);
-
- /*
- * Set V=1 and TV = 0, so any inadvertant pass-thrus cause
- * page faults. Also set SE bit so we aren't swamped with
- * page fault messages
- */
- for (i = 0; i <= AMD_IOMMU_MAX_DEVICEID; i++) {
- /*LINTED*/
- dentry = (uint64_t *)&iommu->aiomt_devtbl
- [i * AMD_IOMMU_DEVTBL_ENTRY_SZ];
- AMD_IOMMU_REG_SET64(dentry, AMD_IOMMU_DEVTBL_V, 1);
- AMD_IOMMU_REG_SET64(&(dentry[1]), AMD_IOMMU_DEVTBL_SE, 1);
- }
-
- addr = (caddr_t)(uintptr_t)iommu->aiomt_buf_dma_cookie.dmac_cookie_addr;
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va),
- AMD_IOMMU_DEVTABBASE, ((uint64_t)(uintptr_t)addr) >> 12);
- sz = (iommu->aiomt_devtbl_sz >> 12) - 1;
- ASSERT(sz <= ((1 << 9) - 1));
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va),
- AMD_IOMMU_DEVTABSIZE, sz);
-
- /*
- * Setup the command buffer pointers
- */
- iommu->aiomt_cmdbuf = iommu->aiomt_devtbl +
- iommu->aiomt_devtbl_sz;
- bzero(iommu->aiomt_cmdbuf, iommu->aiomt_cmdbuf_sz);
- addr += iommu->aiomt_devtbl_sz;
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va),
- AMD_IOMMU_COMBASE, ((uint64_t)(uintptr_t)addr) >> 12);
-
- p2sz = AMD_IOMMU_CMDBUF_SZ;
- ASSERT(p2sz >= AMD_IOMMU_CMDBUF_MINSZ &&
- p2sz <= AMD_IOMMU_CMDBUF_MAXSZ);
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va),
- AMD_IOMMU_COMLEN, p2sz);
- /*LINTED*/
- iommu->aiomt_cmd_tail = (uint32_t *)iommu->aiomt_cmdbuf;
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_head_va),
- AMD_IOMMU_CMDHEADPTR, 0);
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_tail_va),
- AMD_IOMMU_CMDTAILPTR, 0);
-
- /*
- * Setup the event log pointers
- */
- iommu->aiomt_eventlog = iommu->aiomt_cmdbuf +
- iommu->aiomt_eventlog_sz;
- bzero(iommu->aiomt_eventlog, iommu->aiomt_eventlog_sz);
- addr += iommu->aiomt_cmdbuf_sz;
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va),
- AMD_IOMMU_EVENTBASE, ((uint64_t)(uintptr_t)addr) >> 12);
- p2sz = AMD_IOMMU_EVENTLOG_SZ;
- ASSERT(p2sz >= AMD_IOMMU_EVENTLOG_MINSZ &&
- p2sz <= AMD_IOMMU_EVENTLOG_MAXSZ);
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va),
- AMD_IOMMU_EVENTLEN, sz);
- /*LINTED*/
- iommu->aiomt_event_head = (uint32_t *)iommu->aiomt_eventlog;
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_head_va),
- AMD_IOMMU_EVENTHEADPTR, 0);
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_tail_va),
- AMD_IOMMU_EVENTTAILPTR, 0);
-
- /* dma sync so device sees this init */
- SYNC_FORDEV(iommu->aiomt_dmahdl);
-
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_TABLES) {
- cmn_err(CE_NOTE, "%s: %s%d: successfully setup AMD IOMMU "
- "tables, idx=%d", f, driver, instance, iommu->aiomt_idx);
- }
-
- return (DDI_SUCCESS);
-}
-
-static void
-amd_iommu_teardown_tables_and_buffers(amd_iommu_t *iommu)
-{
- dev_info_t *dip = iommu->aiomt_dip;
- int instance = ddi_get_instance(dip);
- const char *driver = ddi_driver_name(dip);
- const char *f = "amd_iommu_teardown_tables_and_buffers";
-
- iommu->aiomt_eventlog = NULL;
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va),
- AMD_IOMMU_EVENTBASE, 0);
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_eventlog_va),
- AMD_IOMMU_EVENTLEN, 0);
-
- iommu->aiomt_cmdbuf = NULL;
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va),
- AMD_IOMMU_COMBASE, 0);
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_cmdbuf_va),
- AMD_IOMMU_COMLEN, 0);
-
- iommu->aiomt_devtbl = NULL;
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va),
- AMD_IOMMU_DEVTABBASE, 0);
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_devtbl_va),
- AMD_IOMMU_DEVTABSIZE, 0);
-
- if (iommu->aiomt_dmahdl == NULL)
- return;
-
- /* Unbind the handle */
- if (ddi_dma_unbind_handle(iommu->aiomt_dmahdl) != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: failed to unbind handle: "
- "%p for IOMMU idx=%d", f, driver, instance,
- (void *)iommu->aiomt_dmahdl, iommu->aiomt_idx);
- }
- iommu->aiomt_buf_dma_cookie.dmac_laddress = 0;
- iommu->aiomt_buf_dma_cookie.dmac_size = 0;
- iommu->aiomt_buf_dma_cookie.dmac_type = 0;
- iommu->aiomt_buf_dma_ncookie = 0;
-
- /* Free the table memory allocated for DMA */
- ddi_dma_mem_free(&iommu->aiomt_dma_mem_hdl);
- iommu->aiomt_dma_mem_hdl = NULL;
- iommu->aiomt_dma_bufva = NULL;
- iommu->aiomt_dma_mem_realsz = 0;
-
- /* Free the DMA handle */
- ddi_dma_free_handle(&iommu->aiomt_dmahdl);
- iommu->aiomt_dmahdl = NULL;
-}
-
-static void
-amd_iommu_enable_interrupts(amd_iommu_t *iommu)
-{
- ASSERT(AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va),
- AMD_IOMMU_CMDBUF_RUN) == 0);
- ASSERT(AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va),
- AMD_IOMMU_EVENT_LOG_RUN) == 0);
-
- /* Must be set prior to enabling command buffer */
- /* Must be set prior to enabling event logging */
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
- AMD_IOMMU_CMDBUF_ENABLE, 1);
- /* No interrupts for completion wait - too heavy weight. use polling */
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
- AMD_IOMMU_COMWAITINT_ENABLE, 0);
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
- AMD_IOMMU_EVENTLOG_ENABLE, 1);
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_ctrl_va),
- AMD_IOMMU_EVENTINT_ENABLE, 1);
-}
-
-static int
-amd_iommu_setup_exclusion(amd_iommu_t *iommu)
-{
- amd_iommu_acpi_ivmd_t *minfop;
-
- minfop = amd_iommu_lookup_all_ivmd();
-
- if (minfop && minfop->acm_ExclRange == 1) {
- cmn_err(CE_NOTE, "Programming exclusion range");
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
- AMD_IOMMU_EXCL_BASE_ADDR,
- minfop->acm_ivmd_phys_start >> 12);
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
- AMD_IOMMU_EXCL_BASE_ALLOW, 1);
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
- AMD_IOMMU_EXCL_BASE_EXEN, 1);
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_lim_va),
- AMD_IOMMU_EXCL_LIM, (minfop->acm_ivmd_phys_start +
- minfop->acm_ivmd_phys_len) >> 12);
- } else {
- if (amd_iommu_debug) {
- cmn_err(CE_NOTE, "Skipping exclusion range");
- }
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
- AMD_IOMMU_EXCL_BASE_ADDR, 0);
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
- AMD_IOMMU_EXCL_BASE_ALLOW, 1);
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_base_va),
- AMD_IOMMU_EXCL_BASE_EXEN, 0);
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_excl_lim_va),
- AMD_IOMMU_EXCL_LIM, 0);
- }
-
- return (DDI_SUCCESS);
-}
-
-static void
-amd_iommu_teardown_exclusion(amd_iommu_t *iommu)
-{
- (void) amd_iommu_setup_exclusion(iommu);
-}
-
-static uint_t
-amd_iommu_intr_handler(caddr_t arg1, caddr_t arg2)
-{
- /*LINTED*/
- amd_iommu_t *iommu = (amd_iommu_t *)arg1;
- dev_info_t *dip = iommu->aiomt_dip;
- int instance = ddi_get_instance(dip);
- const char *driver = ddi_driver_name(dip);
- const char *f = "amd_iommu_intr_handler";
-
- ASSERT(arg1);
- ASSERT(arg2 == NULL);
-
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
- cmn_err(CE_NOTE, "%s: %s%d: IOMMU unit idx=%d. In INTR handler",
- f, driver, instance, iommu->aiomt_idx);
- }
-
- if (AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va),
- AMD_IOMMU_EVENT_LOG_INT) == 1) {
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
- cmn_err(CE_NOTE, "%s: %s%d: IOMMU unit idx=%d "
- "Event Log Interrupt", f, driver, instance,
- iommu->aiomt_idx);
- }
- (void) amd_iommu_read_log(iommu, AMD_IOMMU_LOG_DISPLAY);
- WAIT_SEC(1);
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_status_va),
- AMD_IOMMU_EVENT_LOG_INT, 1);
- return (DDI_INTR_CLAIMED);
- }
-
- if (AMD_IOMMU_REG_GET64(REGADDR64(iommu->aiomt_reg_status_va),
- AMD_IOMMU_EVENT_OVERFLOW_INT) == 1) {
- cmn_err(CE_NOTE, "!%s: %s%d: IOMMU unit idx=%d "
- "Event Overflow Interrupt", f, driver, instance,
- iommu->aiomt_idx);
- (void) amd_iommu_read_log(iommu, AMD_IOMMU_LOG_DISCARD);
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_status_va),
- AMD_IOMMU_EVENT_LOG_INT, 1);
- AMD_IOMMU_REG_SET64(REGADDR64(iommu->aiomt_reg_status_va),
- AMD_IOMMU_EVENT_OVERFLOW_INT, 1);
- return (DDI_INTR_CLAIMED);
- }
-
- return (DDI_INTR_UNCLAIMED);
-}
-
-
-static int
-amd_iommu_setup_interrupts(amd_iommu_t *iommu)
-{
- dev_info_t *dip = iommu->aiomt_dip;
- int instance = ddi_get_instance(dip);
- const char *driver = ddi_driver_name(dip);
- int intrcap0;
- int intrcapN;
- int type;
- int err;
- int req;
- int avail;
- int p2req;
- int actual;
- int i;
- int j;
- const char *f = "amd_iommu_setup_interrupts";
-
- if (ddi_intr_get_supported_types(dip, &type) != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: ddi_intr_get_supported_types "
- "failed: idx=%d", f, driver, instance, iommu->aiomt_idx);
- return (DDI_FAILURE);
- }
-
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
- cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. "
- "Interrupt types supported = 0x%x", f, driver, instance,
- iommu->aiomt_idx, type);
- }
-
- /*
- * for now we only support MSI
- */
- if ((type & DDI_INTR_TYPE_MSI) == 0) {
- cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d. "
- "MSI interrupts not supported. Failing init.",
- f, driver, instance, iommu->aiomt_idx);
- return (DDI_FAILURE);
- }
-
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
- cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. MSI supported",
- f, driver, instance, iommu->aiomt_idx);
- }
-
- err = ddi_intr_get_nintrs(dip, DDI_INTR_TYPE_MSI, &req);
- if (err != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d. "
- "ddi_intr_get_nintrs failed err = %d",
- f, driver, instance, iommu->aiomt_idx, err);
- return (DDI_FAILURE);
- }
-
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
- cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. "
- "MSI number of interrupts requested: %d",
- f, driver, instance, iommu->aiomt_idx, req);
- }
-
- if (req == 0) {
- cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: 0 MSI "
- "interrupts requested. Failing init", f,
- driver, instance, iommu->aiomt_idx);
- return (DDI_FAILURE);
- }
-
- err = ddi_intr_get_navail(dip, DDI_INTR_TYPE_MSI, &avail);
- if (err != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d "
- "ddi_intr_get_navail failed err = %d", f,
- driver, instance, iommu->aiomt_idx, err);
- return (DDI_FAILURE);
- }
-
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
- cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. "
- "MSI number of interrupts available: %d",
- f, driver, instance, iommu->aiomt_idx, avail);
- }
-
- if (avail == 0) {
- cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: 0 MSI "
- "interrupts available. Failing init", f,
- driver, instance, iommu->aiomt_idx);
- return (DDI_FAILURE);
- }
-
- if (avail < req) {
- cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: MSI "
- "interrupts: requested (%d) > available (%d). "
- "Failing init", f, driver, instance, iommu->aiomt_idx,
- req, avail);
- return (DDI_FAILURE);
- }
-
- /* Allocate memory for DDI interrupt handles */
- iommu->aiomt_intr_htable_sz = req * sizeof (ddi_intr_handle_t);
- iommu->aiomt_intr_htable = kmem_zalloc(iommu->aiomt_intr_htable_sz,
- KM_SLEEP);
-
- iommu->aiomt_intr_state = AMD_IOMMU_INTR_TABLE;
-
- /* Convert req to a power of two as required by ddi_intr_alloc */
- p2req = 0;
- while (1<<p2req <= req)
- p2req++;
- p2req--;
- req = 1<<p2req;
-
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
- cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. "
- "MSI power of 2 number of interrupts: %d,%d",
- f, driver, instance, iommu->aiomt_idx, p2req, req);
- }
-
- err = ddi_intr_alloc(iommu->aiomt_dip, iommu->aiomt_intr_htable,
- DDI_INTR_TYPE_MSI, 0, req, &actual, DDI_INTR_ALLOC_STRICT);
- if (err != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: "
- "ddi_intr_alloc failed: err = %d",
- f, driver, instance, iommu->aiomt_idx, err);
- amd_iommu_teardown_interrupts(iommu);
- return (DDI_FAILURE);
- }
-
- iommu->aiomt_actual_intrs = actual;
- iommu->aiomt_intr_state = AMD_IOMMU_INTR_ALLOCED;
-
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
- cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d. "
- "number of interrupts actually allocated %d",
- f, driver, instance, iommu->aiomt_idx, actual);
- }
-
- if (iommu->aiomt_actual_intrs < req) {
- cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: "
- "ddi_intr_alloc failed: actual (%d) < req (%d)",
- f, driver, instance, iommu->aiomt_idx,
- iommu->aiomt_actual_intrs, req);
- amd_iommu_teardown_interrupts(iommu);
- return (DDI_FAILURE);
- }
-
- for (i = 0; i < iommu->aiomt_actual_intrs; i++) {
- if (ddi_intr_add_handler(iommu->aiomt_intr_htable[i],
- amd_iommu_intr_handler, (void *)iommu, NULL)
- != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: "
- "ddi_intr_add_handler failed: intr = %d, err = %d",
- f, driver, instance, iommu->aiomt_idx, i, err);
- for (j = 0; j < i; j++) {
- (void) ddi_intr_remove_handler(
- iommu->aiomt_intr_htable[j]);
- }
- amd_iommu_teardown_interrupts(iommu);
- return (DDI_FAILURE);
- }
- }
- iommu->aiomt_intr_state = AMD_IOMMU_INTR_HANDLER;
-
- intrcap0 = intrcapN = -1;
- if (ddi_intr_get_cap(iommu->aiomt_intr_htable[0], &intrcap0)
- != DDI_SUCCESS ||
- ddi_intr_get_cap(
- iommu->aiomt_intr_htable[iommu->aiomt_actual_intrs - 1], &intrcapN)
- != DDI_SUCCESS || intrcap0 != intrcapN) {
- cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: "
- "ddi_intr_get_cap failed or inconsistent cap among "
- "interrupts: intrcap0 (%d) < intrcapN (%d)",
- f, driver, instance, iommu->aiomt_idx, intrcap0, intrcapN);
- amd_iommu_teardown_interrupts(iommu);
- return (DDI_FAILURE);
- }
- iommu->aiomt_intr_cap = intrcap0;
-
- if (intrcap0 & DDI_INTR_FLAG_BLOCK) {
- /* Need to call block enable */
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
- cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d: "
- "Need to call block enable",
- f, driver, instance, iommu->aiomt_idx);
- }
- if (ddi_intr_block_enable(iommu->aiomt_intr_htable,
- iommu->aiomt_actual_intrs) != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: "
- "ddi_intr_block enable failed ", f, driver,
- instance, iommu->aiomt_idx);
- (void) ddi_intr_block_disable(iommu->aiomt_intr_htable,
- iommu->aiomt_actual_intrs);
- amd_iommu_teardown_interrupts(iommu);
- return (DDI_FAILURE);
- }
- } else {
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
- cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d: "
- "Need to call individual enable",
- f, driver, instance, iommu->aiomt_idx);
- }
- for (i = 0; i < iommu->aiomt_actual_intrs; i++) {
- if (ddi_intr_enable(iommu->aiomt_intr_htable[i])
- != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: AMD IOMMU idx=%d: "
- "ddi_intr_enable failed: intr = %d", f,
- driver, instance, iommu->aiomt_idx, i);
- for (j = 0; j < i; j++) {
- (void) ddi_intr_disable(
- iommu->aiomt_intr_htable[j]);
- }
- amd_iommu_teardown_interrupts(iommu);
- return (DDI_FAILURE);
- }
- }
- }
- iommu->aiomt_intr_state = AMD_IOMMU_INTR_ENABLED;
-
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_INTR) {
- cmn_err(CE_NOTE, "%s: %s%d: AMD IOMMU idx=%d: "
- "Interrupts successfully %s enabled. # of interrupts = %d",
- f, driver, instance, iommu->aiomt_idx,
- (intrcap0 & DDI_INTR_FLAG_BLOCK) ? "(block)" :
- "(individually)", iommu->aiomt_actual_intrs);
- }
-
- return (DDI_SUCCESS);
-}
-
-static void
-amd_iommu_teardown_interrupts(amd_iommu_t *iommu)
-{
- int i;
-
- if (iommu->aiomt_intr_state & AMD_IOMMU_INTR_ENABLED) {
- if (iommu->aiomt_intr_cap & DDI_INTR_FLAG_BLOCK) {
- (void) ddi_intr_block_disable(iommu->aiomt_intr_htable,
- iommu->aiomt_actual_intrs);
- } else {
- for (i = 0; i < iommu->aiomt_actual_intrs; i++) {
- (void) ddi_intr_disable(
- iommu->aiomt_intr_htable[i]);
- }
- }
- }
-
- if (iommu->aiomt_intr_state & AMD_IOMMU_INTR_HANDLER) {
- for (i = 0; i < iommu->aiomt_actual_intrs; i++) {
- (void) ddi_intr_remove_handler(
- iommu->aiomt_intr_htable[i]);
- }
- }
-
- if (iommu->aiomt_intr_state & AMD_IOMMU_INTR_ALLOCED) {
- for (i = 0; i < iommu->aiomt_actual_intrs; i++) {
- (void) ddi_intr_free(iommu->aiomt_intr_htable[i]);
- }
- }
- if (iommu->aiomt_intr_state & AMD_IOMMU_INTR_TABLE) {
- kmem_free(iommu->aiomt_intr_htable,
- iommu->aiomt_intr_htable_sz);
- }
- iommu->aiomt_intr_htable = NULL;
- iommu->aiomt_intr_htable_sz = 0;
- iommu->aiomt_intr_state = AMD_IOMMU_INTR_INVALID;
-}
-
-static amd_iommu_t *
-amd_iommu_init(dev_info_t *dip, ddi_acc_handle_t handle, int idx,
- uint16_t cap_base)
-{
- amd_iommu_t *iommu;
- int instance = ddi_get_instance(dip);
- const char *driver = ddi_driver_name(dip);
- uint32_t caphdr;
- uint32_t low_addr32;
- uint32_t hi_addr32;
- uint32_t range;
- uint32_t misc;
- uint64_t pgoffset;
- amd_iommu_acpi_global_t *global;
- amd_iommu_acpi_ivhd_t *hinfop;
- const char *f = "amd_iommu_init";
-
- global = amd_iommu_lookup_acpi_global();
- hinfop = amd_iommu_lookup_any_ivhd();
-
- low_addr32 = PCI_CAP_GET32(handle, 0, cap_base,
- AMD_IOMMU_CAP_ADDR_LOW_OFF);
- if (!(low_addr32 & AMD_IOMMU_REG_ADDR_LOCKED)) {
- cmn_err(CE_WARN, "%s: %s%d: capability registers not locked. "
- "Unable to use IOMMU unit idx=%d - skipping ...", f, driver,
- instance, idx);
- return (NULL);
- }
-
- iommu = kmem_zalloc(sizeof (amd_iommu_t), KM_SLEEP);
- mutex_init(&iommu->aiomt_mutex, NULL, MUTEX_DRIVER, NULL);
- mutex_enter(&iommu->aiomt_mutex);
-
- mutex_init(&iommu->aiomt_cmdlock, NULL, MUTEX_DRIVER, NULL);
- mutex_init(&iommu->aiomt_eventlock, NULL, MUTEX_DRIVER, NULL);
-
- iommu->aiomt_dip = dip;
- iommu->aiomt_idx = idx;
-
- /*
- * Since everything in the capability block is locked and RO at this
- * point, copy everything into the IOMMU struct
- */
-
- /* Get cap header */
- caphdr = PCI_CAP_GET32(handle, 0, cap_base, AMD_IOMMU_CAP_HDR_OFF);
- iommu->aiomt_cap_hdr = caphdr;
- iommu->aiomt_npcache = AMD_IOMMU_REG_GET32(&caphdr,
- AMD_IOMMU_CAP_NPCACHE);
- iommu->aiomt_httun = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_HTTUN);
-
- if (hinfop)
- iommu->aiomt_iotlb = hinfop->ach_IotlbSup;
- else
- iommu->aiomt_iotlb =
- AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_IOTLB);
-
- iommu->aiomt_captype = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_TYPE);
- iommu->aiomt_capid = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_ID);
-
- /*
- * Get address of IOMMU control registers
- */
- hi_addr32 = PCI_CAP_GET32(handle, 0, cap_base,
- AMD_IOMMU_CAP_ADDR_HI_OFF);
- iommu->aiomt_low_addr32 = low_addr32;
- iommu->aiomt_hi_addr32 = hi_addr32;
- low_addr32 &= ~AMD_IOMMU_REG_ADDR_LOCKED;
-
- if (hinfop) {
- iommu->aiomt_reg_pa = hinfop->ach_IOMMU_reg_base;
- ASSERT(hinfop->ach_IOMMU_pci_seg == 0);
- } else {
- iommu->aiomt_reg_pa = ((uint64_t)hi_addr32 << 32 | low_addr32);
- }
-
- /*
- * Get cap range reg
- */
- range = PCI_CAP_GET32(handle, 0, cap_base, AMD_IOMMU_CAP_RANGE_OFF);
- iommu->aiomt_range = range;
- iommu->aiomt_rng_valid = AMD_IOMMU_REG_GET32(&range,
- AMD_IOMMU_RNG_VALID);
- if (iommu->aiomt_rng_valid) {
- iommu->aiomt_rng_bus = AMD_IOMMU_REG_GET32(&range,
- AMD_IOMMU_RNG_BUS);
- iommu->aiomt_first_devfn = AMD_IOMMU_REG_GET32(&range,
- AMD_IOMMU_FIRST_DEVFN);
- iommu->aiomt_last_devfn = AMD_IOMMU_REG_GET32(&range,
- AMD_IOMMU_LAST_DEVFN);
- } else {
- iommu->aiomt_rng_bus = 0;
- iommu->aiomt_first_devfn = 0;
- iommu->aiomt_last_devfn = 0;
- }
-
- if (hinfop)
- iommu->aiomt_ht_unitid = hinfop->ach_IOMMU_UnitID;
- else
- iommu->aiomt_ht_unitid = AMD_IOMMU_REG_GET32(&range,
- AMD_IOMMU_HT_UNITID);
-
- /*
- * Get cap misc reg
- */
- misc = PCI_CAP_GET32(handle, 0, cap_base, AMD_IOMMU_CAP_MISC_OFF);
- iommu->aiomt_misc = misc;
-
- if (global) {
- iommu->aiomt_htatsresv = global->acg_HtAtsResv;
- iommu->aiomt_vasize = global->acg_VAsize;
- iommu->aiomt_pasize = global->acg_PAsize;
- } else {
- iommu->aiomt_htatsresv = AMD_IOMMU_REG_GET32(&misc,
- AMD_IOMMU_HT_ATSRSV);
- iommu->aiomt_vasize = AMD_IOMMU_REG_GET32(&misc,
- AMD_IOMMU_VA_SIZE);
- iommu->aiomt_pasize = AMD_IOMMU_REG_GET32(&misc,
- AMD_IOMMU_PA_SIZE);
- }
-
- if (hinfop) {
- iommu->aiomt_msinum = hinfop->ach_IOMMU_MSInum;
- } else {
- iommu->aiomt_msinum =
- AMD_IOMMU_REG_GET32(&misc, AMD_IOMMU_MSINUM);
- }
-
- /*
- * Set up mapping between control registers PA and VA
- */
- pgoffset = iommu->aiomt_reg_pa & MMU_PAGEOFFSET;
- ASSERT(pgoffset == 0);
- iommu->aiomt_reg_pages = mmu_btopr(AMD_IOMMU_REG_SIZE + pgoffset);
- iommu->aiomt_reg_size = mmu_ptob(iommu->aiomt_reg_pages);
-
- iommu->aiomt_va = (uintptr_t)device_arena_alloc(
- ptob(iommu->aiomt_reg_pages), VM_SLEEP);
- if (iommu->aiomt_va == 0) {
- cmn_err(CE_WARN, "%s: %s%d: Failed to alloc VA for IOMMU "
- "control regs. Skipping IOMMU idx=%d", f, driver,
- instance, idx);
- mutex_exit(&iommu->aiomt_mutex);
- (void) amd_iommu_fini(iommu);
- return (NULL);
- }
-
- hat_devload(kas.a_hat, (void *)(uintptr_t)iommu->aiomt_va,
- iommu->aiomt_reg_size,
- mmu_btop(iommu->aiomt_reg_pa), PROT_READ | PROT_WRITE
- | HAT_STRICTORDER, HAT_LOAD_LOCK);
-
- iommu->aiomt_reg_va = iommu->aiomt_va + pgoffset;
-
- /*
- * Setup the various control register's VA
- */
- iommu->aiomt_reg_devtbl_va = iommu->aiomt_reg_va +
- AMD_IOMMU_DEVTBL_REG_OFF;
- iommu->aiomt_reg_cmdbuf_va = iommu->aiomt_reg_va +
- AMD_IOMMU_CMDBUF_REG_OFF;
- iommu->aiomt_reg_eventlog_va = iommu->aiomt_reg_va +
- AMD_IOMMU_EVENTLOG_REG_OFF;
- iommu->aiomt_reg_ctrl_va = iommu->aiomt_reg_va +
- AMD_IOMMU_CTRL_REG_OFF;
- iommu->aiomt_reg_excl_base_va = iommu->aiomt_reg_va +
- AMD_IOMMU_EXCL_BASE_REG_OFF;
- iommu->aiomt_reg_excl_lim_va = iommu->aiomt_reg_va +
- AMD_IOMMU_EXCL_LIM_REG_OFF;
- iommu->aiomt_reg_cmdbuf_head_va = iommu->aiomt_reg_va +
- AMD_IOMMU_CMDBUF_HEAD_REG_OFF;
- iommu->aiomt_reg_cmdbuf_tail_va = iommu->aiomt_reg_va +
- AMD_IOMMU_CMDBUF_TAIL_REG_OFF;
- iommu->aiomt_reg_eventlog_head_va = iommu->aiomt_reg_va +
- AMD_IOMMU_EVENTLOG_HEAD_REG_OFF;
- iommu->aiomt_reg_eventlog_tail_va = iommu->aiomt_reg_va +
- AMD_IOMMU_EVENTLOG_TAIL_REG_OFF;
- iommu->aiomt_reg_status_va = iommu->aiomt_reg_va +
- AMD_IOMMU_STATUS_REG_OFF;
-
-
- /*
- * Setup the DEVICE table, CMD buffer, and LOG buffer in
- * memory and setup DMA access to this memory location
- */
- if (amd_iommu_setup_tables_and_buffers(iommu) != DDI_SUCCESS) {
- mutex_exit(&iommu->aiomt_mutex);
- (void) amd_iommu_fini(iommu);
- return (NULL);
- }
-
- if (amd_iommu_setup_exclusion(iommu) != DDI_SUCCESS) {
- mutex_exit(&iommu->aiomt_mutex);
- (void) amd_iommu_fini(iommu);
- return (NULL);
- }
-
- amd_iommu_enable_interrupts(iommu);
-
- if (amd_iommu_setup_interrupts(iommu) != DDI_SUCCESS) {
- mutex_exit(&iommu->aiomt_mutex);
- (void) amd_iommu_fini(iommu);
- return (NULL);
- }
-
- /*
- * need to setup domain table before gfx bypass
- */
- amd_iommu_init_page_tables(iommu);
-
- /*
- * Set pass-thru for special devices like IOAPIC and HPET
- *
- * Also, gfx devices don't use DDI for DMA. No need to register
- * before setting up gfx passthru
- */
- if (amd_iommu_setup_passthru(iommu) != DDI_SUCCESS) {
- mutex_exit(&iommu->aiomt_mutex);
- (void) amd_iommu_fini(iommu);
- return (NULL);
- }
-
- if (amd_iommu_start(iommu) != DDI_SUCCESS) {
- mutex_exit(&iommu->aiomt_mutex);
- (void) amd_iommu_fini(iommu);
- return (NULL);
- }
-
- /* xxx register/start race */
- if (amd_iommu_register(iommu) != DDI_SUCCESS) {
- mutex_exit(&iommu->aiomt_mutex);
- (void) amd_iommu_fini(iommu);
- return (NULL);
- }
-
- if (amd_iommu_debug) {
- cmn_err(CE_NOTE, "%s: %s%d: IOMMU idx=%d inited.", f, driver,
- instance, idx);
- }
-
- return (iommu);
-}
-
-static int
-amd_iommu_fini(amd_iommu_t *iommu)
-{
- int idx = iommu->aiomt_idx;
- dev_info_t *dip = iommu->aiomt_dip;
- int instance = ddi_get_instance(dip);
- const char *driver = ddi_driver_name(dip);
- const char *f = "amd_iommu_fini";
-
- mutex_enter(&iommu->aiomt_mutex);
- if (amd_iommu_unregister(iommu) != DDI_SUCCESS) {
- cmn_err(CE_NOTE, "%s: %s%d: Fini of IOMMU unit failed. "
- "idx = %d", f, driver, instance, idx);
- return (DDI_FAILURE);
- }
- amd_iommu_stop(iommu);
- amd_iommu_fini_page_tables(iommu);
- amd_iommu_teardown_interrupts(iommu);
- amd_iommu_teardown_exclusion(iommu);
- amd_iommu_teardown_tables_and_buffers(iommu);
- if (iommu->aiomt_va != NULL) {
- hat_unload(kas.a_hat, (void *)(uintptr_t)iommu->aiomt_va,
- iommu->aiomt_reg_size, HAT_UNLOAD_UNLOCK);
- device_arena_free((void *)(uintptr_t)iommu->aiomt_va,
- ptob(iommu->aiomt_reg_pages));
- iommu->aiomt_va = NULL;
- iommu->aiomt_reg_va = NULL;
- }
- mutex_destroy(&iommu->aiomt_eventlock);
- mutex_destroy(&iommu->aiomt_cmdlock);
- mutex_exit(&iommu->aiomt_mutex);
- mutex_destroy(&iommu->aiomt_mutex);
- kmem_free(iommu, sizeof (amd_iommu_t));
-
- cmn_err(CE_NOTE, "%s: %s%d: Fini of IOMMU unit complete. idx = %d",
- f, driver, instance, idx);
-
- return (DDI_SUCCESS);
-}
-
-int
-amd_iommu_setup(dev_info_t *dip, amd_iommu_state_t *statep)
-{
- int instance = ddi_get_instance(dip);
- const char *driver = ddi_driver_name(dip);
- ddi_acc_handle_t handle;
- uint8_t base_class;
- uint8_t sub_class;
- uint8_t prog_class;
- int idx;
- uint32_t id;
- uint16_t cap_base;
- uint32_t caphdr;
- uint8_t cap_type;
- uint8_t cap_id;
- amd_iommu_t *iommu;
- const char *f = "amd_iommu_setup";
-
- ASSERT(instance >= 0);
- ASSERT(driver);
-
- /* First setup PCI access to config space */
-
- if (pci_config_setup(dip, &handle) != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: PCI config setup failed: %s%d",
- f, driver, instance);
- return (DDI_FAILURE);
- }
-
- /*
- * The AMD IOMMU is part of an independent PCI function. There may be
- * more than one IOMMU in that PCI function
- */
- base_class = pci_config_get8(handle, PCI_CONF_BASCLASS);
- sub_class = pci_config_get8(handle, PCI_CONF_SUBCLASS);
- prog_class = pci_config_get8(handle, PCI_CONF_PROGCLASS);
-
- if (base_class != PCI_CLASS_PERIPH || sub_class != PCI_PERIPH_IOMMU ||
- prog_class != AMD_IOMMU_PCI_PROG_IF) {
- cmn_err(CE_WARN, "%s: %s%d: invalid PCI class(0x%x)/"
- "subclass(0x%x)/programming interface(0x%x)", f, driver,
- instance, base_class, sub_class, prog_class);
- pci_config_teardown(&handle);
- return (DDI_FAILURE);
- }
-
- /*
- * Find and initialize all IOMMU units in this function
- */
- for (idx = 0; ; idx++) {
- if (pci_cap_probe(handle, idx, &id, &cap_base) != DDI_SUCCESS)
- break;
-
- /* check if cap ID is secure device cap id */
- if (id != PCI_CAP_ID_SECURE_DEV) {
- if (amd_iommu_debug) {
- cmn_err(CE_WARN,
- "%s: %s%d: skipping IOMMU: idx(0x%x) "
- "cap ID (0x%x) != secure dev capid (0x%x)",
- f, driver, instance, idx, id,
- PCI_CAP_ID_SECURE_DEV);
- }
- continue;
- }
-
- /* check if cap type is IOMMU cap type */
- caphdr = PCI_CAP_GET32(handle, 0, cap_base,
- AMD_IOMMU_CAP_HDR_OFF);
- cap_type = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_TYPE);
- cap_id = AMD_IOMMU_REG_GET32(&caphdr, AMD_IOMMU_CAP_ID);
-
- if (cap_type != AMD_IOMMU_CAP) {
- cmn_err(CE_WARN, "%s: %s%d: skipping IOMMU: idx(0x%x) "
- "cap type (0x%x) != AMD IOMMU CAP (0x%x)", f,
- driver, instance, idx, cap_type, AMD_IOMMU_CAP);
- continue;
- }
- ASSERT(cap_id == PCI_CAP_ID_SECURE_DEV);
- ASSERT(cap_id == id);
-
- iommu = amd_iommu_init(dip, handle, idx, cap_base);
- if (iommu == NULL) {
- cmn_err(CE_WARN, "%s: %s%d: skipping IOMMU: idx(0x%x) "
- "failed to init IOMMU", f,
- driver, instance, idx);
- continue;
- }
-
- if (statep->aioms_iommu_start == NULL) {
- statep->aioms_iommu_start = iommu;
- } else {
- statep->aioms_iommu_end->aiomt_next = iommu;
- }
- statep->aioms_iommu_end = iommu;
-
- statep->aioms_nunits++;
- }
-
- pci_config_teardown(&handle);
-
- if (amd_iommu_debug) {
- cmn_err(CE_NOTE, "%s: %s%d: state=%p: setup %d IOMMU units",
- f, driver, instance, (void *)statep, statep->aioms_nunits);
- }
-
- return (DDI_SUCCESS);
-}
-
-int
-amd_iommu_teardown(dev_info_t *dip, amd_iommu_state_t *statep)
-{
- int instance = ddi_get_instance(dip);
- const char *driver = ddi_driver_name(dip);
- amd_iommu_t *iommu;
- int teardown;
- int error = DDI_SUCCESS;
- const char *f = "amd_iommu_teardown";
-
- teardown = 0;
- for (iommu = statep->aioms_iommu_start; iommu;
- iommu = iommu->aiomt_next) {
- ASSERT(statep->aioms_nunits > 0);
- if (amd_iommu_fini(iommu) != DDI_SUCCESS) {
- error = DDI_FAILURE;
- continue;
- }
- statep->aioms_nunits--;
- teardown++;
- }
-
- cmn_err(CE_NOTE, "%s: %s%d: state=%p: toredown %d units. "
- "%d units left", f, driver, instance, (void *)statep,
- teardown, statep->aioms_nunits);
-
- return (error);
-}
-
-/* Interface with IOMMULIB */
-/*ARGSUSED*/
-static int
-amd_iommu_probe(iommulib_handle_t handle, dev_info_t *rdip)
-{
- const char *driver = ddi_driver_name(rdip);
- char *s;
- amd_iommu_t *iommu = iommulib_iommu_getdata(handle);
-
- if (amd_iommu_disable_list) {
- s = strstr(amd_iommu_disable_list, driver);
- if (s == NULL)
- return (DDI_SUCCESS);
- if (s == amd_iommu_disable_list || *(s - 1) == ':') {
- s += strlen(driver);
- if (*s == '\0' || *s == ':') {
- amd_iommu_set_passthru(iommu, rdip);
- return (DDI_FAILURE);
- }
- }
- }
-
- return (DDI_SUCCESS);
-}
-
-/*ARGSUSED*/
-static int
-amd_iommu_allochdl(iommulib_handle_t handle,
- dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
- int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *dma_handlep)
-{
- return (iommulib_iommu_dma_allochdl(dip, rdip, attr, waitfp,
- arg, dma_handlep));
-}
-
-/*ARGSUSED*/
-static int
-amd_iommu_freehdl(iommulib_handle_t handle,
- dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle)
-{
- return (iommulib_iommu_dma_freehdl(dip, rdip, dma_handle));
-}
-
-/*ARGSUSED*/
-static int
-map_current_window(amd_iommu_t *iommu, dev_info_t *rdip, ddi_dma_attr_t *attrp,
- struct ddi_dma_req *dmareq, ddi_dma_cookie_t *cookie_array, uint_t ccount,
- int km_flags)
-{
- const char *driver = ddi_driver_name(iommu->aiomt_dip);
- int instance = ddi_get_instance(iommu->aiomt_dip);
- int idx = iommu->aiomt_idx;
- int i;
- uint64_t start_va;
- char *path;
- int error = DDI_FAILURE;
- const char *f = "map_current_window";
-
- path = kmem_alloc(MAXPATHLEN, km_flags);
- if (path == NULL) {
- return (DDI_DMA_NORESOURCES);
- }
-
- (void) ddi_pathname(rdip, path);
- mutex_enter(&amd_iommu_pgtable_lock);
-
- if (amd_iommu_debug == AMD_IOMMU_DEBUG_PAGE_TABLES) {
- cmn_err(CE_WARN, "%s: %s%d: idx=%d Attempting to get cookies "
- "from handle for device %s",
- f, driver, instance, idx, path);
- }
-
- start_va = 0;
- for (i = 0; i < ccount; i++) {
- if ((error = amd_iommu_map_pa2va(iommu, rdip, attrp, dmareq,
- cookie_array[i].dmac_cookie_addr,
- cookie_array[i].dmac_size,
- AMD_IOMMU_VMEM_MAP, &start_va, km_flags)) != DDI_SUCCESS) {
- break;
- }
- cookie_array[i].dmac_cookie_addr = (uintptr_t)start_va;
- cookie_array[i].dmac_type = 0;
- }
-
- if (i != ccount) {
- cmn_err(CE_WARN, "%s: %s%d: idx=%d Cannot map cookie# %d "
- "for device %s", f, driver, instance, idx, i, path);
- (void) unmap_current_window(iommu, rdip, cookie_array,
- ccount, i, 1);
- goto out;
- }
-
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
- cmn_err(CE_NOTE, "%s: return SUCCESS", f);
- }
-
- error = DDI_DMA_MAPPED;
-out:
- mutex_exit(&amd_iommu_pgtable_lock);
- kmem_free(path, MAXPATHLEN);
- return (error);
-}
-
-/*ARGSUSED*/
-static int
-unmap_current_window(amd_iommu_t *iommu, dev_info_t *rdip,
- ddi_dma_cookie_t *cookie_array, uint_t ccount, int ncookies, int locked)
-{
- const char *driver = ddi_driver_name(iommu->aiomt_dip);
- int instance = ddi_get_instance(iommu->aiomt_dip);
- int idx = iommu->aiomt_idx;
- int i;
- int error = DDI_FAILURE;
- char *path;
- int pathfree;
- const char *f = "unmap_current_window";
-
- if (!locked)
- mutex_enter(&amd_iommu_pgtable_lock);
-
- path = kmem_alloc(MAXPATHLEN, KM_NOSLEEP);
- if (path) {
- (void) ddi_pathname(rdip, path);
- pathfree = 1;
- } else {
- path = "<path-mem-alloc-failed>";
- pathfree = 0;
- }
-
- if (ncookies == -1)
- ncookies = ccount;
-
- for (i = 0; i < ncookies; i++) {
- if (amd_iommu_unmap_va(iommu, rdip,
- cookie_array[i].dmac_cookie_addr,
- cookie_array[i].dmac_size,
- AMD_IOMMU_VMEM_MAP) != DDI_SUCCESS) {
- break;
- }
- }
-
- if (amd_iommu_cmd(iommu, AMD_IOMMU_CMD_COMPL_WAIT, NULL, 0, 0)
- != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: AMD IOMMU completion wait failed for: %s",
- f, path);
- }
-
- if (i != ncookies) {
- cmn_err(CE_WARN, "%s: %s%d: idx=%d Cannot unmap cookie# %d "
- "for device %s", f, driver, instance, idx, i, path);
- error = DDI_FAILURE;
- goto out;
- }
-
- error = DDI_SUCCESS;
-
-out:
- if (pathfree)
- kmem_free(path, MAXPATHLEN);
- if (!locked)
- mutex_exit(&amd_iommu_pgtable_lock);
- return (error);
-}
-
-/*ARGSUSED*/
-static int
-amd_iommu_bindhdl(iommulib_handle_t handle, dev_info_t *dip,
- dev_info_t *rdip, ddi_dma_handle_t dma_handle,
- struct ddi_dma_req *dmareq, ddi_dma_cookie_t *cookiep,
- uint_t *ccountp)
-{
- int dma_error = DDI_DMA_NOMAPPING;
- int error;
- char *path;
- ddi_dma_cookie_t *cookie_array = NULL;
- uint_t ccount = 0;
- ddi_dma_impl_t *hp;
- ddi_dma_attr_t *attrp;
- int km_flags;
- amd_iommu_t *iommu = iommulib_iommu_getdata(handle);
- int instance = ddi_get_instance(rdip);
- const char *driver = ddi_driver_name(rdip);
- const char *f = "amd_iommu_bindhdl";
-
- dma_error = iommulib_iommu_dma_bindhdl(dip, rdip, dma_handle,
- dmareq, cookiep, ccountp);
-
- if (dma_error != DDI_DMA_MAPPED && dma_error != DDI_DMA_PARTIAL_MAP)
- return (dma_error);
-
- km_flags = iommulib_iommu_dma_get_sleep_flags(dip, dma_handle);
-
- path = kmem_alloc(MAXPATHLEN, km_flags);
- if (path) {
- (void) ddi_pathname(rdip, path);
- } else {
- dma_error = DDI_DMA_NORESOURCES;
- goto unbind;
- }
-
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_BIND) {
- cmn_err(CE_NOTE, "%s: %s got cookie (%p), #cookies: %d",
- f, path,
- (void *)cookiep->dmac_cookie_addr,
- *ccountp);
- }
-
- cookie_array = NULL;
- ccount = 0;
- if ((error = iommulib_iommu_dma_get_cookies(dip, dma_handle,
- &cookie_array, &ccount)) != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: Cannot get cookies "
- "for device %s", f, driver, instance, path);
- dma_error = error;
- goto unbind;
- }
-
- hp = (ddi_dma_impl_t *)dma_handle;
- attrp = &hp->dmai_attr;
-
- error = map_current_window(iommu, rdip, attrp, dmareq,
- cookie_array, ccount, km_flags);
- if (error != DDI_SUCCESS) {
- dma_error = error;
- goto unbind;
- }
-
- if ((error = iommulib_iommu_dma_set_cookies(dip, dma_handle,
- cookie_array, ccount)) != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: Cannot set cookies "
- "for device %s", f, driver, instance, path);
- dma_error = error;
- goto unbind;
- }
-
- *cookiep = cookie_array[0];
-
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_BIND) {
- cmn_err(CE_NOTE, "%s: %s remapped cookie (%p), #cookies: %d",
- f, path,
- (void *)(uintptr_t)cookiep->dmac_cookie_addr,
- *ccountp);
- }
-
- kmem_free(path, MAXPATHLEN);
- ASSERT(dma_error == DDI_DMA_MAPPED || dma_error == DDI_DMA_PARTIAL_MAP);
- return (dma_error);
-unbind:
- kmem_free(path, MAXPATHLEN);
- (void) iommulib_iommu_dma_unbindhdl(dip, rdip, dma_handle);
- return (dma_error);
-}
-
-/*ARGSUSED*/
-static int
-amd_iommu_unbindhdl(iommulib_handle_t handle,
- dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle)
-{
- amd_iommu_t *iommu = iommulib_iommu_getdata(handle);
- ddi_dma_cookie_t *cookie_array = NULL;
- uint_t ccount = 0;
- int error = DDI_FAILURE;
- int instance = ddi_get_instance(rdip);
- const char *driver = ddi_driver_name(rdip);
- const char *f = "amd_iommu_unbindhdl";
-
- cookie_array = NULL;
- ccount = 0;
- if (iommulib_iommu_dma_get_cookies(dip, dma_handle, &cookie_array,
- &ccount) != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: Cannot get cookies "
- "for device %p", f, driver, instance, (void *)rdip);
- error = DDI_FAILURE;
- goto out;
- }
-
- if (iommulib_iommu_dma_clear_cookies(dip, dma_handle) != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: Cannot clear cookies "
- "for device %p", f, driver, instance, (void *)rdip);
- error = DDI_FAILURE;
- goto out;
- }
-
- if (iommulib_iommu_dma_unbindhdl(dip, rdip, dma_handle)
- != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: failed to unbindhdl for dip=%p",
- f, driver, instance, (void *)rdip);
- error = DDI_FAILURE;
- goto out;
- }
-
- if (unmap_current_window(iommu, rdip, cookie_array, ccount, -1, 0)
- != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: failed to unmap current window "
- "for dip=%p", f, driver, instance, (void *)rdip);
- error = DDI_FAILURE;
- } else {
- error = DDI_SUCCESS;
- }
-out:
- if (cookie_array)
- kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
- return (error);
-}
-
-/*ARGSUSED*/
-static int
-amd_iommu_sync(iommulib_handle_t handle, dev_info_t *dip,
- dev_info_t *rdip, ddi_dma_handle_t dma_handle, off_t off,
- size_t len, uint_t cache_flags)
-{
- ddi_dma_cookie_t *cookie_array = NULL;
- uint_t ccount = 0;
- int error;
- const char *f = "amd_iommu_sync";
-
- cookie_array = NULL;
- ccount = 0;
- if (iommulib_iommu_dma_get_cookies(dip, dma_handle, &cookie_array,
- &ccount) != DDI_SUCCESS) {
- ASSERT(cookie_array == NULL);
- cmn_err(CE_WARN, "%s: Cannot get cookies "
- "for device %p", f, (void *)rdip);
- error = DDI_FAILURE;
- goto out;
- }
-
- if (iommulib_iommu_dma_clear_cookies(dip, dma_handle) != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: Cannot clear cookies "
- "for device %p", f, (void *)rdip);
- error = DDI_FAILURE;
- goto out;
- }
-
- error = iommulib_iommu_dma_sync(dip, rdip, dma_handle, off,
- len, cache_flags);
-
- if (iommulib_iommu_dma_set_cookies(dip, dma_handle, cookie_array,
- ccount) != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: Cannot set cookies "
- "for device %p", f, (void *)rdip);
- error = DDI_FAILURE;
- } else {
- cookie_array = NULL;
- ccount = 0;
- }
-
-out:
- if (cookie_array)
- kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
- return (error);
-}
-
-/*ARGSUSED*/
-static int
-amd_iommu_win(iommulib_handle_t handle, dev_info_t *dip,
- dev_info_t *rdip, ddi_dma_handle_t dma_handle, uint_t win,
- off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
- uint_t *ccountp)
-{
- int error = DDI_FAILURE;
- amd_iommu_t *iommu = iommulib_iommu_getdata(handle);
- ddi_dma_cookie_t *cookie_array = NULL;
- uint_t ccount = 0;
- int km_flags;
- ddi_dma_impl_t *hp;
- ddi_dma_attr_t *attrp;
- struct ddi_dma_req sdmareq = {0};
- int instance = ddi_get_instance(rdip);
- const char *driver = ddi_driver_name(rdip);
- const char *f = "amd_iommu_win";
-
- km_flags = iommulib_iommu_dma_get_sleep_flags(dip, dma_handle);
-
- cookie_array = NULL;
- ccount = 0;
- if (iommulib_iommu_dma_get_cookies(dip, dma_handle, &cookie_array,
- &ccount) != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: Cannot get cookies "
- "for device %p", f, driver, instance, (void *)rdip);
- error = DDI_FAILURE;
- goto out;
- }
-
- if (iommulib_iommu_dma_clear_cookies(dip, dma_handle) != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: Cannot clear cookies "
- "for device %p", f, driver, instance, (void *)rdip);
- error = DDI_FAILURE;
- goto out;
- }
-
- if (iommulib_iommu_dma_win(dip, rdip, dma_handle, win,
- offp, lenp, cookiep, ccountp) != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: failed switch windows for dip=%p",
- f, driver, instance, (void *)rdip);
- error = DDI_FAILURE;
- goto out;
- }
-
- (void) unmap_current_window(iommu, rdip, cookie_array, ccount, -1, 0);
-
- if (cookie_array) {
- kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
- cookie_array = NULL;
- ccount = 0;
- }
-
- cookie_array = NULL;
- ccount = 0;
- if (iommulib_iommu_dma_get_cookies(dip, dma_handle, &cookie_array,
- &ccount) != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: Cannot get cookies "
- "for device %p", f, driver, instance, (void *)rdip);
- error = DDI_FAILURE;
- goto out;
- }
-
- hp = (ddi_dma_impl_t *)dma_handle;
- attrp = &hp->dmai_attr;
-
- sdmareq.dmar_flags = DDI_DMA_RDWR;
- error = map_current_window(iommu, rdip, attrp, &sdmareq,
- cookie_array, ccount, km_flags);
-
- if (iommulib_iommu_dma_set_cookies(dip, dma_handle, cookie_array,
- ccount) != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: Cannot set cookies "
- "for device %p", f, driver, instance, (void *)rdip);
- error = DDI_FAILURE;
- goto out;
- }
-
- *cookiep = cookie_array[0];
-
- return (error == DDI_SUCCESS ? DDI_SUCCESS : DDI_FAILURE);
-out:
- if (cookie_array)
- kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
-
- return (error);
-}
-
-/* Obsoleted DMA routines */
-
-/*ARGSUSED*/
-static int
-amd_iommu_map(iommulib_handle_t handle, dev_info_t *dip,
- dev_info_t *rdip, struct ddi_dma_req *dmareq,
- ddi_dma_handle_t *dma_handle)
-{
- ASSERT(0);
- return (iommulib_iommu_dma_map(dip, rdip, dmareq, dma_handle));
-}
-
-/*ARGSUSED*/
-static int
-amd_iommu_mctl(iommulib_handle_t handle, dev_info_t *dip,
- dev_info_t *rdip, ddi_dma_handle_t dma_handle,
- enum ddi_dma_ctlops request, off_t *offp, size_t *lenp,
- caddr_t *objpp, uint_t cache_flags)
-{
- ASSERT(0);
- return (iommulib_iommu_dma_mctl(dip, rdip, dma_handle,
- request, offp, lenp, objpp, cache_flags));
-}
-
-uint64_t
-amd_iommu_reg_get64_workaround(uint64_t *regp, uint32_t bits)
-{
- split_t s;
- uint32_t *ptr32 = (uint32_t *)regp;
- uint64_t *s64p = &(s.u64);
-
- s.u32[0] = ptr32[0];
- s.u32[1] = ptr32[1];
-
- return (AMD_IOMMU_REG_GET64_IMPL(s64p, bits));
-}
-
-uint64_t
-amd_iommu_reg_set64_workaround(uint64_t *regp, uint32_t bits, uint64_t value)
-{
- split_t s;
- uint32_t *ptr32 = (uint32_t *)regp;
- uint64_t *s64p = &(s.u64);
-
- s.u32[0] = ptr32[0];
- s.u32[1] = ptr32[1];
-
- AMD_IOMMU_REG_SET64_IMPL(s64p, bits, value);
-
- *regp = s.u64;
-
- return (s.u64);
-}
-
-void
-amd_iommu_read_boot_props(void)
-{
- char *propval;
-
- /*
- * if "amd-iommu = no/false" boot property is set,
- * ignore AMD iommu
- */
- if (ddi_prop_lookup_string(DDI_DEV_T_ANY, ddi_root_node(),
- DDI_PROP_DONTPASS, "amd-iommu", &propval) == DDI_SUCCESS) {
- if (strcmp(propval, "no") == 0 ||
- strcmp(propval, "false") == 0) {
- amd_iommu_disable = 1;
- }
- ddi_prop_free(propval);
- }
-
- /*
- * Copy the list of drivers for which IOMMU is disabled by user.
- */
- if (ddi_prop_lookup_string(DDI_DEV_T_ANY, ddi_root_node(),
- DDI_PROP_DONTPASS, "amd-iommu-disable-list", &propval)
- == DDI_SUCCESS) {
- amd_iommu_disable_list = kmem_alloc(strlen(propval) + 1,
- KM_SLEEP);
- (void) strcpy(amd_iommu_disable_list, propval);
- ddi_prop_free(propval);
- }
-
-}
-
-void
-amd_iommu_lookup_conf_props(dev_info_t *dip)
-{
- char *disable;
-
- if (ddi_prop_lookup_string(DDI_DEV_T_ANY, dip,
- DDI_PROP_DONTPASS|DDI_PROP_NOTPROM, "amd-iommu", &disable)
- == DDI_PROP_SUCCESS) {
- if (strcmp(disable, "no") == 0) {
- amd_iommu_disable = 1;
- }
- ddi_prop_free(disable);
- }
-
- if (ddi_prop_lookup_string(DDI_DEV_T_ANY, dip,
- DDI_PROP_DONTPASS|DDI_PROP_NOTPROM, "amd-iommu-disable-list",
- &disable) == DDI_PROP_SUCCESS) {
- amd_iommu_disable_list = kmem_alloc(strlen(disable) + 1,
- KM_SLEEP);
- (void) strcpy(amd_iommu_disable_list, disable);
- ddi_prop_free(disable);
- }
-}
--- a/usr/src/uts/i86pc/io/amd_iommu/amd_iommu_page_tables.c Thu Sep 17 10:25:33 2009 +0800
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,1699 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-
-/*
- * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
- * Use is subject to license terms.
- */
-
-#include <sys/sunddi.h>
-#include <sys/sunndi.h>
-#include <sys/acpi/acpi.h>
-#include <sys/acpica.h>
-#include <sys/amd_iommu.h>
-#include <sys/bootconf.h>
-#include <sys/sysmacros.h>
-#include <sys/ddidmareq.h>
-
-#include "amd_iommu_impl.h"
-#include "amd_iommu_acpi.h"
-#include "amd_iommu_page_tables.h"
-
-ddi_dma_attr_t amd_iommu_pgtable_dma_attr = {
- DMA_ATTR_V0,
- 0U, /* dma_attr_addr_lo */
- 0xffffffffffffffffULL, /* dma_attr_addr_hi */
- 0xffffffffU, /* dma_attr_count_max */
- (uint64_t)4096, /* dma_attr_align */
- 1, /* dma_attr_burstsizes */
- 64, /* dma_attr_minxfer */
- 0xffffffffU, /* dma_attr_maxxfer */
- 0xffffffffU, /* dma_attr_seg */
- 1, /* dma_attr_sgllen, variable */
- 64, /* dma_attr_granular */
- 0 /* dma_attr_flags */
-};
-
-static amd_iommu_domain_t **amd_iommu_domain_table;
-
-static struct {
- int f_count;
- amd_iommu_page_table_t *f_list;
-} amd_iommu_pgtable_freelist;
-int amd_iommu_no_pgtable_freelist;
-
-/*ARGSUSED*/
-static int
-amd_iommu_get_src_bdf(amd_iommu_t *iommu, int32_t bdf, int32_t *src_bdfp)
-{
- amd_iommu_acpi_ivhd_t *hinfop;
-
- hinfop = amd_iommu_lookup_ivhd(bdf);
- if (hinfop == NULL || hinfop->ach_src_deviceid == -1)
- *src_bdfp = bdf;
- else
- *src_bdfp = hinfop->ach_src_deviceid;
-
- return (DDI_SUCCESS);
-}
-
-static dev_info_t *
-amd_iommu_pci_dip(dev_info_t *rdip, const char *path)
-{
- dev_info_t *pdip;
- const char *driver = ddi_driver_name(rdip);
- int instance = ddi_get_instance(rdip);
- const char *f = "amd_iommu_pci_dip";
-
- /* Hold rdip so it and its parents don't go away */
- ndi_hold_devi(rdip);
-
- if (ddi_is_pci_dip(rdip))
- return (rdip);
-
- pdip = rdip;
- while (pdip = ddi_get_parent(pdip)) {
- if (ddi_is_pci_dip(pdip)) {
- ndi_hold_devi(pdip);
- ndi_rele_devi(rdip);
- return (pdip);
- }
- }
-
- cmn_err(CE_WARN, "%s: %s%d dip = %p has no PCI parent, path = %s",
- f, driver, instance, (void *)rdip, path);
-
- ndi_rele_devi(rdip);
-
- ASSERT(0);
-
- return (NULL);
-}
-
-/*ARGSUSED*/
-static int
-amd_iommu_get_domain(amd_iommu_t *iommu, dev_info_t *rdip, int alias,
- uint16_t deviceid, domain_id_t *domainid, const char *path)
-{
- const char *f = "amd_iommu_get_domain";
-
- *domainid = AMD_IOMMU_INVALID_DOMAIN;
-
- ASSERT(strcmp(ddi_driver_name(rdip), "agpgart") != 0);
-
- switch (deviceid) {
- case AMD_IOMMU_INVALID_DOMAIN:
- case AMD_IOMMU_IDENTITY_DOMAIN:
- case AMD_IOMMU_PASSTHRU_DOMAIN:
- case AMD_IOMMU_SYS_DOMAIN:
- *domainid = AMD_IOMMU_SYS_DOMAIN;
- break;
- default:
- *domainid = deviceid;
- break;
- }
-
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
- cmn_err(CE_NOTE, "%s: domainid for %s = %d",
- f, path, *domainid);
- }
-
- return (DDI_SUCCESS);
-}
-
-static uint16_t
-hash_domain(domain_id_t domainid)
-{
- return (domainid % AMD_IOMMU_DOMAIN_HASH_SZ);
-}
-
-/*ARGSUSED*/
-void
-amd_iommu_init_page_tables(amd_iommu_t *iommu)
-{
- amd_iommu_domain_table = kmem_zalloc(
- sizeof (amd_iommu_domain_t *) * AMD_IOMMU_DOMAIN_HASH_SZ, KM_SLEEP);
-}
-
-/*ARGSUSED*/
-void
-amd_iommu_fini_page_tables(amd_iommu_t *iommu)
-{
- if (amd_iommu_domain_table) {
- kmem_free(amd_iommu_domain_table,
- sizeof (amd_iommu_domain_t *) * AMD_IOMMU_DOMAIN_HASH_SZ);
- amd_iommu_domain_table = NULL;
- }
-}
-
-static amd_iommu_domain_t *
-amd_iommu_lookup_domain(amd_iommu_t *iommu, domain_id_t domainid,
- map_type_t type, int km_flags)
-{
- uint16_t idx;
- amd_iommu_domain_t *dp;
- char name[AMD_IOMMU_VMEM_NAMELEN+1];
-
- ASSERT(amd_iommu_domain_table);
-
- idx = hash_domain(domainid);
-
- for (dp = amd_iommu_domain_table[idx]; dp; dp = dp->d_next) {
- if (dp->d_domainid == domainid)
- return (dp);
- }
-
- ASSERT(type != AMD_IOMMU_INVALID_MAP);
-
- dp = kmem_zalloc(sizeof (*dp), km_flags);
- if (dp == NULL)
- return (NULL);
- dp->d_domainid = domainid;
- dp->d_pgtable_root_4K = 0; /* make this explicit */
-
- if (type == AMD_IOMMU_VMEM_MAP) {
- uint64_t base;
- uint64_t size;
- (void) snprintf(name, sizeof (name), "dvma_idx%d_domain%d",
- iommu->aiomt_idx, domainid);
- base = MMU_PAGESIZE;
- size = AMD_IOMMU_SIZE_4G - MMU_PAGESIZE;
- dp->d_vmem = vmem_create(name, (void *)(uintptr_t)base, size,
- MMU_PAGESIZE, NULL, NULL, NULL, 0,
- km_flags == KM_SLEEP ? VM_SLEEP : VM_NOSLEEP);
- if (dp->d_vmem == NULL) {
- kmem_free(dp, sizeof (*dp));
- return (NULL);
- }
- } else {
- dp->d_vmem = NULL;
- }
-
- dp->d_next = amd_iommu_domain_table[idx];
- dp->d_prev = NULL;
- amd_iommu_domain_table[idx] = dp;
- if (dp->d_next)
- dp->d_next->d_prev = dp;
- dp->d_ref = 0;
-
-
- return (dp);
-}
-
-static void
-amd_iommu_teardown_domain(amd_iommu_t *iommu, amd_iommu_domain_t *dp)
-{
- uint16_t idx;
- int flags;
- amd_iommu_cmdargs_t cmdargs = {0};
- domain_id_t domainid = dp->d_domainid;
- const char *f = "amd_iommu_teardown_domain";
-
- ASSERT(dp->d_ref == 0);
-
- idx = hash_domain(dp->d_domainid);
-
- if (dp->d_prev == NULL)
- amd_iommu_domain_table[idx] = dp->d_next;
- else
- dp->d_prev->d_next = dp->d_next;
-
- if (dp->d_next)
- dp->d_next->d_prev = dp->d_prev;
-
- if (dp->d_vmem != NULL) {
- vmem_destroy(dp->d_vmem);
- dp->d_vmem = NULL;
- }
-
- kmem_free(dp, sizeof (*dp));
-
- cmdargs.ca_domainid = (uint16_t)domainid;
- cmdargs.ca_addr = (uintptr_t)0x7FFFFFFFFFFFF000;
- flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL |
- AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S;
-
- if (amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_IOMMU_PAGES,
- &cmdargs, flags, 0) != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: idx=%d: domainid=%d"
- "Failed to invalidate domain in IOMMU HW cache",
- f, iommu->aiomt_idx, cmdargs.ca_domainid);
- }
-}
-
-static int
-amd_iommu_get_deviceid(amd_iommu_t *iommu, dev_info_t *rdip, int32_t *deviceid,
- int *aliasp, const char *path)
-{
- int bus = -1;
- int device = -1;
- int func = -1;
- uint16_t bdf;
- int32_t src_bdf;
- dev_info_t *idip = iommu->aiomt_dip;
- const char *driver = ddi_driver_name(idip);
- int instance = ddi_get_instance(idip);
- dev_info_t *pci_dip;
- const char *f = "amd_iommu_get_deviceid";
-
- /* be conservative. Always assume an alias */
- *aliasp = 1;
- *deviceid = 0;
-
- /* Check for special special devices (rdip == NULL) */
- if (rdip == NULL) {
- if (amd_iommu_get_src_bdf(iommu, -1, &src_bdf) != DDI_SUCCESS) {
- cmn_err(CE_WARN,
- "%s: %s%d: idx=%d, failed to get SRC BDF "
- "for special-device",
- f, driver, instance, iommu->aiomt_idx);
- return (DDI_DMA_NOMAPPING);
- }
- *deviceid = src_bdf;
- *aliasp = 1;
- return (DDI_SUCCESS);
- }
-
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
- cmn_err(CE_NOTE, "%s: attempting to get deviceid for %s",
- f, path);
- }
-
- pci_dip = amd_iommu_pci_dip(rdip, path);
- if (pci_dip == NULL) {
- cmn_err(CE_WARN, "%s: %s%d: idx = %d, failed to get PCI dip "
- "for rdip=%p, path = %s",
- f, driver, instance, iommu->aiomt_idx, (void *)rdip,
- path);
- return (DDI_DMA_NOMAPPING);
- }
-
- if (acpica_get_bdf(pci_dip, &bus, &device, &func) != DDI_SUCCESS) {
- ndi_rele_devi(pci_dip);
- cmn_err(CE_WARN, "%s: %s%d: idx=%d, failed to get BDF for "
- "PCI dip (%p). rdip path = %s",
- f, driver, instance, iommu->aiomt_idx,
- (void *)pci_dip, path);
- return (DDI_DMA_NOMAPPING);
- }
-
- ndi_rele_devi(pci_dip);
-
- if (bus > UINT8_MAX || bus < 0 ||
- device > UINT8_MAX || device < 0 ||
- func > UINT8_MAX || func < 0) {
- cmn_err(CE_WARN, "%s: %s%d: idx=%d, invalid BDF(%d,%d,%d) "
- "for PCI dip (%p). rdip path = %s", f, driver, instance,
- iommu->aiomt_idx,
- bus, device, func,
- (void *)pci_dip, path);
- return (DDI_DMA_NOMAPPING);
- }
-
- bdf = ((uint8_t)bus << 8) | ((uint8_t)device << 3) | (uint8_t)func;
-
- if (amd_iommu_get_src_bdf(iommu, bdf, &src_bdf) != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: idx=%d, failed to get SRC BDF "
- "for PCI dip (%p) rdip path = %s.",
- f, driver, instance, iommu->aiomt_idx, (void *)pci_dip,
- path);
- return (DDI_DMA_NOMAPPING);
- }
-
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
- cmn_err(CE_NOTE, "%s: Deviceid = %u for path = %s",
- f, src_bdf, path);
- }
-
- *deviceid = src_bdf;
- *aliasp = (src_bdf != bdf);
-
- return (DDI_SUCCESS);
-}
-
-/*ARGSUSED*/
-static int
-init_devtbl(amd_iommu_t *iommu, uint64_t *devtbl_entry, domain_id_t domainid,
- amd_iommu_domain_t *dp)
-{
- uint64_t entry[4] = {0};
- int i;
-
- /* If already passthru, don't touch */
- if (AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V) == 0 &&
- AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV) == 0) {
- return (0);
- }
-
- if (AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V) == 1 &&
- AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV) == 1) {
-
- ASSERT(dp->d_pgtable_root_4K ==
- AMD_IOMMU_REG_GET64(&(devtbl_entry[0]),
- AMD_IOMMU_DEVTBL_ROOT_PGTBL));
-
- ASSERT(dp->d_domainid == AMD_IOMMU_REG_GET64(&(devtbl_entry[1]),
- AMD_IOMMU_DEVTBL_DOMAINID));
-
- return (0);
- }
-
- /* New devtbl entry for this domain. Bump up the domain ref-count */
- dp->d_ref++;
-
- entry[3] = 0;
- entry[2] = 0;
- AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SYSMGT, 1);
- AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_EX, 1);
- AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SD, 0);
- AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_CACHE, 0);
- AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_IOCTL, 1);
- AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SA, 0);
- AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SE, 1);
- AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_IOTLB, 1);
- AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_DOMAINID,
- (uint16_t)domainid);
- AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_IW, 1);
- AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_IR, 1);
- AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_ROOT_PGTBL,
- dp->d_pgtable_root_4K);
- AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_PG_MODE,
- AMD_IOMMU_PGTABLE_MAXLEVEL);
- AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_TV,
- domainid == AMD_IOMMU_PASSTHRU_DOMAIN ? 0 : 1);
- AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_V,
- domainid == AMD_IOMMU_PASSTHRU_DOMAIN ? 0 : 1);
-
- for (i = 1; i < 4; i++) {
- devtbl_entry[i] = entry[i];
- }
- devtbl_entry[0] = entry[0];
-
- /* we did an actual init */
- return (1);
-}
-
-void
-amd_iommu_set_passthru(amd_iommu_t *iommu, dev_info_t *rdip)
-{
- int32_t deviceid;
- int alias;
- uint64_t *devtbl_entry;
- amd_iommu_cmdargs_t cmdargs = {0};
- char *path;
- int pathfree;
- int V;
- int TV;
- int instance;
- const char *driver;
- const char *f = "amd_iommu_set_passthru";
-
- if (rdip) {
- driver = ddi_driver_name(rdip);
- instance = ddi_get_instance(rdip);
- } else {
- driver = "special-device";
- instance = 0;
- }
-
- path = kmem_alloc(MAXPATHLEN, KM_NOSLEEP);
- if (path) {
- if (rdip)
- (void) ddi_pathname(rdip, path);
- else
- (void) strcpy(path, "special-device");
- pathfree = 1;
- } else {
- pathfree = 0;
- path = "<path-mem-alloc-failed>";
- }
-
- if (amd_iommu_get_deviceid(iommu, rdip, &deviceid, &alias, path)
- != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p. "
- "Failed to get device ID for device %s.", f, driver,
- instance,
- iommu->aiomt_idx, (void *)rdip, path);
- goto out;
- }
-
- /* No deviceid */
- if (deviceid == -1) {
- goto out;
- }
-
- if ((deviceid + 1) * AMD_IOMMU_DEVTBL_ENTRY_SZ >
- iommu->aiomt_devtbl_sz) {
- cmn_err(CE_WARN, "%s: %s%d: IOMMU idx=%d, deviceid (%u) "
- "for rdip (%p) exceeds device table size (%u), path=%s",
- f, driver,
- instance, iommu->aiomt_idx, deviceid, (void *)rdip,
- iommu->aiomt_devtbl_sz, path);
- goto out;
- }
-
- /*LINTED*/
- devtbl_entry = (uint64_t *)&iommu->aiomt_devtbl
- [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ];
-
- V = AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V);
- TV = AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV);
-
- /* Already passthru */
- if (V == 0 && TV == 0) {
- goto out;
- }
-
- /* Existing translations */
- if (V == 1 && TV == 1) {
- goto out;
- }
-
- /* Invalid setting */
- if (V == 0 && TV == 1) {
- goto out;
- }
-
- AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V, 0);
-
- cmdargs.ca_deviceid = (uint16_t)deviceid;
- (void) amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY,
- &cmdargs, 0, 0);
-
-out:
- if (pathfree)
- kmem_free(path, MAXPATHLEN);
-}
-
-static int
-amd_iommu_set_devtbl_entry(amd_iommu_t *iommu, dev_info_t *rdip,
- domain_id_t domainid, uint16_t deviceid, amd_iommu_domain_t *dp,
- const char *path)
-{
- uint64_t *devtbl_entry;
- amd_iommu_cmdargs_t cmdargs = {0};
- int error;
- dev_info_t *idip = iommu->aiomt_dip;
- const char *driver = ddi_driver_name(idip);
- int instance = ddi_get_instance(idip);
- const char *f = "amd_iommu_set_devtbl_entry";
-
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
- cmn_err(CE_WARN, "%s: attempting to set devtbl entry for %s",
- f, path);
- }
-
- if ((deviceid + 1) * AMD_IOMMU_DEVTBL_ENTRY_SZ >
- iommu->aiomt_devtbl_sz) {
- cmn_err(CE_WARN, "%s: %s%d: IOMMU idx=%d, deviceid (%u) "
- "for rdip (%p) exceeds device table size (%u), path=%s",
- f, driver,
- instance, iommu->aiomt_idx, deviceid, (void *)rdip,
- iommu->aiomt_devtbl_sz, path);
- return (DDI_DMA_NOMAPPING);
- }
-
- /*LINTED*/
- devtbl_entry = (uint64_t *)&iommu->aiomt_devtbl
- [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ];
-
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
- cmn_err(CE_WARN, "%s: deviceid=%u devtbl entry (%p) for %s",
- f, deviceid, (void *)(uintptr_t)(*devtbl_entry), path);
- }
-
- if (init_devtbl(iommu, devtbl_entry, domainid, dp)) {
- cmdargs.ca_deviceid = deviceid;
- error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY,
- &cmdargs, 0, 0);
- }
-
- return (error);
-}
-
-int
-amd_iommu_clear_devtbl_entry(amd_iommu_t *iommu, dev_info_t *rdip,
- domain_id_t domainid, uint16_t deviceid, amd_iommu_domain_t *dp,
- int *domain_freed, char *path)
-{
- uint64_t *devtbl_entry;
- int error = DDI_SUCCESS;
- amd_iommu_cmdargs_t cmdargs = {0};
- const char *driver = ddi_driver_name(iommu->aiomt_dip);
- int instance = ddi_get_instance(iommu->aiomt_dip);
- const char *f = "amd_iommu_clear_devtbl_entry";
-
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
- cmn_err(CE_NOTE, "%s: attempting to clear devtbl entry for "
- "domainid = %d, deviceid = %u, path = %s",
- f, domainid, deviceid, path);
- }
-
- if ((deviceid + 1) * AMD_IOMMU_DEVTBL_ENTRY_SZ >
- iommu->aiomt_devtbl_sz) {
- cmn_err(CE_WARN, "%s: %s%d: IOMMU idx=%d, deviceid (%u) "
- "for rdip (%p) exceeds device table size (%u), path = %s",
- f, driver, instance,
- iommu->aiomt_idx, deviceid, (void *)rdip,
- iommu->aiomt_devtbl_sz, path);
- return (DDI_FAILURE);
- }
-
- /*LINTED*/
- devtbl_entry = (uint64_t *)&iommu->aiomt_devtbl
- [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ];
-
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) {
- cmn_err(CE_WARN, "%s: deviceid=%u devtbl entry (%p) for %s",
- f, deviceid, (void *)(uintptr_t)(*devtbl_entry), path);
- }
-
- if (AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV) == 0) {
- /* Nothing to do */
- return (DDI_SUCCESS);
- }
-
- ASSERT(dp->d_pgtable_root_4K == AMD_IOMMU_REG_GET64(&(devtbl_entry[0]),
- AMD_IOMMU_DEVTBL_ROOT_PGTBL));
-
- ASSERT(domainid == AMD_IOMMU_REG_GET64(&(devtbl_entry[1]),
- AMD_IOMMU_DEVTBL_DOMAINID));
-
- AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV, 0);
- AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_ROOT_PGTBL, 0);
- AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V, 1);
-
- SYNC_FORDEV(iommu->aiomt_dmahdl);
-
- dp->d_ref--;
- ASSERT(dp->d_ref >= 0);
-
- if (dp->d_ref == 0) {
- *domain_freed = 1;
- }
-
- cmdargs.ca_deviceid = deviceid;
- error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY,
- &cmdargs, 0, 0);
- if (error != DDI_SUCCESS)
- error = DDI_FAILURE;
-
- return (error);
-}
-
-int
-amd_iommu_page_table_hash_init(amd_iommu_page_table_hash_t *ampt)
-{
- ampt->ampt_hash = kmem_zalloc(sizeof (amd_iommu_page_table_t *) *
- AMD_IOMMU_PGTABLE_HASH_SZ, KM_SLEEP);
- return (DDI_SUCCESS);
-}
-
-void
-amd_iommu_page_table_hash_fini(amd_iommu_page_table_hash_t *ampt)
-{
- kmem_free(ampt->ampt_hash,
- sizeof (amd_iommu_page_table_t *) * AMD_IOMMU_PGTABLE_HASH_SZ);
- ampt->ampt_hash = NULL;
-}
-
-static uint32_t
-pt_hashfn(uint64_t pa_4K)
-{
- return (pa_4K % AMD_IOMMU_PGTABLE_HASH_SZ);
-}
-
-static void
-amd_iommu_insert_pgtable_hash(amd_iommu_page_table_t *pt)
-{
- uint64_t pa_4K = ((uint64_t)pt->pt_cookie.dmac_cookie_addr) >> 12;
- uint32_t idx = pt_hashfn(pa_4K);
-
- ASSERT((pt->pt_cookie.dmac_cookie_addr & AMD_IOMMU_PGTABLE_ALIGN) == 0);
-
- mutex_enter(&amd_iommu_page_table_hash.ampt_lock);
-
- pt->pt_next = amd_iommu_page_table_hash.ampt_hash[idx];
- pt->pt_prev = NULL;
- amd_iommu_page_table_hash.ampt_hash[idx] = pt;
- if (pt->pt_next)
- pt->pt_next->pt_prev = pt;
-
- mutex_exit(&amd_iommu_page_table_hash.ampt_lock);
-}
-
-static void
-amd_iommu_remove_pgtable_hash(amd_iommu_page_table_t *pt)
-{
- uint64_t pa_4K = (pt->pt_cookie.dmac_cookie_addr >> 12);
- uint32_t idx = pt_hashfn(pa_4K);
-
- ASSERT((pt->pt_cookie.dmac_cookie_addr & AMD_IOMMU_PGTABLE_ALIGN) == 0);
-
- mutex_enter(&amd_iommu_page_table_hash.ampt_lock);
-
- if (pt->pt_next)
- pt->pt_next->pt_prev = pt->pt_prev;
-
- if (pt->pt_prev)
- pt->pt_prev->pt_next = pt->pt_next;
- else
- amd_iommu_page_table_hash.ampt_hash[idx] = pt->pt_next;
-
- pt->pt_next = NULL;
- pt->pt_prev = NULL;
-
- mutex_exit(&amd_iommu_page_table_hash.ampt_lock);
-}
-
-static amd_iommu_page_table_t *
-amd_iommu_lookup_pgtable_hash(domain_id_t domainid, uint64_t pgtable_pa_4K)
-{
- amd_iommu_page_table_t *pt;
- uint32_t idx = pt_hashfn(pgtable_pa_4K);
-
- mutex_enter(&amd_iommu_page_table_hash.ampt_lock);
- pt = amd_iommu_page_table_hash.ampt_hash[idx];
- for (; pt; pt = pt->pt_next) {
- if (domainid != pt->pt_domainid)
- continue;
- ASSERT((pt->pt_cookie.dmac_cookie_addr &
- AMD_IOMMU_PGTABLE_ALIGN) == 0);
- if ((pt->pt_cookie.dmac_cookie_addr >> 12) == pgtable_pa_4K) {
- break;
- }
- }
- mutex_exit(&amd_iommu_page_table_hash.ampt_lock);
-
- return (pt);
-}
-
-/*ARGSUSED*/
-static amd_iommu_page_table_t *
-amd_iommu_lookup_pgtable(amd_iommu_t *iommu, amd_iommu_page_table_t *ppt,
- amd_iommu_domain_t *dp, int level, uint16_t index)
-{
- uint64_t *pdtep;
- uint64_t pgtable_pa_4K;
-
- ASSERT(level > 0 && level <= AMD_IOMMU_PGTABLE_MAXLEVEL);
- ASSERT(dp);
-
- if (level == AMD_IOMMU_PGTABLE_MAXLEVEL) {
- ASSERT(ppt == NULL);
- ASSERT(index == 0);
- pgtable_pa_4K = dp->d_pgtable_root_4K;
- } else {
- ASSERT(ppt);
- pdtep = &(ppt->pt_pgtblva[index]);
- if (AMD_IOMMU_REG_GET64(pdtep, AMD_IOMMU_PTDE_PR) == 0) {
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
- cmn_err(CE_NOTE, "Skipping PR=0 pdte: 0x%"
- PRIx64, *pdtep);
- }
- return (NULL);
- }
- pgtable_pa_4K = AMD_IOMMU_REG_GET64(pdtep, AMD_IOMMU_PTDE_ADDR);
- }
-
- return (amd_iommu_lookup_pgtable_hash(dp->d_domainid, pgtable_pa_4K));
-}
-
-static amd_iommu_page_table_t *
-amd_iommu_alloc_from_freelist(void)
-{
- int i;
- uint64_t *pte_array;
- amd_iommu_page_table_t *pt;
-
- if (amd_iommu_no_pgtable_freelist == 1)
- return (NULL);
-
- if (amd_iommu_pgtable_freelist.f_count == 0)
- return (NULL);
-
- pt = amd_iommu_pgtable_freelist.f_list;
- amd_iommu_pgtable_freelist.f_list = pt->pt_next;
- amd_iommu_pgtable_freelist.f_count--;
-
- pte_array = pt->pt_pgtblva;
- for (i = 0; i < AMD_IOMMU_PGTABLE_SZ / (sizeof (*pte_array)); i++) {
- ASSERT(pt->pt_pte_ref[i] == 0);
- ASSERT(AMD_IOMMU_REG_GET64(&(pte_array[i]),
- AMD_IOMMU_PTDE_PR) == 0);
- }
-
- return (pt);
-}
-
-static int
-amd_iommu_alloc_pgtable(amd_iommu_t *iommu, domain_id_t domainid,
- const char *path, amd_iommu_page_table_t **ptp, int km_flags)
-{
- int err;
- uint_t ncookies;
- amd_iommu_page_table_t *pt;
- dev_info_t *idip = iommu->aiomt_dip;
- const char *driver = ddi_driver_name(idip);
- int instance = ddi_get_instance(idip);
- const char *f = "amd_iommu_alloc_pgtable";
-
- *ptp = NULL;
-
- pt = amd_iommu_alloc_from_freelist();
- if (pt)
- goto init_pgtable;
-
- pt = kmem_zalloc(sizeof (amd_iommu_page_table_t), km_flags);
- if (pt == NULL)
- return (DDI_DMA_NORESOURCES);
-
- /*
- * Each page table is 4K in size
- */
- pt->pt_mem_reqsz = AMD_IOMMU_PGTABLE_SZ;
-
- /*
- * Alloc a DMA handle. Use the IOMMU dip as we want this DMA
- * to *not* enter the IOMMU - no recursive entrance.
- */
- err = ddi_dma_alloc_handle(idip, &amd_iommu_pgtable_dma_attr,
- km_flags == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT,
- NULL, &pt->pt_dma_hdl);
- if (err != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: domainid = %d, path = %s. "
- "Cannot alloc DMA handle for IO Page Table",
- f, driver, instance, domainid, path);
- kmem_free(pt, sizeof (amd_iommu_page_table_t));
- return (err == DDI_DMA_NORESOURCES ? err : DDI_DMA_NOMAPPING);
- }
-
- /*
- * Alloc memory for IO Page Table.
- * XXX remove size_t cast kludge
- */
- err = ddi_dma_mem_alloc(pt->pt_dma_hdl, pt->pt_mem_reqsz,
- &amd_iommu_devacc, DDI_DMA_CONSISTENT|IOMEM_DATA_UNCACHED,
- km_flags == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT,
- NULL, (caddr_t *)&pt->pt_pgtblva,
- (size_t *)&pt->pt_mem_realsz, &pt->pt_mem_hdl);
- if (err != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: domainid=%d, path = %s. "
- "Cannot allocate DMA memory for IO Page table",
- f, driver, instance, domainid, path);
- ddi_dma_free_handle(&pt->pt_dma_hdl);
- kmem_free(pt, sizeof (amd_iommu_page_table_t));
- return (DDI_DMA_NORESOURCES);
- }
-
- /*
- * The Page table DMA VA must be 4K aligned and
- * size >= than requested memory.
- *
- */
- ASSERT(((uint64_t)(uintptr_t)pt->pt_pgtblva & AMD_IOMMU_PGTABLE_ALIGN)
- == 0);
- ASSERT(pt->pt_mem_realsz >= pt->pt_mem_reqsz);
-
- /*
- * Now bind the handle
- */
- err = ddi_dma_addr_bind_handle(pt->pt_dma_hdl, NULL,
- (caddr_t)pt->pt_pgtblva, pt->pt_mem_realsz,
- DDI_DMA_READ | DDI_DMA_CONSISTENT,
- km_flags == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT,
- NULL, &pt->pt_cookie, &ncookies);
- if (err != DDI_DMA_MAPPED) {
- cmn_err(CE_WARN, "%s: %s%d: domainid=%d, path = %s. "
- "Cannot bind memory for DMA to IO Page Tables. "
- "bufrealsz=%p",
- f, driver, instance, domainid, path,
- (void *)(uintptr_t)pt->pt_mem_realsz);
- ddi_dma_mem_free(&pt->pt_mem_hdl);
- ddi_dma_free_handle(&pt->pt_dma_hdl);
- kmem_free(pt, sizeof (amd_iommu_page_table_t));
- return (err == DDI_DMA_PARTIAL_MAP ? DDI_DMA_NOMAPPING :
- err);
- }
-
- /*
- * We assume the DMA engine on the IOMMU is capable of handling the
- * whole page table in a single cookie. If not and multiple cookies
- * are needed we fail.
- */
- if (ncookies != 1) {
- cmn_err(CE_WARN, "%s: %s%d: domainid = %d, path=%s "
- "Cannot handle multiple "
- "cookies for DMA to IO page Table, #cookies=%u",
- f, driver, instance, domainid, path, ncookies);
- (void) ddi_dma_unbind_handle(pt->pt_dma_hdl);
- ddi_dma_mem_free(&pt->pt_mem_hdl);
- ddi_dma_free_handle(&pt->pt_dma_hdl);
- kmem_free(pt, sizeof (amd_iommu_page_table_t));
- return (DDI_DMA_NOMAPPING);
- }
-
-init_pgtable:
- /*
- * The address in the cookie must be 4K aligned and >= table size
- */
- ASSERT(pt->pt_cookie.dmac_cookie_addr != NULL);
- ASSERT((pt->pt_cookie.dmac_cookie_addr & AMD_IOMMU_PGTABLE_ALIGN) == 0);
- ASSERT(pt->pt_cookie.dmac_size >= pt->pt_mem_realsz);
- ASSERT(pt->pt_cookie.dmac_size >= pt->pt_mem_reqsz);
- ASSERT(pt->pt_mem_reqsz >= AMD_IOMMU_PGTABLE_SIZE);
- ASSERT(pt->pt_mem_realsz >= pt->pt_mem_reqsz);
- ASSERT(pt->pt_pgtblva);
-
- pt->pt_domainid = AMD_IOMMU_INVALID_DOMAIN;
- pt->pt_level = 0x7;
- pt->pt_index = 0;
- pt->pt_ref = 0;
- pt->pt_next = NULL;
- pt->pt_prev = NULL;
- pt->pt_parent = NULL;
-
- bzero(pt->pt_pgtblva, pt->pt_mem_realsz);
- SYNC_FORDEV(pt->pt_dma_hdl);
-
- amd_iommu_insert_pgtable_hash(pt);
-
- *ptp = pt;
-
- return (DDI_SUCCESS);
-}
-
-static int
-amd_iommu_move_to_freelist(amd_iommu_page_table_t *pt)
-{
- if (amd_iommu_no_pgtable_freelist == 1)
- return (DDI_FAILURE);
-
- if (amd_iommu_pgtable_freelist.f_count ==
- AMD_IOMMU_PGTABLE_FREELIST_MAX)
- return (DDI_FAILURE);
-
- pt->pt_next = amd_iommu_pgtable_freelist.f_list;
- amd_iommu_pgtable_freelist.f_list = pt;
- amd_iommu_pgtable_freelist.f_count++;
-
- return (DDI_SUCCESS);
-}
-
-static void
-amd_iommu_free_pgtable(amd_iommu_t *iommu, amd_iommu_page_table_t *pt)
-{
- int i;
- uint64_t *pte_array;
- dev_info_t *dip = iommu->aiomt_dip;
- int instance = ddi_get_instance(dip);
- const char *driver = ddi_driver_name(dip);
- const char *f = "amd_iommu_free_pgtable";
-
- ASSERT(pt->pt_ref == 0);
-
- amd_iommu_remove_pgtable_hash(pt);
-
- pte_array = pt->pt_pgtblva;
- for (i = 0; i < AMD_IOMMU_PGTABLE_SZ / (sizeof (*pte_array)); i++) {
- ASSERT(pt->pt_pte_ref[i] == 0);
- ASSERT(AMD_IOMMU_REG_GET64(&(pte_array[i]),
- AMD_IOMMU_PTDE_PR) == 0);
- }
-
- if (amd_iommu_move_to_freelist(pt) == DDI_SUCCESS)
- return;
-
- /* Unbind the handle */
- if (ddi_dma_unbind_handle(pt->pt_dma_hdl) != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: idx=%d, domainid=%d. "
- "Failed to unbind handle: %p for IOMMU Page Table",
- f, driver, instance, iommu->aiomt_idx, pt->pt_domainid,
- (void *)pt->pt_dma_hdl);
- }
- /* Free the table memory allocated for DMA */
- ddi_dma_mem_free(&pt->pt_mem_hdl);
-
- /* Free the DMA handle */
- ddi_dma_free_handle(&pt->pt_dma_hdl);
-
- kmem_free(pt, sizeof (amd_iommu_page_table_t));
-
-}
-
-static int
-init_pde(amd_iommu_page_table_t *ppt, amd_iommu_page_table_t *pt)
-{
- uint64_t *pdep = &(ppt->pt_pgtblva[pt->pt_index]);
- uint64_t next_pgtable_pa_4K = (pt->pt_cookie.dmac_cookie_addr) >> 12;
-
- /* nothing to set. PDE is already set */
- if (AMD_IOMMU_REG_GET64(pdep, AMD_IOMMU_PTDE_PR) == 1) {
- ASSERT(PT_REF_VALID(ppt));
- ASSERT(PT_REF_VALID(pt));
- ASSERT(ppt->pt_pte_ref[pt->pt_index] == 0);
- ASSERT(AMD_IOMMU_REG_GET64(pdep, AMD_IOMMU_PTDE_ADDR)
- == next_pgtable_pa_4K);
- return (DDI_SUCCESS);
- }
-
- ppt->pt_ref++;
- ASSERT(PT_REF_VALID(ppt));
-
- /* Page Directories are always RW */
- AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_IW, 1);
- AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_IR, 1);
- AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_ADDR,
- next_pgtable_pa_4K);
- pt->pt_parent = ppt;
- AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_NXT_LVL,
- pt->pt_level);
- ppt->pt_pte_ref[pt->pt_index] = 0;
- AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_PR, 1);
- SYNC_FORDEV(ppt->pt_dma_hdl);
- ASSERT(AMD_IOMMU_REG_GET64(pdep, AMD_IOMMU_PTDE_PR) == 1);
-
- return (DDI_SUCCESS);
-}
-
-static int
-init_pte(amd_iommu_page_table_t *pt, uint64_t pa, uint16_t index,
- struct ddi_dma_req *dmareq)
-{
- uint64_t *ptep = &(pt->pt_pgtblva[index]);
- uint64_t pa_4K = pa >> 12;
- int R;
- int W;
-
- /* nothing to set if PTE is already set */
- if (AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_PR) == 1) {
- /*
- * Adjust current permissions
- * DDI_DMA_WRITE means direction of DMA is MEM -> I/O
- * so that requires Memory READ permissions i.e. sense
- * is inverted.
- * Note: either or both of DD_DMA_READ/WRITE may be set
- */
- if (amd_iommu_no_RW_perms == 0) {
- R = AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_IR);
- W = AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_IW);
- if (R == 0 && ((dmareq->dmar_flags & DDI_DMA_WRITE) ||
- (dmareq->dmar_flags & DDI_DMA_RDWR))) {
- AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1);
- }
- if (W == 0 && ((dmareq->dmar_flags & DDI_DMA_READ) ||
- (dmareq->dmar_flags & DDI_DMA_RDWR))) {
- AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1);
- }
- }
- ASSERT(PT_REF_VALID(pt));
- pt->pt_pte_ref[index]++;
- ASSERT(AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_ADDR)
- == pa_4K);
- return (DDI_SUCCESS);
- }
-
- pt->pt_ref++;
- ASSERT(PT_REF_VALID(pt));
-
- /* see comment above about inverting sense of RD/WR */
- if (amd_iommu_no_RW_perms == 0) {
- AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 0);
- AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 0);
- if (dmareq->dmar_flags & DDI_DMA_RDWR) {
- AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1);
- AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1);
- } else {
- if (dmareq->dmar_flags & DDI_DMA_WRITE) {
- AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1);
- }
- if (dmareq->dmar_flags & DDI_DMA_READ) {
- AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1);
- }
- }
- } else {
- AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1);
- AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1);
- }
-
- /* TODO what is correct for FC and U */
- AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTE_FC, 0);
- AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTE_U, 0);
- AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_ADDR, pa_4K);
- AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_NXT_LVL, 0);
- ASSERT(pt->pt_pte_ref[index] == 0);
- pt->pt_pte_ref[index] = 1;
- AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_PR, 1);
- SYNC_FORDEV(pt->pt_dma_hdl);
- ASSERT(AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_PR) == 1);
-
- return (DDI_SUCCESS);
-}
-
-
-static void
-init_pt(amd_iommu_page_table_t *pt, amd_iommu_domain_t *dp,
- int level, uint16_t index)
-{
- ASSERT(dp);
-
- if (level == AMD_IOMMU_PGTABLE_MAXLEVEL) {
- dp->d_pgtable_root_4K = (pt->pt_cookie.dmac_cookie_addr) >> 12;
- } else {
- ASSERT(level >= 1 && level < AMD_IOMMU_PGTABLE_MAXLEVEL);
- }
-
- pt->pt_domainid = dp->d_domainid;
- pt->pt_level = level;
- pt->pt_index = index;
-}
-
-static int
-amd_iommu_setup_1_pgtable(amd_iommu_t *iommu, dev_info_t *rdip,
- struct ddi_dma_req *dmareq,
- domain_id_t domainid, amd_iommu_domain_t *dp,
- amd_iommu_page_table_t *ppt,
- uint16_t index, int level, uint64_t va, uint64_t pa,
- amd_iommu_page_table_t **ptp, uint16_t *next_idxp, const char *path,
- int km_flags)
-{
- int error;
- amd_iommu_page_table_t *pt;
- const char *driver = ddi_driver_name(rdip);
- int instance = ddi_get_instance(rdip);
- const char *f = "amd_iommu_setup_1_pgtable";
-
- *ptp = NULL;
- *next_idxp = 0;
- error = DDI_SUCCESS;
-
- ASSERT(level > 0 && level <= AMD_IOMMU_PGTABLE_MAXLEVEL);
-
- ASSERT(dp);
- if (level == AMD_IOMMU_PGTABLE_MAXLEVEL) {
- ASSERT(ppt == NULL);
- ASSERT(index == 0);
- } else {
- ASSERT(ppt);
- }
-
- /* Check if page table is already allocated */
- if (pt = amd_iommu_lookup_pgtable(iommu, ppt, dp, level, index)) {
- ASSERT(pt->pt_domainid == domainid);
- ASSERT(pt->pt_level == level);
- ASSERT(pt->pt_index == index);
- goto out;
- }
-
- if ((error = amd_iommu_alloc_pgtable(iommu, domainid, path, &pt,
- km_flags)) != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: idx = %u, domainid = %d, va = %p "
- "path = %s", f, driver, instance, iommu->aiomt_idx,
- domainid, (void *)(uintptr_t)va, path);
- return (error);
- }
-
- ASSERT(dp->d_domainid == domainid);
-
- init_pt(pt, dp, level, index);
-
-out:
- if (level != AMD_IOMMU_PGTABLE_MAXLEVEL) {
- error = init_pde(ppt, pt);
- }
-
- if (level == 1) {
- ASSERT(error == DDI_SUCCESS);
- error = init_pte(pt, pa, AMD_IOMMU_VA_BITS(va, level), dmareq);
- } else {
- *next_idxp = AMD_IOMMU_VA_BITS(va, level);
- *ptp = pt;
- }
-
- return (error);
-}
-
-typedef enum {
- PDTE_NOT_TORN = 0x1,
- PDTE_TORN_DOWN = 0x2,
- PGTABLE_TORN_DOWN = 0x4
-} pdte_tear_t;
-
-static pdte_tear_t
-amd_iommu_teardown_pdte(amd_iommu_t *iommu,
- amd_iommu_page_table_t *pt, int index)
-{
- uint8_t next_level;
- pdte_tear_t retval;
- uint64_t *ptdep = &(pt->pt_pgtblva[index]);
-
- next_level = AMD_IOMMU_REG_GET64(ptdep,
- AMD_IOMMU_PTDE_NXT_LVL);
-
- if (AMD_IOMMU_REG_GET64(ptdep, AMD_IOMMU_PTDE_PR) == 1) {
- if (pt->pt_level == 1) {
- ASSERT(next_level == 0);
- /* PTE */
- pt->pt_pte_ref[index]--;
- if (pt->pt_pte_ref[index] != 0) {
- return (PDTE_NOT_TORN);
- }
- } else {
- ASSERT(next_level != 0 && next_level != 7);
- }
- ASSERT(pt->pt_pte_ref[index] == 0);
- ASSERT(PT_REF_VALID(pt));
-
- AMD_IOMMU_REG_SET64(ptdep, AMD_IOMMU_PTDE_PR, 0);
- SYNC_FORDEV(pt->pt_dma_hdl);
- ASSERT(AMD_IOMMU_REG_GET64(ptdep,
- AMD_IOMMU_PTDE_PR) == 0);
- pt->pt_ref--;
- ASSERT(PT_REF_VALID(pt));
- retval = PDTE_TORN_DOWN;
- } else {
- ASSERT(0);
- ASSERT(pt->pt_pte_ref[index] == 0);
- ASSERT(PT_REF_VALID(pt));
- retval = PDTE_NOT_TORN;
- }
-
- if (pt->pt_ref == 0) {
- amd_iommu_free_pgtable(iommu, pt);
- return (PGTABLE_TORN_DOWN);
- }
-
- return (retval);
-}
-
-static int
-amd_iommu_create_pgtables(amd_iommu_t *iommu, dev_info_t *rdip,
- struct ddi_dma_req *dmareq, uint64_t va,
- uint64_t pa, uint16_t deviceid, domain_id_t domainid,
- amd_iommu_domain_t *dp, const char *path, int km_flags)
-{
- int level;
- uint16_t index;
- uint16_t next_idx;
- amd_iommu_page_table_t *pt;
- amd_iommu_page_table_t *ppt;
- int error;
- const char *driver = ddi_driver_name(rdip);
- int instance = ddi_get_instance(rdip);
- const char *f = "amd_iommu_create_pgtables";
-
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
- cmn_err(CE_NOTE, "%s: %s%d: idx = %u, domainid = %d, "
- "deviceid = %u, va = %p, pa = %p, path = %s",
- f, driver, instance,
- iommu->aiomt_idx, domainid, deviceid,
- (void *)(uintptr_t)va,
- (void *)(uintptr_t)pa, path);
- }
-
- if (domainid == AMD_IOMMU_PASSTHRU_DOMAIN) {
- /* No need for pagetables. Just set up device table entry */
- goto passthru;
- }
-
- index = 0;
- ppt = NULL;
- for (level = AMD_IOMMU_PGTABLE_MAXLEVEL; level > 0;
- level--, pt = NULL, next_idx = 0) {
- if ((error = amd_iommu_setup_1_pgtable(iommu, rdip, dmareq,
- domainid, dp, ppt, index, level, va, pa, &pt,
- &next_idx, path, km_flags)) != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, "
- "deviceid=%u, va= %p, pa = %p, Failed to setup "
- "page table(s) at level = %d, path = %s.",
- f, driver, instance, iommu->aiomt_idx,
- domainid, deviceid, (void *)(uintptr_t)va,
- (void *)(uintptr_t)pa, level, path);
- return (error);
- }
-
- if (level > 1) {
- ASSERT(pt);
- ASSERT(pt->pt_domainid == domainid);
- ppt = pt;
- index = next_idx;
- } else {
- ASSERT(level == 1);
- ASSERT(pt == NULL);
- ASSERT(next_idx == 0);
- ppt = NULL;
- index = 0;
- }
- }
-
-passthru:
- if ((error = amd_iommu_set_devtbl_entry(iommu, rdip, domainid, deviceid,
- dp, path)) != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p, deviceid=%u, "
- "domainid=%d."
- "Failed to set device table entry for path %s.",
- f, driver, instance,
- iommu->aiomt_idx, (void *)rdip, deviceid, domainid, path);
- return (error);
- }
-
- SYNC_FORDEV(iommu->aiomt_dmahdl);
-
- return (DDI_SUCCESS);
-}
-
-static int
-amd_iommu_destroy_pgtables(amd_iommu_t *iommu, dev_info_t *rdip,
- uint64_t pageva, uint16_t deviceid, domain_id_t domainid,
- amd_iommu_domain_t *dp, map_type_t type, int *domain_freed, char *path)
-{
- int level;
- int flags;
- amd_iommu_cmdargs_t cmdargs = {0};
- uint16_t index;
- uint16_t prev_index;
- amd_iommu_page_table_t *pt;
- amd_iommu_page_table_t *ppt;
- pdte_tear_t retval;
- int tear_level;
- int invalidate_pte;
- int invalidate_pde;
- int error = DDI_FAILURE;
- const char *driver = ddi_driver_name(iommu->aiomt_dip);
- int instance = ddi_get_instance(iommu->aiomt_dip);
- const char *f = "amd_iommu_destroy_pgtables";
-
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
- cmn_err(CE_NOTE, "%s: %s%d: idx = %u, domainid = %d, "
- "deviceid = %u, va = %p, path = %s",
- f, driver, instance,
- iommu->aiomt_idx, domainid, deviceid,
- (void *)(uintptr_t)pageva, path);
- }
-
- if (domainid == AMD_IOMMU_PASSTHRU_DOMAIN) {
- /*
- * there are no pagetables for the passthru domain.
- * Just the device table entry
- */
- error = DDI_SUCCESS;
- goto passthru;
- }
-
- ppt = NULL;
- index = 0;
- for (level = AMD_IOMMU_PGTABLE_MAXLEVEL; level > 0; level--) {
- pt = amd_iommu_lookup_pgtable(iommu, ppt, dp, level, index);
- if (pt) {
- ppt = pt;
- index = AMD_IOMMU_VA_BITS(pageva, level);
- continue;
- }
- break;
- }
-
- if (level == 0) {
- uint64_t *ptep;
- uint64_t pa_4K;
-
- ASSERT(pt);
- ASSERT(pt == ppt);
- ASSERT(pt->pt_domainid == dp->d_domainid);
-
- ptep = &(pt->pt_pgtblva[index]);
-
- pa_4K = AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_ADDR);
- if (amd_iommu_unity_map || type == AMD_IOMMU_UNITY_MAP) {
- ASSERT(pageva == (pa_4K << MMU_PAGESHIFT));
- }
- }
-
- tear_level = -1;
- invalidate_pde = 0;
- invalidate_pte = 0;
- for (++level; level <= AMD_IOMMU_PGTABLE_MAXLEVEL; level++) {
- prev_index = pt->pt_index;
- ppt = pt->pt_parent;
- retval = amd_iommu_teardown_pdte(iommu, pt, index);
- switch (retval) {
- case PDTE_NOT_TORN:
- goto invalidate;
- case PDTE_TORN_DOWN:
- invalidate_pte = 1;
- goto invalidate;
- case PGTABLE_TORN_DOWN:
- invalidate_pte = 1;
- invalidate_pde = 1;
- tear_level = level;
- break;
- }
- index = prev_index;
- pt = ppt;
- }
-
-invalidate:
- /*
- * Now teardown the IOMMU HW caches if applicable
- */
- if (invalidate_pte) {
- cmdargs.ca_domainid = (uint16_t)domainid;
- if (amd_iommu_pageva_inval_all) {
- cmdargs.ca_addr = (uintptr_t)0x7FFFFFFFFFFFF000;
- flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL |
- AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S;
- } else if (invalidate_pde) {
- cmdargs.ca_addr =
- (uintptr_t)AMD_IOMMU_VA_INVAL(pageva, tear_level);
- flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL |
- AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S;
- } else {
- cmdargs.ca_addr = (uintptr_t)pageva;
- flags = 0;
- }
- if (amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_IOMMU_PAGES,
- &cmdargs, flags, 0) != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, "
- "rdip=%p. Failed to invalidate IOMMU HW cache "
- "for %s", f, driver, instance,
- iommu->aiomt_idx, domainid, (void *)rdip, path);
- error = DDI_FAILURE;
- goto out;
- }
- }
-
-passthru:
- if (tear_level == AMD_IOMMU_PGTABLE_MAXLEVEL) {
- error = amd_iommu_clear_devtbl_entry(iommu, rdip, domainid,
- deviceid, dp, domain_freed, path);
- } else {
- error = DDI_SUCCESS;
- }
-
-out:
- SYNC_FORDEV(iommu->aiomt_dmahdl);
-
- return (error);
-}
-
-static int
-cvt_bind_error(int error)
-{
- switch (error) {
- case DDI_DMA_MAPPED:
- case DDI_DMA_PARTIAL_MAP:
- case DDI_DMA_NORESOURCES:
- case DDI_DMA_NOMAPPING:
- break;
- default:
- cmn_err(CE_PANIC, "Unsupported error code: %d", error);
- /*NOTREACHED*/
- }
- return (error);
-}
-
-int
-amd_iommu_map_pa2va(amd_iommu_t *iommu, dev_info_t *rdip, ddi_dma_attr_t *attrp,
- struct ddi_dma_req *dmareq, uint64_t start_pa, uint64_t pa_sz,
- map_type_t type, uint64_t *start_vap, int km_flags)
-{
- pfn_t pfn_start;
- pfn_t pfn_end;
- pfn_t pfn;
- int alias;
- int32_t deviceid;
- domain_id_t domainid;
- amd_iommu_domain_t *dp;
- uint64_t end_pa;
- uint64_t start_va;
- uint64_t end_va;
- uint64_t pg_start;
- uint64_t pg_end;
- uint64_t pg;
- uint64_t va_sz;
- char *path;
- int error = DDI_DMA_NOMAPPING;
- const char *driver = ddi_driver_name(iommu->aiomt_dip);
- int instance = ddi_get_instance(iommu->aiomt_dip);
- const char *f = "amd_iommu_map_pa2va";
-
- ASSERT(pa_sz != 0);
-
- *start_vap = 0;
-
- ASSERT(rdip);
-
- path = kmem_alloc(MAXPATHLEN, km_flags);
- if (path == NULL) {
- error = DDI_DMA_NORESOURCES;
- goto out;
- }
- (void) ddi_pathname(rdip, path);
-
- /*
- * First get deviceid
- */
- if (amd_iommu_get_deviceid(iommu, rdip, &deviceid, &alias, path)
- != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p. "
- "Failed to get device ID for %s.", f, driver, instance,
- iommu->aiomt_idx, (void *)rdip, path);
- error = DDI_DMA_NOMAPPING;
- goto out;
- }
-
- /*
- * Next get the domain for this rdip
- */
- if (amd_iommu_get_domain(iommu, rdip, alias, deviceid, &domainid, path)
- != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p, path=%s. "
- "Failed to get domain.", f, driver, instance,
- iommu->aiomt_idx, (void *)rdip, path);
- error = DDI_DMA_NOMAPPING;
- goto out;
- }
-
- dp = amd_iommu_lookup_domain(iommu, domainid, type, km_flags);
- if (dp == NULL) {
- cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, rdip=%p. "
- "Failed to get device ID for %s.", f, driver, instance,
- iommu->aiomt_idx, domainid, (void *)rdip, path);
- error = DDI_DMA_NORESOURCES;
- goto out;
- }
-
- ASSERT(dp->d_domainid == domainid);
-
- pfn_start = start_pa >> MMU_PAGESHIFT;
-
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
- cmn_err(CE_NOTE, "pa = %p, pfn_new = %p, pfn_start = %p, "
- "pgshift = %d",
- (void *)(uintptr_t)start_pa,
- (void *)(uintptr_t)(start_pa >> MMU_PAGESHIFT),
- (void *)(uintptr_t)pfn_start, MMU_PAGESHIFT);
- }
-
- end_pa = start_pa + pa_sz - 1;
- pfn_end = end_pa >> MMU_PAGESHIFT;
-
- if (amd_iommu_unity_map || type == AMD_IOMMU_UNITY_MAP) {
- start_va = start_pa;
- end_va = end_pa;
- va_sz = pa_sz;
- *start_vap = start_va;
- } else {
- va_sz = mmu_ptob(pfn_end - pfn_start + 1);
- start_va = (uintptr_t)vmem_xalloc(dp->d_vmem, va_sz,
- MAX(attrp->dma_attr_align, MMU_PAGESIZE),
- 0,
- attrp->dma_attr_seg + 1,
- (void *)(uintptr_t)attrp->dma_attr_addr_lo,
- (void *)(uintptr_t)MIN((attrp->dma_attr_addr_hi + 1),
- AMD_IOMMU_SIZE_4G), /* XXX rollover */
- km_flags == KM_SLEEP ? VM_SLEEP : VM_NOSLEEP);
- if (start_va == 0) {
- cmn_err(CE_WARN, "%s: No VA resources",
- amd_iommu_modname);
- error = DDI_DMA_NORESOURCES;
- goto out;
- }
- ASSERT((start_va & MMU_PAGEOFFSET) == 0);
- end_va = start_va + va_sz - 1;
- *start_vap = start_va + (start_pa & MMU_PAGEOFFSET);
- }
-
- pg_start = start_va >> MMU_PAGESHIFT;
- pg_end = end_va >> MMU_PAGESHIFT;
-
- pg = pg_start;
- for (pfn = pfn_start; pfn <= pfn_end; pfn++, pg++) {
-
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
- cmn_err(CE_WARN, "%s: attempting to create page tables "
- "for pfn = %p, va = %p, path = %s",
- f, (void *)(uintptr_t)(pfn << MMU_PAGESHIFT),
- (void *)(uintptr_t)(pg << MMU_PAGESHIFT), path);
-
- }
-
- if (amd_iommu_unity_map || type == AMD_IOMMU_UNITY_MAP) {
- ASSERT(pfn == pg);
- }
-
- if ((error = amd_iommu_create_pgtables(iommu, rdip, dmareq,
- pg << MMU_PAGESHIFT,
- pfn << MMU_PAGESHIFT, deviceid, domainid, dp, path,
- km_flags)) != DDI_SUCCESS) {
- cmn_err(CE_WARN, "Failed to create_pgtables");
- goto out;
- }
-
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) {
- cmn_err(CE_WARN, "%s: successfuly created page tables "
- "for pfn = %p, vapg = %p, path = %s",
- f, (void *)(uintptr_t)pfn,
- (void *)(uintptr_t)pg, path);
- }
-
- }
- ASSERT(pg == pg_end + 1);
-
-
- if (amd_iommu_debug & AMD_IOMMU_DEBUG_PA2VA) {
- cmn_err(CE_NOTE, "pa=%p, va=%p",
- (void *)(uintptr_t)start_pa,
- (void *)(uintptr_t)(*start_vap));
- }
- error = DDI_DMA_MAPPED;
-
-out:
- kmem_free(path, MAXPATHLEN);
- return (cvt_bind_error(error));
-}
-
-int
-amd_iommu_unmap_va(amd_iommu_t *iommu, dev_info_t *rdip, uint64_t start_va,
- uint64_t va_sz, map_type_t type)
-{
- uint64_t end_va;
- uint64_t pg_start;
- uint64_t pg_end;
- uint64_t pg;
- uint64_t actual_sz;
- char *path;
- int pathfree;
- int alias;
- int32_t deviceid;
- domain_id_t domainid;
- amd_iommu_domain_t *dp;
- int error;
- int domain_freed;
- const char *driver = ddi_driver_name(iommu->aiomt_dip);
- int instance = ddi_get_instance(iommu->aiomt_dip);
- const char *f = "amd_iommu_unmap_va";
-
- if (amd_iommu_no_unmap)
- return (DDI_SUCCESS);
-
- path = kmem_alloc(MAXPATHLEN, KM_NOSLEEP);
- if (path) {
- (void) ddi_pathname(rdip, path);
- pathfree = 1;
- } else {
- pathfree = 0;
- path = "<path-mem-alloc-failed>";
- }
-
- /*
- * First get deviceid
- */
- if (amd_iommu_get_deviceid(iommu, rdip, &deviceid, &alias, path)
- != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p. "
- "Failed to get device ID for %s.", f, driver, instance,
- iommu->aiomt_idx, (void *)rdip, path);
- error = DDI_FAILURE;
- goto out;
- }
-
- /*
- * Next get the domain for this rdip
- */
- if (amd_iommu_get_domain(iommu, rdip, alias, deviceid, &domainid, path)
- != DDI_SUCCESS) {
- cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p, path=%s. "
- "Failed to get domain.", f, driver, instance,
- iommu->aiomt_idx, (void *)rdip, path);
- error = DDI_FAILURE;
- goto out;
- }
-
- /* should never result in domain allocation/vmem_create */
- dp = amd_iommu_lookup_domain(iommu, domainid, AMD_IOMMU_INVALID_MAP,
- KM_NOSLEEP);
- if (dp == NULL) {
- cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, rdip=%p. "
- "Failed to get device ID for %s.", f, driver, instance,
- iommu->aiomt_idx, domainid, (void *)rdip, path);
- error = DDI_FAILURE;
- goto out;
- }
-
- ASSERT(dp->d_domainid == domainid);
-
- pg_start = start_va >> MMU_PAGESHIFT;
- end_va = start_va + va_sz - 1;
- pg_end = end_va >> MMU_PAGESHIFT;
- actual_sz = (pg_end - pg_start + 1) << MMU_PAGESHIFT;
-
- domain_freed = 0;
- for (pg = pg_start; pg <= pg_end; pg++) {
- domain_freed = 0;
- if (amd_iommu_destroy_pgtables(iommu, rdip,
- pg << MMU_PAGESHIFT, deviceid, domainid, dp, type,
- &domain_freed, path) != DDI_SUCCESS) {
- error = DDI_FAILURE;
- goto out;
- }
- if (domain_freed) {
- ASSERT(pg == pg_end);
- break;
- }
- }
-
- /*
- * vmem_xalloc() must be paired with vmem_xfree
- */
- if (type == AMD_IOMMU_VMEM_MAP && !amd_iommu_unity_map) {
- vmem_xfree(dp->d_vmem,
- (void *)(uintptr_t)(pg_start << MMU_PAGESHIFT), actual_sz);
- }
-
- if (domain_freed)
- amd_iommu_teardown_domain(iommu, dp);
-
- error = DDI_SUCCESS;
-out:
- if (pathfree)
- kmem_free(path, MAXPATHLEN);
- return (error);
-}
--- a/usr/src/uts/i86pc/sys/amd_iommu.h Thu Sep 17 10:25:33 2009 +0800
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,56 +0,0 @@
-/*
- * CDDL HEADER START
- *
- * The contents of this file are subject to the terms of the
- * Common Development and Distribution License (the "License").
- * You may not use this file except in compliance with the License.
- *
- * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
- * or http://www.opensolaris.org/os/licensing.
- * See the License for the specific language governing permissions
- * and limitations under the License.
- *
- * When distributing Covered Code, include this CDDL HEADER in each
- * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
- * If applicable, add the following below this CDDL HEADER, with the
- * fields enclosed by brackets "[]" replaced with your own identifying
- * information: Portions Copyright [yyyy] [name of copyright owner]
- *
- * CDDL HEADER END
- */
-/*
- * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
- * Use is subject to license terms.
- */
-
-#ifndef _SYS_AMD_IOMMU_H
-#define _SYS_AMD_IOMMU_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <sys/sunddi.h>
-#include <sys/iommulib.h>
-
-#ifdef _KERNEL
-
-typedef struct amd_iommu_state {
- int aioms_instance; /* instance */
- dev_info_t *aioms_devi; /* dip */
- struct amd_iommu *aioms_iommu_start; /* start of list of IOMMUs */
- struct amd_iommu *aioms_iommu_end; /* end of list of IOMMUs */
- int aioms_nunits; /* # of IOMMUs in function */
-} amd_iommu_state_t;
-
-int amd_iommu_setup(dev_info_t *dip, amd_iommu_state_t *statep);
-int amd_iommu_teardown(dev_info_t *dip, amd_iommu_state_t *statep);
-int amd_iommu_lookup_src_bdf(uint16_t bdf, uint16_t *src_bdfp);
-
-#endif /* _KERNEL */
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _SYS_AMD_IOMMU_H */