/*
 * Copyright (c) 2010, Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program; If not, see <http://www.gnu.org/licenses/>.
 *
 * Author: Allen Kay <allen.m.kay@intel.com>
 */

#include <xen/irq.h>
#include <xen/sched.h>
#include <xen/xmalloc.h>
#include <xen/domain_page.h>
#include <xen/iommu.h>
#include <xen/numa.h>
#include <xen/softirq.h>
#include <xen/time.h>
#include <xen/pci.h>
#include <xen/pci_ids.h>
#include <xen/pci_regs.h>
#include <xen/keyhandler.h>
#include <asm/msi.h>
#include <asm/irq.h>
#include <asm/pci.h>
#include <mach_apic.h>
#include "iommu.h"
#include "dmar.h"
#include "extern.h"
#include "vtd.h"

#define IOH_DEV      0
#define IGD_DEV      2

#define IGD_BAR_MASK 0xFFFFFFFFFFFF0000
#define GGC 0x52
#define GGC_MEMORY_VT_ENABLED  (0x8 << 8)

#define IS_CTG(id)    (id == 0x2a408086)
#define IS_ILK(id)    (id == 0x00408086 || id == 0x00448086 || id== 0x00628086 || id == 0x006A8086)
#define IS_CPT(id)    (id == 0x01008086 || id == 0x01048086)

/* SandyBridge IGD timeouts in milliseconds */
#define SNB_IGD_TIMEOUT_LEGACY    1000
#define SNB_IGD_TIMEOUT            670
static unsigned int snb_igd_timeout;

static u32 __read_mostly ioh_id;
static u32 __initdata igd_id;
bool_t __read_mostly rwbf_quirk;
static bool_t __read_mostly is_cantiga_b3;
static bool_t __read_mostly is_snb_gfx;
static u8 *__read_mostly igd_reg_va;
static spinlock_t igd_lock;

/*
 * QUIRK to workaround Xen boot issue on Calpella/Ironlake OEM BIOS
 * not enabling VT-d properly in IGD.  The workaround is to not enabling
 * IGD VT-d translation if VT is not enabled in IGD.
 */
int is_igd_vt_enabled_quirk(void)
{
    u16 ggc;

    if ( !IS_ILK(ioh_id) )
        return 1;

    /* integrated graphics on Intel platforms is located at 0:2.0 */
    ggc = pci_conf_read16(0, 0, IGD_DEV, 0, GGC);
    return ( ggc & GGC_MEMORY_VT_ENABLED ? 1 : 0 );
}

/*
 * QUIRK to workaround cantiga VT-d buffer flush issue.
 * The workaround is to force write buffer flush even if
 * VT-d capability indicates it is not required.
 */
static void __init cantiga_b3_errata_init(void)
{
    u16 vid;
    u8 did_hi, rid;

    vid = pci_conf_read16(0, 0, IGD_DEV, 0, 0);
    if ( vid != 0x8086 )
        return;

    did_hi = pci_conf_read8(0, 0, IGD_DEV, 0, 3);
    rid = pci_conf_read8(0, 0, IGD_DEV, 0, 8);

    if ( (did_hi == 0x2A) && (rid == 0x7) )
        is_cantiga_b3 = 1;
}

/* check for Sandybridge IGD device ID's */
static void __init snb_errata_init(void)
{
    is_snb_gfx = IS_SNB_GFX(igd_id);
    spin_lock_init(&igd_lock);
}

/*
 * QUIRK to workaround Cantiga IGD VT-d low power errata.
 * This errata impacts IGD assignment on Cantiga systems
 * and can potentially cause VT-d operations to hang.
 * The workaround is to access an IGD PCI config register
 * to get IGD out of low power state before VT-d translation
 * enable/disable and IOTLB flushes.
 */

/*
 * map IGD MMIO+0x2000 page to allow Xen access to IGD 3D register.
 */
static void __init map_igd_reg(void)
{
    u64 igd_mmio;

    if ( !is_cantiga_b3 && !is_snb_gfx )
        return;

    if ( igd_reg_va )
        return;

    igd_mmio   = pci_conf_read32(0, 0, IGD_DEV, 0, PCI_BASE_ADDRESS_1);
    igd_mmio <<= 32;
    igd_mmio  += pci_conf_read32(0, 0, IGD_DEV, 0, PCI_BASE_ADDRESS_0);
    igd_reg_va = ioremap(igd_mmio & IGD_BAR_MASK, 0x3000);
}

/*
 * force IGD to exit low power mode by accessing a IGD 3D regsiter.
 */
static int cantiga_vtd_ops_preamble(struct iommu* iommu)
{
    struct intel_iommu *intel = iommu->intel;
    struct acpi_drhd_unit *drhd = intel ? intel->drhd : NULL;

    if ( !is_igd_drhd(drhd) || !is_cantiga_b3 )
        return 0;

    if ( !igd_reg_va )
        return 0;

    /*
     * Read IGD register at IGD MMIO + 0x20A4 to force IGD
     * to exit low power state.
     */
    return *(volatile int *)(igd_reg_va + 0x20A4);
}

/*
 * Sandybridge RC6 power management inhibit state erratum.
 * This can cause power high power consumption.
 * Workaround is to prevent graphics get into RC6
 * state when doing VT-d IOTLB operations, do the VT-d
 * IOTLB operation, and then re-enable RC6 state.
 *
 * This quirk is enabled with the snb_igd_quirk command
 * line parameter.  Specifying snb_igd_quirk with no value
 * (or any of the standard boolean values) enables this
 * quirk and sets the timeout to the legacy timeout of
 * 1000 msec.  Setting this parameter to the string
 * "cap" enables this quirk and sets the timeout to
 * the theoretical maximum of 670 msec.  Setting this
 * parameter to a numerical value enables the quirk and
 * sets the timeout to that numerical number of msecs.
 */
static void snb_vtd_ops_preamble(struct iommu* iommu)
{
    struct intel_iommu *intel = iommu->intel;
    struct acpi_drhd_unit *drhd = intel ? intel->drhd : NULL;
    s_time_t start_time;

    if ( !is_igd_drhd(drhd) || !is_snb_gfx )
        return;

    if ( !igd_reg_va )
        return;

    *(volatile u32 *)(igd_reg_va + 0x2054) = 0x000FFFFF;
    *(volatile u32 *)(igd_reg_va + 0x2700) = 0;

    start_time = NOW();
    while ( (*(volatile u32 *)(igd_reg_va + 0x22AC) & 0xF) != 0 )
    {
        if ( NOW() > start_time + snb_igd_timeout )
        {
            dprintk(XENLOG_INFO VTDPREFIX,
                    "snb_vtd_ops_preamble: failed to disable idle handshake\n");
            break;
        }
        cpu_relax();
    }

    *(volatile u32 *)(igd_reg_va + 0x2050) = 0x10001;
}

static void snb_vtd_ops_postamble(struct iommu* iommu)
{
    struct intel_iommu *intel = iommu->intel;
    struct acpi_drhd_unit *drhd = intel ? intel->drhd : NULL;

    if ( !is_igd_drhd(drhd) || !is_snb_gfx )
        return;

    if ( !igd_reg_va )
        return;

    *(volatile u32 *)(igd_reg_va + 0x2054) = 0xA;
    *(volatile u32 *)(igd_reg_va + 0x2050) = 0x10000;
}

/*
 * call before VT-d translation enable and IOTLB flush operations.
 */

void vtd_ops_preamble_quirk(struct iommu* iommu)
{
    cantiga_vtd_ops_preamble(iommu);
    if ( snb_igd_timeout != 0 )
    {
        spin_lock(&igd_lock);

        /* match unlock in postamble */
        snb_vtd_ops_preamble(iommu);
    }
}

/*
 * call after VT-d translation enable and IOTLB flush operations.
 */
void vtd_ops_postamble_quirk(struct iommu* iommu)
{
    if ( snb_igd_timeout != 0 )
    {
        snb_vtd_ops_postamble(iommu);

        /* match the lock in preamble */
        spin_unlock(&igd_lock);
    }
}

static int __init parse_snb_timeout(const char *s)
{
    int t;
    const char *q = NULL;

    t = parse_bool(s, NULL);
    if ( t < 0 )
    {
        if ( *s == '\0' )
            t = SNB_IGD_TIMEOUT_LEGACY;
        else if ( strcmp(s, "cap") == 0 )
            t = SNB_IGD_TIMEOUT;
        else
            t = strtoul(s, &q, 0);
    }
    else
        t = t ? SNB_IGD_TIMEOUT_LEGACY : 0;
    snb_igd_timeout = MILLISECS(t);

    return (q && *q) ? -EINVAL : 0;
}
custom_param("snb_igd_quirk", parse_snb_timeout);

/* 5500/5520/X58 Chipset Interrupt remapping errata, for stepping B-3.
 * Fixed in stepping C-2. */
static void __init tylersburg_intremap_quirk(void)
{
    uint32_t bus, device;
    uint8_t rev;

    for ( bus = 0; bus < 0x100; bus++ )
    {
        /* Match on System Management Registers on Device 20 Function 0 */
        device = pci_conf_read32(0, bus, 20, 0, PCI_VENDOR_ID);
        rev = pci_conf_read8(0, bus, 20, 0, PCI_REVISION_ID);

        if ( rev == 0x13 && device == 0x342e8086 )
        {
            printk(XENLOG_WARNING VTDPREFIX
                   "Disabling IOMMU due to Intel 5500/5520/X58 Chipset errata #47, #53\n");
            iommu_enable = 0;
            break;
        }
    }
}

/* initialize platform identification flags */
void __init platform_quirks_init(void)
{
    ioh_id = pci_conf_read32(0, 0, IOH_DEV, 0, 0);
    igd_id = pci_conf_read32(0, 0, IGD_DEV, 0, 0);

    /* Mobile 4 Series Chipset neglects to set RWBF capability. */
    if ( ioh_id == 0x2a408086 )
    {
        dprintk(XENLOG_INFO VTDPREFIX, "DMAR: Forcing write-buffer flush\n");
        rwbf_quirk = 1;
    }

    /* initialize cantiga B3 identification */
    cantiga_b3_errata_init();

    snb_errata_init();

    /* ioremap IGD MMIO+0x2000 page */
    map_igd_reg();

    /* Tylersburg interrupt remap quirk */
    if ( iommu_intremap )
        tylersburg_intremap_quirk();
}

/*
 * QUIRK to workaround wifi direct assignment issue.  This issue
 * impacts only cases where Intel integrated wifi device is directly
 * is directly assigned to a guest.
 *
 * The workaround is to map ME phantom device 0:3.7 or 0:22.7
 * to the ME vt-d engine if detect the user is trying to directly
 * assigning Intel integrated wifi device to a guest.
 */

static int __must_check map_me_phantom_function(struct domain *domain,
                                                u32 dev, int map)
{
    struct acpi_drhd_unit *drhd;
    struct pci_dev *pdev;
    int rc;

    /* find ME VT-d engine base on a real ME device */
    pdev = pci_get_pdev(0, 0, PCI_DEVFN(dev, 0));
    drhd = acpi_find_matched_drhd_unit(pdev);

    /* map or unmap ME phantom function */
    if ( map )
        rc = domain_context_mapping_one(domain, drhd->iommu, 0,
                                        PCI_DEVFN(dev, 7), NULL);
    else
        rc = domain_context_unmap_one(domain, drhd->iommu, 0,
                                      PCI_DEVFN(dev, 7));

    return rc;
}

int me_wifi_quirk(struct domain *domain, u8 bus, u8 devfn, int map)
{
    u32 id;
    int rc = 0;

    id = pci_conf_read32(0, 0, 0, 0, 0);
    if ( IS_CTG(id) )
    {
        /* quit if ME does not exist */
        if ( pci_conf_read32(0, 0, 3, 0, 0) == 0xffffffff )
            return 0;

        /* if device is WLAN device, map ME phantom device 0:3.7 */
        id = pci_conf_read32(0, bus, PCI_SLOT(devfn), PCI_FUNC(devfn), 0);
        switch (id)
        {
            case 0x42328086:
            case 0x42358086:
            case 0x42368086:
            case 0x42378086:
            case 0x423a8086:
            case 0x423b8086:
            case 0x423c8086:
            case 0x423d8086:
                rc = map_me_phantom_function(domain, 3, map);
                break;
            default:
                break;
        }
    }
    else if ( IS_ILK(id) || IS_CPT(id) )
    {
        /* quit if ME does not exist */
        if ( pci_conf_read32(0, 0, 22, 0, 0) == 0xffffffff )
            return 0;

        /* if device is WLAN device, map ME phantom device 0:22.7 */
        id = pci_conf_read32(0, bus, PCI_SLOT(devfn), PCI_FUNC(devfn), 0);
        switch (id)
        {
            case 0x00878086:        /* Kilmer Peak */
            case 0x00898086:
            case 0x00828086:        /* Taylor Peak */
            case 0x00858086:
            case 0x008F8086:        /* Rainbow Peak */
            case 0x00908086:
            case 0x00918086:
            case 0x42388086:        /* Puma Peak */
            case 0x422b8086:
            case 0x422c8086:
                rc = map_me_phantom_function(domain, 22, map);
                break;
            default:
                break;
        }
    }

    return rc;
}

void pci_vtd_quirk(const struct pci_dev *pdev)
{
    int seg = pdev->seg;
    int bus = pdev->bus;
    int dev = PCI_SLOT(pdev->devfn);
    int func = PCI_FUNC(pdev->devfn);
    int pos;
    bool_t ff;
    u32 val, val2;
    u64 bar;
    paddr_t pa;
    const char *action;

    if ( pci_conf_read16(seg, bus, dev, func, PCI_VENDOR_ID) !=
         PCI_VENDOR_ID_INTEL )
        return;

    switch ( pci_conf_read16(seg, bus, dev, func, PCI_DEVICE_ID) )
    {
    /*
     * Mask reporting Intel VT-d faults to IOH core logic:
     *   - Some platform escalates VT-d faults to platform errors.
     *   - This can cause system failure upon non-fatal VT-d faults.
     *   - Potential security issue if malicious guest trigger VT-d faults.
     */
    case 0x342e: /* Tylersburg chipset (Nehalem / Westmere systems) */
    case 0x3728: /* Xeon C5500/C3500 (JasperForest) */
    case 0x3c28: /* Sandybridge */
        val = pci_conf_read32(seg, bus, dev, func, 0x1AC);
        pci_conf_write32(seg, bus, dev, func, 0x1AC, val | (1 << 31));
        printk(XENLOG_INFO "Masked VT-d error signaling on %04x:%02x:%02x.%u\n",
               seg, bus, dev, func);
        break;

    /* Tylersburg (EP)/Boxboro (MP) chipsets (NHM-EP/EX, WSM-EP/EX) */
    case 0x3400 ... 0x3407: /* host bridges */
    case 0x3408 ... 0x3411: case 0x3420 ... 0x3421: /* root ports */
    /* JasperForest (Intel Xeon Processor C5500/C3500 */
    case 0x3700 ... 0x370f: /* host bridges */
    case 0x3720 ... 0x3724: /* root ports */
    /* Sandybridge-EP (Romley) */
    case 0x3c00: /* host bridge */
    case 0x3c01 ... 0x3c0b: /* root ports */
        pos = pci_find_ext_capability(seg, bus, pdev->devfn,
                                      PCI_EXT_CAP_ID_ERR);
        if ( !pos )
        {
            pos = pci_find_ext_capability(seg, bus, pdev->devfn,
                                          PCI_EXT_CAP_ID_VNDR);
            while ( pos )
            {
                val = pci_conf_read32(seg, bus, dev, func, pos + PCI_VNDR_HEADER);
                if ( PCI_VNDR_HEADER_ID(val) == 4 && PCI_VNDR_HEADER_REV(val) == 1 )
                {
                    pos += PCI_VNDR_HEADER;
                    break;
                }
                pos = pci_find_next_ext_capability(seg, bus, pdev->devfn, pos,
                                                   PCI_EXT_CAP_ID_VNDR);
            }
            ff = 0;
        }
        else
            ff = pcie_aer_get_firmware_first(pdev);
        if ( !pos )
        {
            printk(XENLOG_WARNING "%04x:%02x:%02x.%u without AER capability?\n",
                   seg, bus, dev, func);
            break;
        }

        val = pci_conf_read32(seg, bus, dev, func, pos + PCI_ERR_UNCOR_MASK);
        val2 = pci_conf_read32(seg, bus, dev, func, pos + PCI_ERR_COR_MASK);
        if ( (val & PCI_ERR_UNC_UNSUP) && (val2 & PCI_ERR_COR_ADV_NFAT) )
            action = "Found masked";
        else if ( !ff )
        {
            pci_conf_write32(seg, bus, dev, func, pos + PCI_ERR_UNCOR_MASK,
                             val | PCI_ERR_UNC_UNSUP);
            pci_conf_write32(seg, bus, dev, func, pos + PCI_ERR_COR_MASK,
                             val2 | PCI_ERR_COR_ADV_NFAT);
            action = "Masked";
        }
        else
            action = "Must not mask";

        /* XPUNCERRMSK Send Completion with Unsupported Request */
        val = pci_conf_read32(seg, bus, dev, func, 0x20c);
        pci_conf_write32(seg, bus, dev, func, 0x20c, val | (1 << 4));

        printk(XENLOG_INFO "%s UR signaling on %04x:%02x:%02x.%u\n",
               action, seg, bus, dev, func);
        break;

    case 0x0040: case 0x0044: case 0x0048: /* Nehalem/Westmere */
    case 0x0100: case 0x0104: case 0x0108: /* Sandybridge */
    case 0x0150: case 0x0154: case 0x0158: /* Ivybridge */
    case 0x0a00: case 0x0a04: case 0x0a08: case 0x0a0f: /* Haswell ULT */
    case 0x0c00: case 0x0c04: case 0x0c08: case 0x0c0f: /* Haswell */
    case 0x0d00: case 0x0d04: case 0x0d08: case 0x0d0f: /* Haswell */
    case 0x1600: case 0x1604: case 0x1608: case 0x160f: /* Broadwell */
    case 0x1610: case 0x1614: case 0x1618: /* Broadwell */
    case 0x1900: case 0x1904: case 0x1908: case 0x190c: case 0x190f: /* Skylake */
    case 0x1910: case 0x1918: case 0x191f: /* Skylake */
        bar = pci_conf_read32(seg, bus, dev, func, 0x6c);
        bar = (bar << 32) | pci_conf_read32(seg, bus, dev, func, 0x68);
        pa = bar & 0x7ffffff000UL; /* bits 12...38 */
        if ( (bar & 1) && pa &&
             page_is_ram_type(paddr_to_pfn(pa), RAM_TYPE_RESERVED) )
        {
            u32 __iomem *va = ioremap(pa, PAGE_SIZE);

            if ( va )
            {
                __set_bit(0x1c8 * 8 + 20, va);
                iounmap(va);
                printk(XENLOG_INFO "Masked UR signaling on %04x:%02x:%02x.%u\n",
                       seg, bus, dev, func);
            }
            else
                printk(XENLOG_ERR "Could not map %"PRIpaddr" for %04x:%02x:%02x.%u\n",
                       pa, seg, bus, dev, func);
        }
        else
            printk(XENLOG_WARNING "Bogus DMIBAR %#"PRIx64" on %04x:%02x:%02x.%u\n",
                   bar, seg, bus, dev, func);
        break;
    }
}