]> git.ipfire.org Git - thirdparty/linux.git/blame - drivers/ata/sata_nv.c
projects / thirdparty / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
x86: mm: ptdump: calculate effective permissions correctly
[thirdparty/linux.git] / drivers / ata / sata_nv.c
CommitLineData
c82ee6d3 1// SPDX-License-Identifier: GPL-2.0-or-later
1da177e4
LT		 2/*
3 * sata_nv.c - NVIDIA nForce SATA
4 *
5 * Copyright 2004 NVIDIA Corp. All rights reserved.
6 * Copyright 2004 Andrew Chew
7 *
af36d7f0 8 * libata documentation is available via 'make {ps|pdf}docs',
19285f3c 9 * as Documentation/driver-api/libata.rst
af36d7f0
JG		 10 *
11 * No hardware documentation available outside of NVIDIA.
12 * This driver programs the NVIDIA SATA controller in a similar
13 * fashion as with other PCI IDE BMDMA controllers, with a few
14 * NV-specific details such as register offsets, SATA phy location,
15 * hotplug info, etc.
16 *
fbbb262d
RH		 17 * CK804/MCP04 controllers support an alternate programming interface
18 * similar to the ADMA specification (with some modifications).
19 * This allows the use of NCQ. Non-DMA-mapped ATA commands are still
20 * sent through the legacy interface.
1da177e4
LT		 21 */
22
1da177e4
LT		 23#include <linux/kernel.h>
24#include <linux/module.h>
5a0e3ad6 25#include <linux/gfp.h>
1da177e4 26#include <linux/pci.h>
1da177e4
LT		 27#include <linux/blkdev.h>
28#include <linux/delay.h>
29#include <linux/interrupt.h>
a9524a76 30#include <linux/device.h>
1da177e4 31#include <scsi/scsi_host.h>
fbbb262d 32#include <scsi/scsi_device.h>
1da177e4
LT		 33#include <linux/libata.h>
34
35#define DRV_NAME "sata_nv"
2a3103ce 36#define DRV_VERSION "3.5"
fbbb262d
RH		 37
38#define NV_ADMA_DMA_BOUNDARY 0xffffffffUL
1da177e4 39
10ad05df 40enum {
0d5ff566
TH		 41 NV_MMIO_BAR = 5,
42
10ad05df 43 NV_PORTS = 2,
14bdef98
EIB		 44 NV_PIO_MASK = ATA_PIO4,
45 NV_MWDMA_MASK = ATA_MWDMA2,
46 NV_UDMA_MASK = ATA_UDMA6,
10ad05df
JG		 47 NV_PORT0_SCR_REG_OFFSET = 0x00,
48 NV_PORT1_SCR_REG_OFFSET = 0x40,
1da177e4 49
27e4b274 50 /* INT_STATUS/ENABLE */
10ad05df 51 NV_INT_STATUS = 0x10,
10ad05df 52 NV_INT_ENABLE = 0x11,
27e4b274 53 NV_INT_STATUS_CK804 = 0x440,
10ad05df 54 NV_INT_ENABLE_CK804 = 0x441,
1da177e4 55
27e4b274
TH		 56 /* INT_STATUS/ENABLE bits */
57 NV_INT_DEV = 0x01,
58 NV_INT_PM = 0x02,
59 NV_INT_ADDED = 0x04,
60 NV_INT_REMOVED = 0x08,
61
62 NV_INT_PORT_SHIFT = 4, /* each port occupies 4 bits */
63
39f87582 64 NV_INT_ALL = 0x0f,
5a44efff
TH		 65 NV_INT_MASK = NV_INT_DEV |
66 NV_INT_ADDED | NV_INT_REMOVED,
39f87582 67
27e4b274 68 /* INT_CONFIG */
10ad05df
JG		 69 NV_INT_CONFIG = 0x12,
70 NV_INT_CONFIG_METHD = 0x01, // 0 = INT, 1 = SMI
1da177e4 71
10ad05df
JG		 72 // For PCI config register 20
73 NV_MCP_SATA_CFG_20 = 0x50,
74 NV_MCP_SATA_CFG_20_SATA_SPACE_EN = 0x04,
fbbb262d
RH		 75 NV_MCP_SATA_CFG_20_PORT0_EN = (1 << 17),
76 NV_MCP_SATA_CFG_20_PORT1_EN = (1 << 16),
77 NV_MCP_SATA_CFG_20_PORT0_PWB_EN = (1 << 14),
78 NV_MCP_SATA_CFG_20_PORT1_PWB_EN = (1 << 12),
79
80 NV_ADMA_MAX_CPBS = 32,
81 NV_ADMA_CPB_SZ = 128,
82 NV_ADMA_APRD_SZ = 16,
83 NV_ADMA_SGTBL_LEN = (1024 - NV_ADMA_CPB_SZ) /
84 NV_ADMA_APRD_SZ,
85 NV_ADMA_SGTBL_TOTAL_LEN = NV_ADMA_SGTBL_LEN + 5,
86 NV_ADMA_SGTBL_SZ = NV_ADMA_SGTBL_LEN * NV_ADMA_APRD_SZ,
87 NV_ADMA_PORT_PRIV_DMA_SZ = NV_ADMA_MAX_CPBS *
88 (NV_ADMA_CPB_SZ + NV_ADMA_SGTBL_SZ),
89
90 /* BAR5 offset to ADMA general registers */
91 NV_ADMA_GEN = 0x400,
92 NV_ADMA_GEN_CTL = 0x00,
93 NV_ADMA_NOTIFIER_CLEAR = 0x30,
94
95 /* BAR5 offset to ADMA ports */
96 NV_ADMA_PORT = 0x480,
97
98 /* size of ADMA port register space */
99 NV_ADMA_PORT_SIZE = 0x100,
100
101 /* ADMA port registers */
102 NV_ADMA_CTL = 0x40,
103 NV_ADMA_CPB_COUNT = 0x42,
104 NV_ADMA_NEXT_CPB_IDX = 0x43,
105 NV_ADMA_STAT = 0x44,
106 NV_ADMA_CPB_BASE_LOW = 0x48,
107 NV_ADMA_CPB_BASE_HIGH = 0x4C,
108 NV_ADMA_APPEND = 0x50,
109 NV_ADMA_NOTIFIER = 0x68,
110 NV_ADMA_NOTIFIER_ERROR = 0x6C,
111
112 /* NV_ADMA_CTL register bits */
113 NV_ADMA_CTL_HOTPLUG_IEN = (1 << 0),
114 NV_ADMA_CTL_CHANNEL_RESET = (1 << 5),
115 NV_ADMA_CTL_GO = (1 << 7),
116 NV_ADMA_CTL_AIEN = (1 << 8),
117 NV_ADMA_CTL_READ_NON_COHERENT = (1 << 11),
118 NV_ADMA_CTL_WRITE_NON_COHERENT = (1 << 12),
119
120 /* CPB response flag bits */
121 NV_CPB_RESP_DONE = (1 << 0),
122 NV_CPB_RESP_ATA_ERR = (1 << 3),
123 NV_CPB_RESP_CMD_ERR = (1 << 4),
124 NV_CPB_RESP_CPB_ERR = (1 << 7),
125
126 /* CPB control flag bits */
127 NV_CPB_CTL_CPB_VALID = (1 << 0),
128 NV_CPB_CTL_QUEUE = (1 << 1),
129 NV_CPB_CTL_APRD_VALID = (1 << 2),
130 NV_CPB_CTL_IEN = (1 << 3),
131 NV_CPB_CTL_FPDMA = (1 << 4),
132
133 /* APRD flags */
134 NV_APRD_WRITE = (1 << 1),
135 NV_APRD_END = (1 << 2),
136 NV_APRD_CONT = (1 << 3),
137
138 /* NV_ADMA_STAT flags */
139 NV_ADMA_STAT_TIMEOUT = (1 << 0),
140 NV_ADMA_STAT_HOTUNPLUG = (1 << 1),
141 NV_ADMA_STAT_HOTPLUG = (1 << 2),
142 NV_ADMA_STAT_CPBERR = (1 << 4),
143 NV_ADMA_STAT_SERROR = (1 << 5),
144 NV_ADMA_STAT_CMD_COMPLETE = (1 << 6),
145 NV_ADMA_STAT_IDLE = (1 << 8),
146 NV_ADMA_STAT_LEGACY = (1 << 9),
147 NV_ADMA_STAT_STOPPED = (1 << 10),
148 NV_ADMA_STAT_DONE = (1 << 12),
149 NV_ADMA_STAT_ERR = NV_ADMA_STAT_CPBERR |
2dcb407e 150 NV_ADMA_STAT_TIMEOUT,
fbbb262d
RH		 151
152 /* port flags */
153 NV_ADMA_PORT_REGISTER_MODE = (1 << 0),
2dec7555 154 NV_ADMA_ATAPI_SETUP_COMPLETE = (1 << 1),
fbbb262d 155
f140f0f1
KL		 156 /* MCP55 reg offset */
157 NV_CTL_MCP55 = 0x400,
158 NV_INT_STATUS_MCP55 = 0x440,
159 NV_INT_ENABLE_MCP55 = 0x444,
160 NV_NCQ_REG_MCP55 = 0x448,
161
162 /* MCP55 */
163 NV_INT_ALL_MCP55 = 0xffff,
164 NV_INT_PORT_SHIFT_MCP55 = 16, /* each port occupies 16 bits */
165 NV_INT_MASK_MCP55 = NV_INT_ALL_MCP55 & 0xfffd,
166
167 /* SWNCQ ENABLE BITS*/
168 NV_CTL_PRI_SWNCQ = 0x02,
169 NV_CTL_SEC_SWNCQ = 0x04,
170
171 /* SW NCQ status bits*/
172 NV_SWNCQ_IRQ_DEV = (1 << 0),
173 NV_SWNCQ_IRQ_PM = (1 << 1),
174 NV_SWNCQ_IRQ_ADDED = (1 << 2),
175 NV_SWNCQ_IRQ_REMOVED = (1 << 3),
176
177 NV_SWNCQ_IRQ_BACKOUT = (1 << 4),
178 NV_SWNCQ_IRQ_SDBFIS = (1 << 5),
179 NV_SWNCQ_IRQ_DHREGFIS = (1 << 6),
180 NV_SWNCQ_IRQ_DMASETUP = (1 << 7),
181
182 NV_SWNCQ_IRQ_HOTPLUG = NV_SWNCQ_IRQ_ADDED |
183 NV_SWNCQ_IRQ_REMOVED,
184
fbbb262d
RH		 185};
186
187/* ADMA Physical Region Descriptor - one SG segment */
188struct nv_adma_prd {
189 __le64 addr;
190 __le32 len;
191 u8 flags;
192 u8 packet_len;
193 __le16 reserved;
194};
195
196enum nv_adma_regbits {
197 CMDEND = (1 << 15), /* end of command list */
198 WNB = (1 << 14), /* wait-not-BSY */
199 IGN = (1 << 13), /* ignore this entry */
200 CS1n = (1 << (4 + 8)), /* std. PATA signals follow... */
201 DA2 = (1 << (2 + 8)),
202 DA1 = (1 << (1 + 8)),
203 DA0 = (1 << (0 + 8)),
204};
205
206/* ADMA Command Parameter Block
207 The first 5 SG segments are stored inside the Command Parameter Block itself.
208 If there are more than 5 segments the remainder are stored in a separate
209 memory area indicated by next_aprd. */
210struct nv_adma_cpb {
211 u8 resp_flags; /* 0 */
212 u8 reserved1; /* 1 */
213 u8 ctl_flags; /* 2 */
214 /* len is length of taskfile in 64 bit words */
2dcb407e 215 u8 len; /* 3 */
fbbb262d
RH		 216 u8 tag; /* 4 */
217 u8 next_cpb_idx; /* 5 */
218 __le16 reserved2; /* 6-7 */
219 __le16 tf[12]; /* 8-31 */
220 struct nv_adma_prd aprd[5]; /* 32-111 */
221 __le64 next_aprd; /* 112-119 */
222 __le64 reserved3; /* 120-127 */
10ad05df 223};
1da177e4 224
fbbb262d
RH		 225
226struct nv_adma_port_priv {
227 struct nv_adma_cpb *cpb;
228 dma_addr_t cpb_dma;
229 struct nv_adma_prd *aprd;
230 dma_addr_t aprd_dma;
2dcb407e
JG		 231 void __iomem *ctl_block;
232 void __iomem *gen_block;
233 void __iomem *notifier_clear_block;
8959d300 234 u64 adma_dma_mask;
fbbb262d 235 u8 flags;
5e5c74a5 236 int last_issue_ncq;
fbbb262d
RH		 237};
238
cdf56bcf
RH		 239struct nv_host_priv {
240 unsigned long type;
241};
242
f140f0f1
KL		 243struct defer_queue {
244 u32 defer_bits;
245 unsigned int head;
246 unsigned int tail;
247 unsigned int tag[ATA_MAX_QUEUE];
248};
249
250enum ncq_saw_flag_list {
251 ncq_saw_d2h = (1U << 0),
252 ncq_saw_dmas = (1U << 1),
253 ncq_saw_sdb = (1U << 2),
254 ncq_saw_backout = (1U << 3),
255};
256
257struct nv_swncq_port_priv {
f60d7011 258 struct ata_bmdma_prd *prd; /* our SG list */
f140f0f1
KL		 259 dma_addr_t prd_dma; /* and its DMA mapping */
260 void __iomem *sactive_block;
261 void __iomem *irq_block;
262 void __iomem *tag_block;
263 u32 qc_active;
264
265 unsigned int last_issue_tag;
266
267 /* fifo circular queue to store deferral command */
268 struct defer_queue defer_queue;
269
270 /* for NCQ interrupt analysis */
271 u32 dhfis_bits;
272 u32 dmafis_bits;
273 u32 sdbfis_bits;
274
275 unsigned int ncq_flags;
276};
277
278
5796d1c4 279#define NV_ADMA_CHECK_INTR(GCTL, PORT) ((GCTL) & (1 << (19 + (12 * (PORT)))))
fbbb262d 280
2dcb407e 281static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
58eb8cd5 282#ifdef CONFIG_PM_SLEEP
cdf56bcf 283static int nv_pci_device_resume(struct pci_dev *pdev);
438ac6d5 284#endif
cca3974e 285static void nv_ck804_host_stop(struct ata_host *host);
7d12e780
DH		 286static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance);
287static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance);
288static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance);
82ef04fb
TH		 289static int nv_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val);
290static int nv_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val);
1da177e4 291
7f4774b3
TH		 292static int nv_hardreset(struct ata_link *link, unsigned int *class,
293 unsigned long deadline);
39f87582
TH		 294static void nv_nf2_freeze(struct ata_port *ap);
295static void nv_nf2_thaw(struct ata_port *ap);
296static void nv_ck804_freeze(struct ata_port *ap);
297static void nv_ck804_thaw(struct ata_port *ap);
fbbb262d 298static int nv_adma_slave_config(struct scsi_device *sdev);
2dec7555 299static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc);
95364f36 300static enum ata_completion_errors nv_adma_qc_prep(struct ata_queued_cmd *qc);
fbbb262d
RH		 301static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc);
302static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance);
303static void nv_adma_irq_clear(struct ata_port *ap);
304static int nv_adma_port_start(struct ata_port *ap);
305static void nv_adma_port_stop(struct ata_port *ap);
438ac6d5 306#ifdef CONFIG_PM
cdf56bcf
RH		 307static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg);
308static int nv_adma_port_resume(struct ata_port *ap);
438ac6d5 309#endif
53014e25
RH		 310static void nv_adma_freeze(struct ata_port *ap);
311static void nv_adma_thaw(struct ata_port *ap);
fbbb262d
RH		 312static void nv_adma_error_handler(struct ata_port *ap);
313static void nv_adma_host_stop(struct ata_host *host);
f5ecac2d 314static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc);
f2fb344b 315static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf);
39f87582 316
f140f0f1
KL		 317static void nv_mcp55_thaw(struct ata_port *ap);
318static void nv_mcp55_freeze(struct ata_port *ap);
319static void nv_swncq_error_handler(struct ata_port *ap);
320static int nv_swncq_slave_config(struct scsi_device *sdev);
321static int nv_swncq_port_start(struct ata_port *ap);
95364f36 322static enum ata_completion_errors nv_swncq_qc_prep(struct ata_queued_cmd *qc);
f140f0f1
KL		 323static void nv_swncq_fill_sg(struct ata_queued_cmd *qc);
324static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc);
325static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis);
326static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance);
327#ifdef CONFIG_PM
328static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg);
329static int nv_swncq_port_resume(struct ata_port *ap);
330#endif
331
1da177e4
LT		 332enum nv_host_type
333{
334 GENERIC,
335 NFORCE2,
27e4b274 336 NFORCE3 = NFORCE2, /* NF2 == NF3 as far as sata_nv is concerned */
fbbb262d 337 CK804,
f140f0f1 338 ADMA,
2d775708 339 MCP5x,
f140f0f1 340 SWNCQ,
1da177e4
LT		 341};
342
3b7d697d 343static const struct pci_device_id nv_pci_tbl[] = {
54bb3a94
JG		 344 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE2S_SATA), NFORCE2 },
345 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA), NFORCE3 },
346 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA2), NFORCE3 },
347 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA), CK804 },
348 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA2), CK804 },
349 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA), CK804 },
350 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA2), CK804 },
2d775708
TH		 351 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA), MCP5x },
352 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2), MCP5x },
353 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA), MCP5x },
354 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2), MCP5x },
e2e031eb
KL		 355 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA), GENERIC },
356 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA2), GENERIC },
357 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA3), GENERIC },
2d2744fc
JG		 358
359 { } /* terminate list */
1da177e4
LT		 360};
361
1da177e4
LT		 362static struct pci_driver nv_pci_driver = {
363 .name = DRV_NAME,
364 .id_table = nv_pci_tbl,
365 .probe = nv_init_one,
58eb8cd5 366#ifdef CONFIG_PM_SLEEP
cdf56bcf
RH		 367 .suspend = ata_pci_device_suspend,
368 .resume = nv_pci_device_resume,
438ac6d5 369#endif
1daf9ce7 370 .remove = ata_pci_remove_one,
1da177e4
LT		 371};
372
193515d5 373static struct scsi_host_template nv_sht = {
68d1d07b 374 ATA_BMDMA_SHT(DRV_NAME),
1da177e4
LT		 375};
376
fbbb262d 377static struct scsi_host_template nv_adma_sht = {
68d1d07b 378 ATA_NCQ_SHT(DRV_NAME),
fbbb262d 379 .can_queue = NV_ADMA_MAX_CPBS,
fbbb262d 380 .sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN,
fbbb262d
RH		 381 .dma_boundary = NV_ADMA_DMA_BOUNDARY,
382 .slave_configure = nv_adma_slave_config,
fbbb262d
RH		 383};
384
f140f0f1 385static struct scsi_host_template nv_swncq_sht = {
68d1d07b 386 ATA_NCQ_SHT(DRV_NAME),
ba80c3a5 387 .can_queue = ATA_MAX_QUEUE - 1,
f140f0f1 388 .sg_tablesize = LIBATA_MAX_PRD,
f140f0f1
KL		 389 .dma_boundary = ATA_DMA_BOUNDARY,
390 .slave_configure = nv_swncq_slave_config,
f140f0f1
KL		 391};
392
7f4774b3
TH		 393/*
394 * NV SATA controllers have various different problems with hardreset
395 * protocol depending on the specific controller and device.
396 *
397 * GENERIC:
398 *
399 * bko11195 reports that link doesn't come online after hardreset on
400 * generic nv's and there have been several other similar reports on
401 * linux-ide.
402 *
403 * bko12351#c23 reports that warmplug on MCP61 doesn't work with
404 * softreset.
405 *
406 * NF2/3:
407 *
408 * bko3352 reports nf2/3 controllers can't determine device signature
409 * reliably after hardreset. The following thread reports detection
410 * failure on cold boot with the standard debouncing timing.
411 *
412 * http://thread.gmane.org/gmane.linux.ide/34098
413 *
414 * bko12176 reports that hardreset fails to bring up the link during
415 * boot on nf2.
416 *
417 * CK804:
418 *
419 * For initial probing after boot and hot plugging, hardreset mostly
420 * works fine on CK804 but curiously, reprobing on the initial port
421 * by rescanning or rmmod/insmod fails to acquire the initial D2H Reg
422 * FIS in somewhat undeterministic way.
423 *
424 * SWNCQ:
425 *
426 * bko12351 reports that when SWNCQ is enabled, for hotplug to work,
427 * hardreset should be used and hardreset can't report proper
428 * signature, which suggests that mcp5x is closer to nf2 as long as
429 * reset quirkiness is concerned.
430 *
431 * bko12703 reports that boot probing fails for intel SSD with
432 * hardreset. Link fails to come online. Softreset works fine.
433 *
434 * The failures are varied but the following patterns seem true for
435 * all flavors.
436 *
437 * - Softreset during boot always works.
438 *
439 * - Hardreset during boot sometimes fails to bring up the link on
440 * certain comibnations and device signature acquisition is
441 * unreliable.
442 *
443 * - Hardreset is often necessary after hotplug.
444 *
445 * So, preferring softreset for boot probing and error handling (as
446 * hardreset might bring down the link) but using hardreset for
447 * post-boot probing should work around the above issues in most
448 * cases. Define nv_hardreset() which only kicks in for post-boot
449 * probing and use it for all variants.
450 */
451static struct ata_port_operations nv_generic_ops = {
029cfd6b 452 .inherits = &ata_bmdma_port_ops,
c96f1732 453 .lost_interrupt = ATA_OP_NULL,
1da177e4
LT		 454 .scr_read = nv_scr_read,
455 .scr_write = nv_scr_write,
7f4774b3 456 .hardreset = nv_hardreset,
1da177e4
LT		 457};
458
029cfd6b 459static struct ata_port_operations nv_nf2_ops = {
7dac745b 460 .inherits = &nv_generic_ops,
39f87582
TH		 461 .freeze = nv_nf2_freeze,
462 .thaw = nv_nf2_thaw,
ada364e8
TH		 463};
464
029cfd6b 465static struct ata_port_operations nv_ck804_ops = {
7f4774b3 466 .inherits = &nv_generic_ops,
39f87582
TH		 467 .freeze = nv_ck804_freeze,
468 .thaw = nv_ck804_thaw,
ada364e8
TH		 469 .host_stop = nv_ck804_host_stop,
470};
471
029cfd6b 472static struct ata_port_operations nv_adma_ops = {
3c324283 473 .inherits = &nv_ck804_ops,
029cfd6b 474
2dec7555 475 .check_atapi_dma = nv_adma_check_atapi_dma,
5682ed33 476 .sff_tf_read = nv_adma_tf_read,
31cc23b3 477 .qc_defer = ata_std_qc_defer,
fbbb262d
RH		 478 .qc_prep = nv_adma_qc_prep,
479 .qc_issue = nv_adma_qc_issue,
5682ed33 480 .sff_irq_clear = nv_adma_irq_clear,
029cfd6b 481
53014e25
RH		 482 .freeze = nv_adma_freeze,
483 .thaw = nv_adma_thaw,
fbbb262d 484 .error_handler = nv_adma_error_handler,
f5ecac2d 485 .post_internal_cmd = nv_adma_post_internal_cmd,
029cfd6b 486
fbbb262d
RH		 487 .port_start = nv_adma_port_start,
488 .port_stop = nv_adma_port_stop,
438ac6d5 489#ifdef CONFIG_PM
cdf56bcf
RH		 490 .port_suspend = nv_adma_port_suspend,
491 .port_resume = nv_adma_port_resume,
438ac6d5 492#endif
fbbb262d
RH		 493 .host_stop = nv_adma_host_stop,
494};
495
029cfd6b 496static struct ata_port_operations nv_swncq_ops = {
7f4774b3 497 .inherits = &nv_generic_ops,
029cfd6b 498
f140f0f1
KL		 499 .qc_defer = ata_std_qc_defer,
500 .qc_prep = nv_swncq_qc_prep,
501 .qc_issue = nv_swncq_qc_issue,
029cfd6b 502
f140f0f1
KL		 503 .freeze = nv_mcp55_freeze,
504 .thaw = nv_mcp55_thaw,
505 .error_handler = nv_swncq_error_handler,
029cfd6b 506
f140f0f1
KL		 507#ifdef CONFIG_PM
508 .port_suspend = nv_swncq_port_suspend,
509 .port_resume = nv_swncq_port_resume,
510#endif
511 .port_start = nv_swncq_port_start,
512};
513
95947193
TH		 514struct nv_pi_priv {
515 irq_handler_t irq_handler;
516 struct scsi_host_template *sht;
517};
518
519#define NV_PI_PRIV(_irq_handler, _sht) \
520 &(struct nv_pi_priv){ .irq_handler = _irq_handler, .sht = _sht }
521
1626aeb8 522static const struct ata_port_info nv_port_info[] = {
ada364e8
TH		 523 /* generic */
524 {
9cbe056f 525 .flags = ATA_FLAG_SATA,
ada364e8
TH		 526 .pio_mask = NV_PIO_MASK,
527 .mwdma_mask = NV_MWDMA_MASK,
528 .udma_mask = NV_UDMA_MASK,
529 .port_ops = &nv_generic_ops,
95947193 530 .private_data = NV_PI_PRIV(nv_generic_interrupt, &nv_sht),
ada364e8
TH		 531 },
532 /* nforce2/3 */
533 {
9cbe056f 534 .flags = ATA_FLAG_SATA,
ada364e8
TH		 535 .pio_mask = NV_PIO_MASK,
536 .mwdma_mask = NV_MWDMA_MASK,
537 .udma_mask = NV_UDMA_MASK,
538 .port_ops = &nv_nf2_ops,
95947193 539 .private_data = NV_PI_PRIV(nv_nf2_interrupt, &nv_sht),
ada364e8
TH		 540 },
541 /* ck804 */
542 {
9cbe056f 543 .flags = ATA_FLAG_SATA,
ada364e8
TH		 544 .pio_mask = NV_PIO_MASK,
545 .mwdma_mask = NV_MWDMA_MASK,
546 .udma_mask = NV_UDMA_MASK,
547 .port_ops = &nv_ck804_ops,
95947193 548 .private_data = NV_PI_PRIV(nv_ck804_interrupt, &nv_sht),
ada364e8 549 },
fbbb262d
RH		 550 /* ADMA */
551 {
9cbe056f 552 .flags = ATA_FLAG_SATA | ATA_FLAG_NCQ,
fbbb262d
RH		 553 .pio_mask = NV_PIO_MASK,
554 .mwdma_mask = NV_MWDMA_MASK,
555 .udma_mask = NV_UDMA_MASK,
556 .port_ops = &nv_adma_ops,
95947193 557 .private_data = NV_PI_PRIV(nv_adma_interrupt, &nv_adma_sht),
fbbb262d 558 },
2d775708
TH		 559 /* MCP5x */
560 {
9cbe056f 561 .flags = ATA_FLAG_SATA,
2d775708
TH		 562 .pio_mask = NV_PIO_MASK,
563 .mwdma_mask = NV_MWDMA_MASK,
564 .udma_mask = NV_UDMA_MASK,
7f4774b3 565 .port_ops = &nv_generic_ops,
2d775708
TH		 566 .private_data = NV_PI_PRIV(nv_generic_interrupt, &nv_sht),
567 },
f140f0f1
KL		 568 /* SWNCQ */
569 {
9cbe056f 570 .flags = ATA_FLAG_SATA | ATA_FLAG_NCQ,
f140f0f1
KL		 571 .pio_mask = NV_PIO_MASK,
572 .mwdma_mask = NV_MWDMA_MASK,
573 .udma_mask = NV_UDMA_MASK,
574 .port_ops = &nv_swncq_ops,
95947193 575 .private_data = NV_PI_PRIV(nv_swncq_interrupt, &nv_swncq_sht),
f140f0f1 576 },
1da177e4
LT		 577};
578
579MODULE_AUTHOR("NVIDIA");
580MODULE_DESCRIPTION("low-level driver for NVIDIA nForce SATA controller");
581MODULE_LICENSE("GPL");
582MODULE_DEVICE_TABLE(pci, nv_pci_tbl);
583MODULE_VERSION(DRV_VERSION);
584
90ab5ee9 585static bool adma_enabled;
c13aff32 586static bool swncq_enabled = true;
90ab5ee9 587static bool msi_enabled;
fbbb262d 588
2dec7555
RH		 589static void nv_adma_register_mode(struct ata_port *ap)
590{
2dec7555 591 struct nv_adma_port_priv *pp = ap->private_data;
cdf56bcf 592 void __iomem *mmio = pp->ctl_block;
a2cfe81a
RH		 593 u16 tmp, status;
594 int count = 0;
2dec7555
RH		 595
596 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
597 return;
598
a2cfe81a 599 status = readw(mmio + NV_ADMA_STAT);
2dcb407e 600 while (!(status & NV_ADMA_STAT_IDLE) && count < 20) {
a2cfe81a
RH		 601 ndelay(50);
602 status = readw(mmio + NV_ADMA_STAT);
603 count++;
604 }
2dcb407e 605 if (count == 20)
a9a79dfe
JP		 606 ata_port_warn(ap, "timeout waiting for ADMA IDLE, stat=0x%hx\n",
607 status);
a2cfe81a 608
2dec7555
RH		 609 tmp = readw(mmio + NV_ADMA_CTL);
610 writew(tmp & ~NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
611
a2cfe81a
RH		 612 count = 0;
613 status = readw(mmio + NV_ADMA_STAT);
2dcb407e 614 while (!(status & NV_ADMA_STAT_LEGACY) && count < 20) {
a2cfe81a
RH		 615 ndelay(50);
616 status = readw(mmio + NV_ADMA_STAT);
617 count++;
618 }
2dcb407e 619 if (count == 20)
a9a79dfe
JP		 620 ata_port_warn(ap,
621 "timeout waiting for ADMA LEGACY, stat=0x%hx\n",
622 status);
a2cfe81a 623
2dec7555
RH		 624 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
625}
626
627static void nv_adma_mode(struct ata_port *ap)
628{
2dec7555 629 struct nv_adma_port_priv *pp = ap->private_data;
cdf56bcf 630 void __iomem *mmio = pp->ctl_block;
a2cfe81a
RH		 631 u16 tmp, status;
632 int count = 0;
2dec7555
RH		 633
634 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE))
635 return;
f20b16ff 636
2dec7555
RH		 637 WARN_ON(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
638
639 tmp = readw(mmio + NV_ADMA_CTL);
640 writew(tmp | NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);
641
a2cfe81a 642 status = readw(mmio + NV_ADMA_STAT);
2dcb407e 643 while (((status & NV_ADMA_STAT_LEGACY) ||
a2cfe81a
RH		 644 !(status & NV_ADMA_STAT_IDLE)) && count < 20) {
645 ndelay(50);
646 status = readw(mmio + NV_ADMA_STAT);
647 count++;
648 }
2dcb407e 649 if (count == 20)
a9a79dfe 650 ata_port_warn(ap,
a2cfe81a
RH		 651 "timeout waiting for ADMA LEGACY clear and IDLE, stat=0x%hx\n",
652 status);
653
2dec7555
RH		 654 pp->flags &= ~NV_ADMA_PORT_REGISTER_MODE;
655}
656
fbbb262d
RH		 657static int nv_adma_slave_config(struct scsi_device *sdev)
658{
659 struct ata_port *ap = ata_shost_to_port(sdev->host);
2dec7555 660 struct nv_adma_port_priv *pp = ap->private_data;
8959d300 661 struct nv_adma_port_priv *port0, *port1;
2dec7555 662 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
8959d300 663 unsigned long segment_boundary, flags;
fbbb262d
RH		 664 unsigned short sg_tablesize;
665 int rc;
2dec7555
RH		 666 int adma_enable;
667 u32 current_reg, new_reg, config_mask;
fbbb262d
RH		 668
669 rc = ata_scsi_slave_config(sdev);
670
671 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
672 /* Not a proper libata device, ignore */
673 return rc;
674
8959d300
RH		 675 spin_lock_irqsave(ap->lock, flags);
676
9af5c9c9 677 if (ap->link.device[sdev->id].class == ATA_DEV_ATAPI) {
fbbb262d
RH		 678 /*
679 * NVIDIA reports that ADMA mode does not support ATAPI commands.
680 * Therefore ATAPI commands are sent through the legacy interface.
681 * However, the legacy interface only supports 32-bit DMA.
682 * Restrict DMA parameters as required by the legacy interface
683 * when an ATAPI device is connected.
684 */
fbbb262d
RH		 685 segment_boundary = ATA_DMA_BOUNDARY;
686 /* Subtract 1 since an extra entry may be needed for padding, see
687 libata-scsi.c */
688 sg_tablesize = LIBATA_MAX_PRD - 1;
f20b16ff 689
2dec7555
RH		 690 /* Since the legacy DMA engine is in use, we need to disable ADMA
691 on the port. */
692 adma_enable = 0;
693 nv_adma_register_mode(ap);
2dcb407e 694 } else {
fbbb262d
RH		 695 segment_boundary = NV_ADMA_DMA_BOUNDARY;
696 sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN;
2dec7555 697 adma_enable = 1;
fbbb262d 698 }
f20b16ff 699
2dec7555
RH		 700 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &current_reg);
701
2dcb407e 702 if (ap->port_no == 1)
2dec7555
RH		 703 config_mask = NV_MCP_SATA_CFG_20_PORT1_EN |
704 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
705 else
706 config_mask = NV_MCP_SATA_CFG_20_PORT0_EN |
707 NV_MCP_SATA_CFG_20_PORT0_PWB_EN;
f20b16ff 708
2dcb407e 709 if (adma_enable) {
2dec7555
RH		 710 new_reg = current_reg | config_mask;
711 pp->flags &= ~NV_ADMA_ATAPI_SETUP_COMPLETE;
2dcb407e 712 } else {
2dec7555
RH		 713 new_reg = current_reg & ~config_mask;
714 pp->flags |= NV_ADMA_ATAPI_SETUP_COMPLETE;
715 }
f20b16ff 716
2dcb407e 717 if (current_reg != new_reg)
2dec7555 718 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, new_reg);
f20b16ff 719
8959d300
RH		 720 port0 = ap->host->ports[0]->private_data;
721 port1 = ap->host->ports[1]->private_data;
8959d300
RH		 722 if ((port0->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
723 (port1->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)) {
258c9fde
CH		 724 /*
725 * We have to set the DMA mask to 32-bit if either port is in
726 * ATAPI mode, since they are on the same PCI device which is
727 * used for DMA mapping. If either SCSI device is not allocated
728 * yet, it's OK since that port will discover its correct
729 * setting when it does get allocated.
730 */
731 rc = dma_set_mask(&pdev->dev, ATA_DMA_MASK);
8959d300 732 } else {
258c9fde 733 rc = dma_set_mask(&pdev->dev, pp->adma_dma_mask);
8959d300
RH		 734 }
735
fbbb262d 736 blk_queue_segment_boundary(sdev->request_queue, segment_boundary);
8a78362c 737 blk_queue_max_segments(sdev->request_queue, sg_tablesize);
a9a79dfe
JP		 738 ata_port_info(ap,
739 "DMA mask 0x%llX, segment boundary 0x%lX, hw segs %hu\n",
740 (unsigned long long)*ap->host->dev->dma_mask,
741 segment_boundary, sg_tablesize);
8959d300
RH		 742
743 spin_unlock_irqrestore(ap->lock, flags);
744
fbbb262d
RH		 745 return rc;
746}
747
2dec7555
RH		 748static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc)
749{
750 struct nv_adma_port_priv *pp = qc->ap->private_data;
751 return !(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE);
752}
753
f2fb344b
RH		 754static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf)
755{
3f3debdb
RH		 756 /* Other than when internal or pass-through commands are executed,
757 the only time this function will be called in ADMA mode will be
758 if a command fails. In the failure case we don't care about going
759 into register mode with ADMA commands pending, as the commands will
760 all shortly be aborted anyway. We assume that NCQ commands are not
761 issued via passthrough, which is the only way that switching into
762 ADMA mode could abort outstanding commands. */
f2fb344b
RH		 763 nv_adma_register_mode(ap);
764
9363c382 765 ata_sff_tf_read(ap, tf);
f2fb344b
RH		 766}
767
2dec7555 768static unsigned int nv_adma_tf_to_cpb(struct ata_taskfile *tf, __le16 *cpb)
fbbb262d
RH		 769{
770 unsigned int idx = 0;
771
2dcb407e 772 if (tf->flags & ATA_TFLAG_ISADDR) {
ac3d6b86
RH		 773 if (tf->flags & ATA_TFLAG_LBA48) {
774 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->hob_feature | WNB);
775 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->hob_nsect);
776 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->hob_lbal);
777 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->hob_lbam);
778 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->hob_lbah);
779 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature);
780 } else
781 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature | WNB);
a84471fe 782
ac3d6b86
RH		 783 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->nsect);
784 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->lbal);
785 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->lbam);
786 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->lbah);
fbbb262d 787 }
a84471fe 788
2dcb407e 789 if (tf->flags & ATA_TFLAG_DEVICE)
ac3d6b86 790 cpb[idx++] = cpu_to_le16((ATA_REG_DEVICE << 8) | tf->device);
fbbb262d
RH		 791
792 cpb[idx++] = cpu_to_le16((ATA_REG_CMD << 8) | tf->command | CMDEND);
a84471fe 793
2dcb407e 794 while (idx < 12)
ac3d6b86 795 cpb[idx++] = cpu_to_le16(IGN);
fbbb262d
RH		 796
797 return idx;
798}
799
5bd28a4b 800static int nv_adma_check_cpb(struct ata_port *ap, int cpb_num, int force_err)
fbbb262d
RH		 801{
802 struct nv_adma_port_priv *pp = ap->private_data;
2dec7555 803 u8 flags = pp->cpb[cpb_num].resp_flags;
fbbb262d
RH		 804
805 VPRINTK("CPB %d, flags=0x%x\n", cpb_num, flags);
806
5bd28a4b
RH		 807 if (unlikely((force_err ||
808 flags & (NV_CPB_RESP_ATA_ERR |
809 NV_CPB_RESP_CMD_ERR |
810 NV_CPB_RESP_CPB_ERR)))) {
9af5c9c9 811 struct ata_eh_info *ehi = &ap->link.eh_info;
5bd28a4b
RH		 812 int freeze = 0;
813
814 ata_ehi_clear_desc(ehi);
2dcb407e 815 __ata_ehi_push_desc(ehi, "CPB resp_flags 0x%x: ", flags);
5bd28a4b 816 if (flags & NV_CPB_RESP_ATA_ERR) {
b64bbc39 817 ata_ehi_push_desc(ehi, "ATA error");
5bd28a4b
RH		 818 ehi->err_mask |= AC_ERR_DEV;
819 } else if (flags & NV_CPB_RESP_CMD_ERR) {
b64bbc39 820 ata_ehi_push_desc(ehi, "CMD error");
5bd28a4b
RH		 821 ehi->err_mask |= AC_ERR_DEV;
822 } else if (flags & NV_CPB_RESP_CPB_ERR) {
b64bbc39 823 ata_ehi_push_desc(ehi, "CPB error");
5bd28a4b
RH		 824 ehi->err_mask |= AC_ERR_SYSTEM;
825 freeze = 1;
826 } else {
827 /* notifier error, but no error in CPB flags? */
b64bbc39 828 ata_ehi_push_desc(ehi, "unknown");
5bd28a4b
RH		 829 ehi->err_mask |= AC_ERR_OTHER;
830 freeze = 1;
831 }
832 /* Kill all commands. EH will determine what actually failed. */
833 if (freeze)
834 ata_port_freeze(ap);
835 else
836 ata_port_abort(ap);
1aadf5c3 837 return -1;
fbbb262d 838 }
5bd28a4b 839
1aadf5c3
TH		 840 if (likely(flags & NV_CPB_RESP_DONE))
841 return 1;
5bd28a4b 842 return 0;
fbbb262d
RH		 843}
844
2dec7555
RH		 845static int nv_host_intr(struct ata_port *ap, u8 irq_stat)
846{
9af5c9c9 847 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->link.active_tag);
2dec7555
RH		 848
849 /* freeze if hotplugged */
850 if (unlikely(irq_stat & (NV_INT_ADDED | NV_INT_REMOVED))) {
851 ata_port_freeze(ap);
852 return 1;
853 }
854
855 /* bail out if not our interrupt */
856 if (!(irq_stat & NV_INT_DEV))
857 return 0;
858
859 /* DEV interrupt w/ no active qc? */
860 if (unlikely(!qc || (qc->tf.flags & ATA_TFLAG_POLLING))) {
9363c382 861 ata_sff_check_status(ap);
2dec7555
RH		 862 return 1;
863 }
864
865 /* handle interrupt */
c3b28894 866 return ata_bmdma_port_intr(ap, qc);
2dec7555
RH		 867}
868
fbbb262d
RH		 869static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance)
870{
871 struct ata_host *host = dev_instance;
872 int i, handled = 0;
2dec7555 873 u32 notifier_clears[2];
fbbb262d
RH		 874
875 spin_lock(&host->lock);
876
877 for (i = 0; i < host->n_ports; i++) {
878 struct ata_port *ap = host->ports[i];
3e4ec344
TH		 879 struct nv_adma_port_priv *pp = ap->private_data;
880 void __iomem *mmio = pp->ctl_block;
881 u16 status;
882 u32 gen_ctl;
883 u32 notifier, notifier_error;
884
2dec7555 885 notifier_clears[i] = 0;
fbbb262d 886
3e4ec344
TH		 887 /* if ADMA is disabled, use standard ata interrupt handler */
888 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
889 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
890 >> (NV_INT_PORT_SHIFT * i);
891 handled += nv_host_intr(ap, irq_stat);
892 continue;
893 }
fbbb262d 894
3e4ec344
TH		 895 /* if in ATA register mode, check for standard interrupts */
896 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) {
897 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804)
898 >> (NV_INT_PORT_SHIFT * i);
899 if (ata_tag_valid(ap->link.active_tag))
900 /** NV_INT_DEV indication seems unreliable
901 at times at least in ADMA mode. Force it
902 on always when a command is active, to
903 prevent losing interrupts. */
904 irq_stat |= NV_INT_DEV;
905 handled += nv_host_intr(ap, irq_stat);
906 }
907
908 notifier = readl(mmio + NV_ADMA_NOTIFIER);
909 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
910 notifier_clears[i] = notifier | notifier_error;
911
912 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
913
914 if (!NV_ADMA_CHECK_INTR(gen_ctl, ap->port_no) && !notifier &&
915 !notifier_error)
916 /* Nothing to do */
917 continue;
918
919 status = readw(mmio + NV_ADMA_STAT);
920
921 /*
922 * Clear status. Ensure the controller sees the
923 * clearing before we start looking at any of the CPB
924 * statuses, so that any CPB completions after this
925 * point in the handler will raise another interrupt.
926 */
927 writew(status, mmio + NV_ADMA_STAT);
928 readw(mmio + NV_ADMA_STAT); /* flush posted write */
929 rmb();
fbbb262d 930
3e4ec344
TH		 931 handled++; /* irq handled if we got here */
932
933 /* freeze if hotplugged or controller error */
934 if (unlikely(status & (NV_ADMA_STAT_HOTPLUG |
935 NV_ADMA_STAT_HOTUNPLUG |
936 NV_ADMA_STAT_TIMEOUT |
937 NV_ADMA_STAT_SERROR))) {
938 struct ata_eh_info *ehi = &ap->link.eh_info;
939
940 ata_ehi_clear_desc(ehi);
941 __ata_ehi_push_desc(ehi, "ADMA status 0x%08x: ", status);
942 if (status & NV_ADMA_STAT_TIMEOUT) {
943 ehi->err_mask |= AC_ERR_SYSTEM;
944 ata_ehi_push_desc(ehi, "timeout");
945 } else if (status & NV_ADMA_STAT_HOTPLUG) {
946 ata_ehi_hotplugged(ehi);
947 ata_ehi_push_desc(ehi, "hotplug");
948 } else if (status & NV_ADMA_STAT_HOTUNPLUG) {
949 ata_ehi_hotplugged(ehi);
950 ata_ehi_push_desc(ehi, "hot unplug");
951 } else if (status & NV_ADMA_STAT_SERROR) {
952 /* let EH analyze SError and figure out cause */
953 ata_ehi_push_desc(ehi, "SError");
954 } else
955 ata_ehi_push_desc(ehi, "unknown");
956 ata_port_freeze(ap);
957 continue;
958 }
959
960 if (status & (NV_ADMA_STAT_DONE |
961 NV_ADMA_STAT_CPBERR |
962 NV_ADMA_STAT_CMD_COMPLETE)) {
963 u32 check_commands = notifier_clears[i];
1aadf5c3 964 u32 done_mask = 0;
752e386c 965 int pos, rc;
3e4ec344
TH		 966
967 if (status & NV_ADMA_STAT_CPBERR) {
968 /* check all active commands */
969 if (ata_tag_valid(ap->link.active_tag))
970 check_commands = 1 <<
971 ap->link.active_tag;
972 else
973 check_commands = ap->link.sactive;
fbbb262d
RH		 974 }
975
3e4ec344 976 /* check CPBs for completed commands */
752e386c 977 while ((pos = ffs(check_commands))) {
3e4ec344 978 pos--;
752e386c 979 rc = nv_adma_check_cpb(ap, pos,
5796d1c4 980 notifier_error & (1 << pos));
1aadf5c3
TH		 981 if (rc > 0)
982 done_mask |= 1 << pos;
983 else if (unlikely(rc < 0))
752e386c 984 check_commands = 0;
3e4ec344 985 check_commands &= ~(1 << pos);
fbbb262d 986 }
8385d756 987 ata_qc_complete_multiple(ap, ata_qc_get_active(ap) ^ done_mask);
fbbb262d
RH		 988 }
989 }
f20b16ff 990
b447916e 991 if (notifier_clears[0] || notifier_clears[1]) {
2dec7555
RH		 992 /* Note: Both notifier clear registers must be written
993 if either is set, even if one is zero, according to NVIDIA. */
cdf56bcf
RH		 994 struct nv_adma_port_priv *pp = host->ports[0]->private_data;
995 writel(notifier_clears[0], pp->notifier_clear_block);
996 pp = host->ports[1]->private_data;
997 writel(notifier_clears[1], pp->notifier_clear_block);
2dec7555 998 }
fbbb262d
RH		 999
1000 spin_unlock(&host->lock);
1001
1002 return IRQ_RETVAL(handled);
1003}
1004
53014e25
RH		 1005static void nv_adma_freeze(struct ata_port *ap)
1006{
1007 struct nv_adma_port_priv *pp = ap->private_data;
1008 void __iomem *mmio = pp->ctl_block;
1009 u16 tmp;
1010
1011 nv_ck804_freeze(ap);
1012
1013 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1014 return;
1015
1016 /* clear any outstanding CK804 notifications */
2dcb407e 1017 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
53014e25
RH		 1018 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
1019
1020 /* Disable interrupt */
1021 tmp = readw(mmio + NV_ADMA_CTL);
2dcb407e 1022 writew(tmp & ~(NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
53014e25 1023 mmio + NV_ADMA_CTL);
5796d1c4 1024 readw(mmio + NV_ADMA_CTL); /* flush posted write */
53014e25
RH		 1025}
1026
1027static void nv_adma_thaw(struct ata_port *ap)
1028{
1029 struct nv_adma_port_priv *pp = ap->private_data;
1030 void __iomem *mmio = pp->ctl_block;
1031 u16 tmp;
1032
1033 nv_ck804_thaw(ap);
1034
1035 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
1036 return;
1037
1038 /* Enable interrupt */
1039 tmp = readw(mmio + NV_ADMA_CTL);
2dcb407e 1040 writew(tmp | (NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN),
53014e25 1041 mmio + NV_ADMA_CTL);
5796d1c4 1042 readw(mmio + NV_ADMA_CTL); /* flush posted write */
53014e25
RH		 1043}
1044
fbbb262d
RH		 1045static void nv_adma_irq_clear(struct ata_port *ap)
1046{
cdf56bcf
RH		 1047 struct nv_adma_port_priv *pp = ap->private_data;
1048 void __iomem *mmio = pp->ctl_block;
53014e25 1049 u32 notifier_clears[2];
fbbb262d 1050
53014e25 1051 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) {
37f65b8b 1052 ata_bmdma_irq_clear(ap);
53014e25
RH		 1053 return;
1054 }
1055
1056 /* clear any outstanding CK804 notifications */
2dcb407e 1057 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT),
53014e25 1058 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
fbbb262d 1059
53014e25
RH		 1060 /* clear ADMA status */
1061 writew(0xffff, mmio + NV_ADMA_STAT);
a617c09f 1062
53014e25
RH		 1063 /* clear notifiers - note both ports need to be written with
1064 something even though we are only clearing on one */
1065 if (ap->port_no == 0) {
1066 notifier_clears[0] = 0xFFFFFFFF;
1067 notifier_clears[1] = 0;
1068 } else {
1069 notifier_clears[0] = 0;
1070 notifier_clears[1] = 0xFFFFFFFF;
1071 }
1072 pp = ap->host->ports[0]->private_data;
1073 writel(notifier_clears[0], pp->notifier_clear_block);
1074 pp = ap->host->ports[1]->private_data;
1075 writel(notifier_clears[1], pp->notifier_clear_block);
fbbb262d
RH		 1076}
1077
f5ecac2d 1078static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc)
fbbb262d 1079{
f5ecac2d 1080 struct nv_adma_port_priv *pp = qc->ap->private_data;
fbbb262d 1081
b447916e 1082 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE)
fe06e5f9 1083 ata_bmdma_post_internal_cmd(qc);
fbbb262d
RH		 1084}
1085
1086static int nv_adma_port_start(struct ata_port *ap)
1087{
1088 struct device *dev = ap->host->dev;
1089 struct nv_adma_port_priv *pp;
1090 int rc;
1091 void *mem;
1092 dma_addr_t mem_dma;
cdf56bcf 1093 void __iomem *mmio;
8959d300 1094 struct pci_dev *pdev = to_pci_dev(dev);
fbbb262d
RH		 1095 u16 tmp;
1096
1097 VPRINTK("ENTER\n");
1098
258c9fde
CH		 1099 /*
1100 * Ensure DMA mask is set to 32-bit before allocating legacy PRD and
1101 * pad buffers.
1102 */
1103 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
8959d300
RH		 1104 if (rc)
1105 return rc;
1106
c7087652
TH		 1107 /* we might fallback to bmdma, allocate bmdma resources */
1108 rc = ata_bmdma_port_start(ap);
fbbb262d
RH		 1109 if (rc)
1110 return rc;
1111
24dc5f33
TH		 1112 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1113 if (!pp)
1114 return -ENOMEM;
fbbb262d 1115
0d5ff566 1116 mmio = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_PORT +
cdf56bcf
RH		 1117 ap->port_no * NV_ADMA_PORT_SIZE;
1118 pp->ctl_block = mmio;
0d5ff566 1119 pp->gen_block = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_GEN;
cdf56bcf
RH		 1120 pp->notifier_clear_block = pp->gen_block +
1121 NV_ADMA_NOTIFIER_CLEAR + (4 * ap->port_no);
1122
258c9fde
CH		 1123 /*
1124 * Now that the legacy PRD and padding buffer are allocated we can
51872b66 1125 * raise the DMA mask to allocate the CPB/APRD table.
258c9fde 1126 */
51872b66
CH		 1127 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1128
8959d300
RH		 1129 pp->adma_dma_mask = *dev->dma_mask;
1130
24dc5f33
TH		 1131 mem = dmam_alloc_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ,
1132 &mem_dma, GFP_KERNEL);
1133 if (!mem)
1134 return -ENOMEM;
fbbb262d
RH		 1135
1136 /*
1137 * First item in chunk of DMA memory:
1138 * 128-byte command parameter block (CPB)
1139 * one for each command tag
1140 */
1141 pp->cpb = mem;
1142 pp->cpb_dma = mem_dma;
1143
1144 writel(mem_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
5796d1c4 1145 writel((mem_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
fbbb262d
RH		 1146
1147 mem += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1148 mem_dma += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ;
1149
1150 /*
1151 * Second item: block of ADMA_SGTBL_LEN s/g entries
1152 */
1153 pp->aprd = mem;
1154 pp->aprd_dma = mem_dma;
1155
1156 ap->private_data = pp;
1157
1158 /* clear any outstanding interrupt conditions */
1159 writew(0xffff, mmio + NV_ADMA_STAT);
1160
1161 /* initialize port variables */
1162 pp->flags = NV_ADMA_PORT_REGISTER_MODE;
1163
1164 /* clear CPB fetch count */
1165 writew(0, mmio + NV_ADMA_CPB_COUNT);
1166
cdf56bcf 1167 /* clear GO for register mode, enable interrupt */
fbbb262d 1168 tmp = readw(mmio + NV_ADMA_CTL);
5796d1c4
JG		 1169 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1170 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
fbbb262d
RH		 1171
1172 tmp = readw(mmio + NV_ADMA_CTL);
1173 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
5796d1c4 1174 readw(mmio + NV_ADMA_CTL); /* flush posted write */
fbbb262d
RH		 1175 udelay(1);
1176 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
5796d1c4 1177 readw(mmio + NV_ADMA_CTL); /* flush posted write */
fbbb262d
RH		 1178
1179 return 0;
fbbb262d
RH		 1180}
1181
1182static void nv_adma_port_stop(struct ata_port *ap)
1183{
fbbb262d 1184 struct nv_adma_port_priv *pp = ap->private_data;
cdf56bcf 1185 void __iomem *mmio = pp->ctl_block;
fbbb262d
RH		 1186
1187 VPRINTK("ENTER\n");
fbbb262d 1188 writew(0, mmio + NV_ADMA_CTL);
fbbb262d
RH		 1189}
1190
438ac6d5 1191#ifdef CONFIG_PM
cdf56bcf
RH		 1192static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg)
1193{
1194 struct nv_adma_port_priv *pp = ap->private_data;
1195 void __iomem *mmio = pp->ctl_block;
1196
1197 /* Go to register mode - clears GO */
1198 nv_adma_register_mode(ap);
1199
1200 /* clear CPB fetch count */
1201 writew(0, mmio + NV_ADMA_CPB_COUNT);
1202
1203 /* disable interrupt, shut down port */
1204 writew(0, mmio + NV_ADMA_CTL);
1205
1206 return 0;
1207}
1208
1209static int nv_adma_port_resume(struct ata_port *ap)
1210{
1211 struct nv_adma_port_priv *pp = ap->private_data;
1212 void __iomem *mmio = pp->ctl_block;
1213 u16 tmp;
1214
1215 /* set CPB block location */
1216 writel(pp->cpb_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW);
5796d1c4 1217 writel((pp->cpb_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH);
cdf56bcf
RH		 1218
1219 /* clear any outstanding interrupt conditions */
1220 writew(0xffff, mmio + NV_ADMA_STAT);
1221
1222 /* initialize port variables */
1223 pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
1224
1225 /* clear CPB fetch count */
1226 writew(0, mmio + NV_ADMA_CPB_COUNT);
1227
1228 /* clear GO for register mode, enable interrupt */
1229 tmp = readw(mmio + NV_ADMA_CTL);
5796d1c4
JG		 1230 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN |
1231 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);
cdf56bcf
RH		 1232
1233 tmp = readw(mmio + NV_ADMA_CTL);
1234 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
5796d1c4 1235 readw(mmio + NV_ADMA_CTL); /* flush posted write */
cdf56bcf
RH		 1236 udelay(1);
1237 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
5796d1c4 1238 readw(mmio + NV_ADMA_CTL); /* flush posted write */
cdf56bcf
RH		 1239
1240 return 0;
1241}
438ac6d5 1242#endif
fbbb262d 1243
9a829ccf 1244static void nv_adma_setup_port(struct ata_port *ap)
fbbb262d 1245{
9a829ccf
TH		 1246 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1247 struct ata_ioports *ioport = &ap->ioaddr;
fbbb262d
RH		 1248
1249 VPRINTK("ENTER\n");
1250
9a829ccf 1251 mmio += NV_ADMA_PORT + ap->port_no * NV_ADMA_PORT_SIZE;
fbbb262d 1252
0d5ff566
TH		 1253 ioport->cmd_addr = mmio;
1254 ioport->data_addr = mmio + (ATA_REG_DATA * 4);
fbbb262d 1255 ioport->error_addr =
0d5ff566
TH		 1256 ioport->feature_addr = mmio + (ATA_REG_ERR * 4);
1257 ioport->nsect_addr = mmio + (ATA_REG_NSECT * 4);
1258 ioport->lbal_addr = mmio + (ATA_REG_LBAL * 4);
1259 ioport->lbam_addr = mmio + (ATA_REG_LBAM * 4);
1260 ioport->lbah_addr = mmio + (ATA_REG_LBAH * 4);
1261 ioport->device_addr = mmio + (ATA_REG_DEVICE * 4);
fbbb262d 1262 ioport->status_addr =
0d5ff566 1263 ioport->command_addr = mmio + (ATA_REG_STATUS * 4);
fbbb262d 1264 ioport->altstatus_addr =
0d5ff566 1265 ioport->ctl_addr = mmio + 0x20;
fbbb262d
RH		 1266}
1267
9a829ccf 1268static int nv_adma_host_init(struct ata_host *host)
fbbb262d 1269{
9a829ccf 1270 struct pci_dev *pdev = to_pci_dev(host->dev);
fbbb262d
RH		 1271 unsigned int i;
1272 u32 tmp32;
1273
1274 VPRINTK("ENTER\n");
1275
1276 /* enable ADMA on the ports */
1277 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
1278 tmp32 |= NV_MCP_SATA_CFG_20_PORT0_EN |
1279 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
1280 NV_MCP_SATA_CFG_20_PORT1_EN |
1281 NV_MCP_SATA_CFG_20_PORT1_PWB_EN;
1282
1283 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
1284
9a829ccf
TH		 1285 for (i = 0; i < host->n_ports; i++)
1286 nv_adma_setup_port(host->ports[i]);
fbbb262d 1287
fbbb262d
RH		 1288 return 0;
1289}
1290
1291static void nv_adma_fill_aprd(struct ata_queued_cmd *qc,
1292 struct scatterlist *sg,
1293 int idx,
1294 struct nv_adma_prd *aprd)
1295{
41949ed5 1296 u8 flags = 0;
fbbb262d
RH		 1297 if (qc->tf.flags & ATA_TFLAG_WRITE)
1298 flags |= NV_APRD_WRITE;
1299 if (idx == qc->n_elem - 1)
1300 flags |= NV_APRD_END;
1301 else if (idx != 4)
1302 flags |= NV_APRD_CONT;
1303
1304 aprd->addr = cpu_to_le64(((u64)sg_dma_address(sg)));
1305 aprd->len = cpu_to_le32(((u32)sg_dma_len(sg))); /* len in bytes */
2dec7555 1306 aprd->flags = flags;
41949ed5 1307 aprd->packet_len = 0;
fbbb262d
RH		 1308}
1309
1310static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb)
1311{
1312 struct nv_adma_port_priv *pp = qc->ap->private_data;
fbbb262d
RH		 1313 struct nv_adma_prd *aprd;
1314 struct scatterlist *sg;
ff2aeb1e 1315 unsigned int si;
fbbb262d
RH		 1316
1317 VPRINTK("ENTER\n");
1318
ff2aeb1e
TH		 1319 for_each_sg(qc->sg, sg, qc->n_elem, si) {
1320 aprd = (si < 5) ? &cpb->aprd[si] :
4e5b6260 1321 &pp->aprd[NV_ADMA_SGTBL_LEN * qc->hw_tag + (si-5)];
ff2aeb1e 1322 nv_adma_fill_aprd(qc, sg, si, aprd);
fbbb262d 1323 }
ff2aeb1e 1324 if (si > 5)
4e5b6260 1325 cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->hw_tag)));
41949ed5
RH		 1326 else
1327 cpb->next_aprd = cpu_to_le64(0);
fbbb262d
RH		 1328}
1329
382a6652
RH		 1330static int nv_adma_use_reg_mode(struct ata_queued_cmd *qc)
1331{
1332 struct nv_adma_port_priv *pp = qc->ap->private_data;
1333
1334 /* ADMA engine can only be used for non-ATAPI DMA commands,
3f3debdb 1335 or interrupt-driven no-data commands. */
b447916e 1336 if ((pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) ||
3f3debdb 1337 (qc->tf.flags & ATA_TFLAG_POLLING))
382a6652
RH		 1338 return 1;
1339
b447916e 1340 if ((qc->flags & ATA_QCFLAG_DMAMAP) ||
382a6652
RH		 1341 (qc->tf.protocol == ATA_PROT_NODATA))
1342 return 0;
1343
1344 return 1;
1345}
1346
95364f36 1347static enum ata_completion_errors nv_adma_qc_prep(struct ata_queued_cmd *qc)
fbbb262d
RH		 1348{
1349 struct nv_adma_port_priv *pp = qc->ap->private_data;
4e5b6260 1350 struct nv_adma_cpb *cpb = &pp->cpb[qc->hw_tag];
fbbb262d 1351 u8 ctl_flags = NV_CPB_CTL_CPB_VALID |
fbbb262d
RH		 1352 NV_CPB_CTL_IEN;
1353
382a6652 1354 if (nv_adma_use_reg_mode(qc)) {
3f3debdb
RH		 1355 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1356 (qc->flags & ATA_QCFLAG_DMAMAP));
2dec7555 1357 nv_adma_register_mode(qc->ap);
f47451c4 1358 ata_bmdma_qc_prep(qc);
95364f36 1359 return AC_ERR_OK;
fbbb262d
RH		 1360 }
1361
41949ed5
RH		 1362 cpb->resp_flags = NV_CPB_RESP_DONE;
1363 wmb();
1364 cpb->ctl_flags = 0;
1365 wmb();
fbbb262d
RH		 1366
1367 cpb->len = 3;
4e5b6260 1368 cpb->tag = qc->hw_tag;
fbbb262d
RH		 1369 cpb->next_cpb_idx = 0;
1370
1371 /* turn on NCQ flags for NCQ commands */
1372 if (qc->tf.protocol == ATA_PROT_NCQ)
1373 ctl_flags |= NV_CPB_CTL_QUEUE | NV_CPB_CTL_FPDMA;
1374
cdf56bcf
RH		 1375 VPRINTK("qc->flags = 0x%lx\n", qc->flags);
1376
fbbb262d
RH		 1377 nv_adma_tf_to_cpb(&qc->tf, cpb->tf);
1378
b447916e 1379 if (qc->flags & ATA_QCFLAG_DMAMAP) {
382a6652
RH		 1380 nv_adma_fill_sg(qc, cpb);
1381 ctl_flags |= NV_CPB_CTL_APRD_VALID;
1382 } else
1383 memset(&cpb->aprd[0], 0, sizeof(struct nv_adma_prd) * 5);
fbbb262d 1384
5796d1c4
JG		 1385 /* Be paranoid and don't let the device see NV_CPB_CTL_CPB_VALID
1386 until we are finished filling in all of the contents */
fbbb262d
RH		 1387 wmb();
1388 cpb->ctl_flags = ctl_flags;
41949ed5
RH		 1389 wmb();
1390 cpb->resp_flags = 0;
95364f36
JS		 1391
1392 return AC_ERR_OK;
fbbb262d
RH		 1393}
1394
1395static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc)
1396{
2dec7555 1397 struct nv_adma_port_priv *pp = qc->ap->private_data;
cdf56bcf 1398 void __iomem *mmio = pp->ctl_block;
5e5c74a5 1399 int curr_ncq = (qc->tf.protocol == ATA_PROT_NCQ);
fbbb262d
RH		 1400
1401 VPRINTK("ENTER\n");
1402
3f3debdb
RH		 1403 /* We can't handle result taskfile with NCQ commands, since
1404 retrieving the taskfile switches us out of ADMA mode and would abort
1405 existing commands. */
1406 if (unlikely(qc->tf.protocol == ATA_PROT_NCQ &&
1407 (qc->flags & ATA_QCFLAG_RESULT_TF))) {
a9a79dfe 1408 ata_dev_err(qc->dev, "NCQ w/ RESULT_TF not allowed\n");
3f3debdb
RH		 1409 return AC_ERR_SYSTEM;
1410 }
1411
382a6652 1412 if (nv_adma_use_reg_mode(qc)) {
fbbb262d 1413 /* use ATA register mode */
382a6652 1414 VPRINTK("using ATA register mode: 0x%lx\n", qc->flags);
3f3debdb
RH		 1415 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) &&
1416 (qc->flags & ATA_QCFLAG_DMAMAP));
fbbb262d 1417 nv_adma_register_mode(qc->ap);
360ff783 1418 return ata_bmdma_qc_issue(qc);
fbbb262d
RH		 1419 } else
1420 nv_adma_mode(qc->ap);
1421
1422 /* write append register, command tag in lower 8 bits
1423 and (number of cpbs to append -1) in top 8 bits */
1424 wmb();
5e5c74a5 1425
b447916e 1426 if (curr_ncq != pp->last_issue_ncq) {
5796d1c4
JG		 1427 /* Seems to need some delay before switching between NCQ and
1428 non-NCQ commands, else we get command timeouts and such. */
5e5c74a5
RH		 1429 udelay(20);
1430 pp->last_issue_ncq = curr_ncq;
1431 }
1432
4e5b6260 1433 writew(qc->hw_tag, mmio + NV_ADMA_APPEND);
fbbb262d 1434
4e5b6260 1435 DPRINTK("Issued tag %u\n", qc->hw_tag);
fbbb262d
RH		 1436
1437 return 0;
1438}
1439
7d12e780 1440static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance)
1da177e4 1441{
cca3974e 1442 struct ata_host *host = dev_instance;
1da177e4
LT		 1443 unsigned int i;
1444 unsigned int handled = 0;
1445 unsigned long flags;
1446
cca3974e 1447 spin_lock_irqsave(&host->lock, flags);
1da177e4 1448
cca3974e 1449 for (i = 0; i < host->n_ports; i++) {
3e4ec344
TH		 1450 struct ata_port *ap = host->ports[i];
1451 struct ata_queued_cmd *qc;
1da177e4 1452
3e4ec344
TH		 1453 qc = ata_qc_from_tag(ap, ap->link.active_tag);
1454 if (qc && (!(qc->tf.flags & ATA_TFLAG_POLLING))) {
c3b28894 1455 handled += ata_bmdma_port_intr(ap, qc);
3e4ec344
TH		 1456 } else {
1457 /*
1458 * No request pending? Clear interrupt status
1459 * anyway, in case there's one pending.
1460 */
1461 ap->ops->sff_check_status(ap);
1462 }
1da177e4
LT		 1463 }
1464
cca3974e 1465 spin_unlock_irqrestore(&host->lock, flags);
1da177e4
LT		 1466
1467 return IRQ_RETVAL(handled);
1468}
1469
cca3974e 1470static irqreturn_t nv_do_interrupt(struct ata_host *host, u8 irq_stat)
ada364e8
TH		 1471{
1472 int i, handled = 0;
1473
cca3974e 1474 for (i = 0; i < host->n_ports; i++) {
3e4ec344 1475 handled += nv_host_intr(host->ports[i], irq_stat);
ada364e8
TH		 1476 irq_stat >>= NV_INT_PORT_SHIFT;
1477 }
1478
1479 return IRQ_RETVAL(handled);
1480}
1481
7d12e780 1482static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance)
ada364e8 1483{
cca3974e 1484 struct ata_host *host = dev_instance;
ada364e8
TH		 1485 u8 irq_stat;
1486 irqreturn_t ret;
1487
cca3974e 1488 spin_lock(&host->lock);
0d5ff566 1489 irq_stat = ioread8(host->ports[0]->ioaddr.scr_addr + NV_INT_STATUS);
cca3974e
JG		 1490 ret = nv_do_interrupt(host, irq_stat);
1491 spin_unlock(&host->lock);
ada364e8
TH		 1492
1493 return ret;
1494}
1495
7d12e780 1496static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance)
ada364e8 1497{
cca3974e 1498 struct ata_host *host = dev_instance;
ada364e8
TH		 1499 u8 irq_stat;
1500 irqreturn_t ret;
1501
cca3974e 1502 spin_lock(&host->lock);
0d5ff566 1503 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804);
cca3974e
JG		 1504 ret = nv_do_interrupt(host, irq_stat);
1505 spin_unlock(&host->lock);
ada364e8
TH		 1506
1507 return ret;
1508}
1509
82ef04fb 1510static int nv_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val)
1da177e4 1511{
1da177e4 1512 if (sc_reg > SCR_CONTROL)
da3dbb17 1513 return -EINVAL;
1da177e4 1514
82ef04fb 1515 *val = ioread32(link->ap->ioaddr.scr_addr + (sc_reg * 4));
da3dbb17 1516 return 0;
1da177e4
LT		 1517}
1518
82ef04fb 1519static int nv_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val)
1da177e4 1520{
1da177e4 1521 if (sc_reg > SCR_CONTROL)
da3dbb17 1522 return -EINVAL;
1da177e4 1523
82ef04fb 1524 iowrite32(val, link->ap->ioaddr.scr_addr + (sc_reg * 4));
da3dbb17 1525 return 0;
1da177e4
LT		 1526}
1527
7f4774b3
TH		 1528static int nv_hardreset(struct ata_link *link, unsigned int *class,
1529 unsigned long deadline)
e8caa3c7 1530{
7f4774b3 1531 struct ata_eh_context *ehc = &link->eh_context;
e8caa3c7 1532
7f4774b3
TH		 1533 /* Do hardreset iff it's post-boot probing, please read the
1534 * comment above port ops for details.
1535 */
1536 if (!(link->ap->pflags & ATA_PFLAG_LOADING) &&
1537 !ata_dev_enabled(link->device))
1538 sata_link_hardreset(link, sata_deb_timing_hotplug, deadline,
1539 NULL, NULL);
6489e326
TH		 1540 else {
1541 const unsigned long *timing = sata_ehc_deb_timing(ehc);
1542 int rc;
1543
1544 if (!(ehc->i.flags & ATA_EHI_QUIET))
a9a79dfe
JP		 1545 ata_link_info(link,
1546 "nv: skipping hardreset on occupied port\n");
6489e326
TH		 1547
1548 /* make sure the link is online */
1549 rc = sata_link_resume(link, timing, deadline);
1550 /* whine about phy resume failure but proceed */
1551 if (rc && rc != -EOPNOTSUPP)
a9a79dfe
JP		 1552 ata_link_warn(link, "failed to resume link (errno=%d)\n",
1553 rc);
6489e326 1554 }
7f4774b3
TH		 1555
1556 /* device signature acquisition is unreliable */
1557 return -EAGAIN;
e8caa3c7
TH		 1558}
1559
39f87582
TH		 1560static void nv_nf2_freeze(struct ata_port *ap)
1561{
0d5ff566 1562 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
39f87582
TH		 1563 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1564 u8 mask;
1565
0d5ff566 1566 mask = ioread8(scr_addr + NV_INT_ENABLE);
39f87582 1567 mask &= ~(NV_INT_ALL << shift);
0d5ff566 1568 iowrite8(mask, scr_addr + NV_INT_ENABLE);
39f87582
TH		 1569}
1570
1571static void nv_nf2_thaw(struct ata_port *ap)
1572{
0d5ff566 1573 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr;
39f87582
TH		 1574 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1575 u8 mask;
1576
0d5ff566 1577 iowrite8(NV_INT_ALL << shift, scr_addr + NV_INT_STATUS);
39f87582 1578
0d5ff566 1579 mask = ioread8(scr_addr + NV_INT_ENABLE);
39f87582 1580 mask |= (NV_INT_MASK << shift);
0d5ff566 1581 iowrite8(mask, scr_addr + NV_INT_ENABLE);
39f87582
TH		 1582}
1583
1584static void nv_ck804_freeze(struct ata_port *ap)
1585{
0d5ff566 1586 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
39f87582
TH		 1587 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1588 u8 mask;
1589
1590 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1591 mask &= ~(NV_INT_ALL << shift);
1592 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1593}
1594
1595static void nv_ck804_thaw(struct ata_port *ap)
1596{
0d5ff566 1597 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
39f87582
TH		 1598 int shift = ap->port_no * NV_INT_PORT_SHIFT;
1599 u8 mask;
1600
1601 writeb(NV_INT_ALL << shift, mmio_base + NV_INT_STATUS_CK804);
1602
1603 mask = readb(mmio_base + NV_INT_ENABLE_CK804);
1604 mask |= (NV_INT_MASK << shift);
1605 writeb(mask, mmio_base + NV_INT_ENABLE_CK804);
1606}
1607
f140f0f1
KL		 1608static void nv_mcp55_freeze(struct ata_port *ap)
1609{
1610 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1611 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1612 u32 mask;
1613
1614 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1615
1616 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1617 mask &= ~(NV_INT_ALL_MCP55 << shift);
1618 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
f140f0f1
KL		 1619}
1620
1621static void nv_mcp55_thaw(struct ata_port *ap)
1622{
1623 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR];
1624 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55;
1625 u32 mask;
1626
1627 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55);
1628
1629 mask = readl(mmio_base + NV_INT_ENABLE_MCP55);
1630 mask |= (NV_INT_MASK_MCP55 << shift);
1631 writel(mask, mmio_base + NV_INT_ENABLE_MCP55);
f140f0f1
KL		 1632}
1633
fbbb262d
RH		 1634static void nv_adma_error_handler(struct ata_port *ap)
1635{
1636 struct nv_adma_port_priv *pp = ap->private_data;
b447916e 1637 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
cdf56bcf 1638 void __iomem *mmio = pp->ctl_block;
fbbb262d
RH		 1639 int i;
1640 u16 tmp;
a84471fe 1641
b447916e 1642 if (ata_tag_valid(ap->link.active_tag) || ap->link.sactive) {
2cb27853
RH		 1643 u32 notifier = readl(mmio + NV_ADMA_NOTIFIER);
1644 u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);
1645 u32 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL);
1646 u32 status = readw(mmio + NV_ADMA_STAT);
08af7414
RH		 1647 u8 cpb_count = readb(mmio + NV_ADMA_CPB_COUNT);
1648 u8 next_cpb_idx = readb(mmio + NV_ADMA_NEXT_CPB_IDX);
2cb27853 1649
a9a79dfe 1650 ata_port_err(ap,
5796d1c4 1651 "EH in ADMA mode, notifier 0x%X "
08af7414
RH		 1652 "notifier_error 0x%X gen_ctl 0x%X status 0x%X "
1653 "next cpb count 0x%X next cpb idx 0x%x\n",
1654 notifier, notifier_error, gen_ctl, status,
1655 cpb_count, next_cpb_idx);
2cb27853 1656
b447916e 1657 for (i = 0; i < NV_ADMA_MAX_CPBS; i++) {
2cb27853 1658 struct nv_adma_cpb *cpb = &pp->cpb[i];
b447916e 1659 if ((ata_tag_valid(ap->link.active_tag) && i == ap->link.active_tag) ||
5796d1c4 1660 ap->link.sactive & (1 << i))
a9a79dfe 1661 ata_port_err(ap,
2cb27853
RH		 1662 "CPB %d: ctl_flags 0x%x, resp_flags 0x%x\n",
1663 i, cpb->ctl_flags, cpb->resp_flags);
1664 }
1665 }
fbbb262d 1666
fbbb262d
RH		 1667 /* Push us back into port register mode for error handling. */
1668 nv_adma_register_mode(ap);
1669
5796d1c4
JG		 1670 /* Mark all of the CPBs as invalid to prevent them from
1671 being executed */
b447916e 1672 for (i = 0; i < NV_ADMA_MAX_CPBS; i++)
fbbb262d
RH		 1673 pp->cpb[i].ctl_flags &= ~NV_CPB_CTL_CPB_VALID;
1674
1675 /* clear CPB fetch count */
1676 writew(0, mmio + NV_ADMA_CPB_COUNT);
1677
1678 /* Reset channel */
1679 tmp = readw(mmio + NV_ADMA_CTL);
1680 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
b447916e 1681 readw(mmio + NV_ADMA_CTL); /* flush posted write */
fbbb262d
RH		 1682 udelay(1);
1683 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
b447916e 1684 readw(mmio + NV_ADMA_CTL); /* flush posted write */
fbbb262d
RH		 1685 }
1686
fe06e5f9 1687 ata_bmdma_error_handler(ap);
fbbb262d
RH		 1688}
1689
f140f0f1
KL		 1690static void nv_swncq_qc_to_dq(struct ata_port *ap, struct ata_queued_cmd *qc)
1691{
1692 struct nv_swncq_port_priv *pp = ap->private_data;
1693 struct defer_queue *dq = &pp->defer_queue;
1694
1695 /* queue is full */
1696 WARN_ON(dq->tail - dq->head == ATA_MAX_QUEUE);
4e5b6260
JA		 1697 dq->defer_bits |= (1 << qc->hw_tag);
1698 dq->tag[dq->tail++ & (ATA_MAX_QUEUE - 1)] = qc->hw_tag;
f140f0f1
KL		 1699}
1700
1701static struct ata_queued_cmd *nv_swncq_qc_from_dq(struct ata_port *ap)
1702{
1703 struct nv_swncq_port_priv *pp = ap->private_data;
1704 struct defer_queue *dq = &pp->defer_queue;
1705 unsigned int tag;
1706
1707 if (dq->head == dq->tail) /* null queue */
1708 return NULL;
1709
1710 tag = dq->tag[dq->head & (ATA_MAX_QUEUE - 1)];
1711 dq->tag[dq->head++ & (ATA_MAX_QUEUE - 1)] = ATA_TAG_POISON;
1712 WARN_ON(!(dq->defer_bits & (1 << tag)));
1713 dq->defer_bits &= ~(1 << tag);
1714
1715 return ata_qc_from_tag(ap, tag);
1716}
1717
1718static void nv_swncq_fis_reinit(struct ata_port *ap)
1719{
1720 struct nv_swncq_port_priv *pp = ap->private_data;
1721
1722 pp->dhfis_bits = 0;
1723 pp->dmafis_bits = 0;
1724 pp->sdbfis_bits = 0;
1725 pp->ncq_flags = 0;
1726}
1727
1728static void nv_swncq_pp_reinit(struct ata_port *ap)
1729{
1730 struct nv_swncq_port_priv *pp = ap->private_data;
1731 struct defer_queue *dq = &pp->defer_queue;
1732
1733 dq->head = 0;
1734 dq->tail = 0;
1735 dq->defer_bits = 0;
1736 pp->qc_active = 0;
1737 pp->last_issue_tag = ATA_TAG_POISON;
1738 nv_swncq_fis_reinit(ap);
1739}
1740
1741static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis)
1742{
1743 struct nv_swncq_port_priv *pp = ap->private_data;
1744
1745 writew(fis, pp->irq_block);
1746}
1747
1748static void __ata_bmdma_stop(struct ata_port *ap)
1749{
1750 struct ata_queued_cmd qc;
1751
1752 qc.ap = ap;
1753 ata_bmdma_stop(&qc);
1754}
1755
1756static void nv_swncq_ncq_stop(struct ata_port *ap)
1757{
1758 struct nv_swncq_port_priv *pp = ap->private_data;
1759 unsigned int i;
1760 u32 sactive;
1761 u32 done_mask;
1762
e3ed8939 1763 ata_port_err(ap, "EH in SWNCQ mode,QC:qc_active 0x%llX sactive 0x%X\n",
a9a79dfe
JP		 1764 ap->qc_active, ap->link.sactive);
1765 ata_port_err(ap,
f140f0f1
KL		 1766 "SWNCQ:qc_active 0x%X defer_bits 0x%X last_issue_tag 0x%x\n "
1767 "dhfis 0x%X dmafis 0x%X sdbfis 0x%X\n",
1768 pp->qc_active, pp->defer_queue.defer_bits, pp->last_issue_tag,
1769 pp->dhfis_bits, pp->dmafis_bits, pp->sdbfis_bits);
1770
a9a79dfe
JP		 1771 ata_port_err(ap, "ATA_REG 0x%X ERR_REG 0x%X\n",
1772 ap->ops->sff_check_status(ap),
1773 ioread8(ap->ioaddr.error_addr));
f140f0f1
KL		 1774
1775 sactive = readl(pp->sactive_block);
1776 done_mask = pp->qc_active ^ sactive;
1777
a9a79dfe 1778 ata_port_err(ap, "tag : dhfis dmafis sdbfis sactive\n");
f140f0f1
KL		 1779 for (i = 0; i < ATA_MAX_QUEUE; i++) {
1780 u8 err = 0;
1781 if (pp->qc_active & (1 << i))
1782 err = 0;
1783 else if (done_mask & (1 << i))
1784 err = 1;
1785 else
1786 continue;
1787
a9a79dfe
JP		 1788 ata_port_err(ap,
1789 "tag 0x%x: %01x %01x %01x %01x %s\n", i,
1790 (pp->dhfis_bits >> i) & 0x1,
1791 (pp->dmafis_bits >> i) & 0x1,
1792 (pp->sdbfis_bits >> i) & 0x1,
1793 (sactive >> i) & 0x1,
1794 (err ? "error! tag doesn't exit" : " "));
f140f0f1
KL		 1795 }
1796
1797 nv_swncq_pp_reinit(ap);
5682ed33 1798 ap->ops->sff_irq_clear(ap);
f140f0f1
KL		 1799 __ata_bmdma_stop(ap);
1800 nv_swncq_irq_clear(ap, 0xffff);
1801}
1802
1803static void nv_swncq_error_handler(struct ata_port *ap)
1804{
1805 struct ata_eh_context *ehc = &ap->link.eh_context;
1806
1807 if (ap->link.sactive) {
1808 nv_swncq_ncq_stop(ap);
cf480626 1809 ehc->i.action |= ATA_EH_RESET;
f140f0f1
KL		 1810 }
1811
fe06e5f9 1812 ata_bmdma_error_handler(ap);
f140f0f1
KL		 1813}
1814
1815#ifdef CONFIG_PM
1816static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg)
1817{
1818 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1819 u32 tmp;
1820
1821 /* clear irq */
1822 writel(~0, mmio + NV_INT_STATUS_MCP55);
1823
1824 /* disable irq */
1825 writel(0, mmio + NV_INT_ENABLE_MCP55);
1826
1827 /* disable swncq */
1828 tmp = readl(mmio + NV_CTL_MCP55);
1829 tmp &= ~(NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ);
1830 writel(tmp, mmio + NV_CTL_MCP55);
1831
1832 return 0;
1833}
1834
1835static int nv_swncq_port_resume(struct ata_port *ap)
1836{
1837 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1838 u32 tmp;
1839
1840 /* clear irq */
1841 writel(~0, mmio + NV_INT_STATUS_MCP55);
1842
1843 /* enable irq */
1844 writel(0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1845
1846 /* enable swncq */
1847 tmp = readl(mmio + NV_CTL_MCP55);
1848 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1849
1850 return 0;
1851}
1852#endif
1853
1854static void nv_swncq_host_init(struct ata_host *host)
1855{
1856 u32 tmp;
1857 void __iomem *mmio = host->iomap[NV_MMIO_BAR];
1858 struct pci_dev *pdev = to_pci_dev(host->dev);
1859 u8 regval;
1860
1861 /* disable ECO 398 */
1862 pci_read_config_byte(pdev, 0x7f, &regval);
1863 regval &= ~(1 << 7);
1864 pci_write_config_byte(pdev, 0x7f, regval);
1865
1866 /* enable swncq */
1867 tmp = readl(mmio + NV_CTL_MCP55);
1868 VPRINTK("HOST_CTL:0x%X\n", tmp);
1869 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55);
1870
1871 /* enable irq intr */
1872 tmp = readl(mmio + NV_INT_ENABLE_MCP55);
1873 VPRINTK("HOST_ENABLE:0x%X\n", tmp);
1874 writel(tmp | 0x00fd00fd, mmio + NV_INT_ENABLE_MCP55);
1875
1876 /* clear port irq */
1877 writel(~0x0, mmio + NV_INT_STATUS_MCP55);
1878}
1879
1880static int nv_swncq_slave_config(struct scsi_device *sdev)
1881{
1882 struct ata_port *ap = ata_shost_to_port(sdev->host);
1883 struct pci_dev *pdev = to_pci_dev(ap->host->dev);
1884 struct ata_device *dev;
1885 int rc;
1886 u8 rev;
1887 u8 check_maxtor = 0;
1888 unsigned char model_num[ATA_ID_PROD_LEN + 1];
1889
1890 rc = ata_scsi_slave_config(sdev);
1891 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun)
1892 /* Not a proper libata device, ignore */
1893 return rc;
1894
1895 dev = &ap->link.device[sdev->id];
1896 if (!(ap->flags & ATA_FLAG_NCQ) || dev->class == ATA_DEV_ATAPI)
1897 return rc;
1898
1899 /* if MCP51 and Maxtor, then disable ncq */
1900 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA ||
1901 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2)
1902 check_maxtor = 1;
1903
1904 /* if MCP55 and rev <= a2 and Maxtor, then disable ncq */
1905 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA ||
1906 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2) {
1907 pci_read_config_byte(pdev, 0x8, &rev);
1908 if (rev <= 0xa2)
1909 check_maxtor = 1;
1910 }
1911
1912 if (!check_maxtor)
1913 return rc;
1914
1915 ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num));
1916
1917 if (strncmp(model_num, "Maxtor", 6) == 0) {
db5ed4df 1918 ata_scsi_change_queue_depth(sdev, 1);
a9a79dfe
JP		 1919 ata_dev_notice(dev, "Disabling SWNCQ mode (depth %x)\n",
1920 sdev->queue_depth);
f140f0f1
KL		 1921 }
1922
1923 return rc;
1924}
1925
1926static int nv_swncq_port_start(struct ata_port *ap)
1927{
1928 struct device *dev = ap->host->dev;
1929 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR];
1930 struct nv_swncq_port_priv *pp;
1931 int rc;
1932
c7087652
TH		 1933 /* we might fallback to bmdma, allocate bmdma resources */
1934 rc = ata_bmdma_port_start(ap);
f140f0f1
KL		 1935 if (rc)
1936 return rc;
1937
1938 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1939 if (!pp)
1940 return -ENOMEM;
1941
1942 pp->prd = dmam_alloc_coherent(dev, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE,
1943 &pp->prd_dma, GFP_KERNEL);
1944 if (!pp->prd)
1945 return -ENOMEM;
f140f0f1
KL		 1946
1947 ap->private_data = pp;
1948 pp->sactive_block = ap->ioaddr.scr_addr + 4 * SCR_ACTIVE;
1949 pp->irq_block = mmio + NV_INT_STATUS_MCP55 + ap->port_no * 2;
1950 pp->tag_block = mmio + NV_NCQ_REG_MCP55 + ap->port_no * 2;
1951
1952 return 0;
1953}
1954
95364f36 1955static enum ata_completion_errors nv_swncq_qc_prep(struct ata_queued_cmd *qc)
f140f0f1
KL		 1956{
1957 if (qc->tf.protocol != ATA_PROT_NCQ) {
f47451c4 1958 ata_bmdma_qc_prep(qc);
95364f36 1959 return AC_ERR_OK;
f140f0f1
KL		 1960 }
1961
1962 if (!(qc->flags & ATA_QCFLAG_DMAMAP))
95364f36 1963 return AC_ERR_OK;
f140f0f1
KL		 1964
1965 nv_swncq_fill_sg(qc);
95364f36
JS		 1966
1967 return AC_ERR_OK;
f140f0f1
KL		 1968}
1969
1970static void nv_swncq_fill_sg(struct ata_queued_cmd *qc)
1971{
1972 struct ata_port *ap = qc->ap;
1973 struct scatterlist *sg;
f140f0f1 1974 struct nv_swncq_port_priv *pp = ap->private_data;
f60d7011 1975 struct ata_bmdma_prd *prd;
ff2aeb1e 1976 unsigned int si, idx;
f140f0f1 1977
4e5b6260 1978 prd = pp->prd + ATA_MAX_PRD * qc->hw_tag;
f140f0f1
KL		 1979
1980 idx = 0;
ff2aeb1e 1981 for_each_sg(qc->sg, sg, qc->n_elem, si) {
f140f0f1
KL		 1982 u32 addr, offset;
1983 u32 sg_len, len;
1984
1985 addr = (u32)sg_dma_address(sg);
1986 sg_len = sg_dma_len(sg);
1987
1988 while (sg_len) {
1989 offset = addr & 0xffff;
1990 len = sg_len;
1991 if ((offset + sg_len) > 0x10000)
1992 len = 0x10000 - offset;
1993
1994 prd[idx].addr = cpu_to_le32(addr);
1995 prd[idx].flags_len = cpu_to_le32(len & 0xffff);
1996
1997 idx++;
1998 sg_len -= len;
1999 addr += len;
2000 }
2001 }
2002
ff2aeb1e 2003 prd[idx - 1].flags_len |= cpu_to_le32(ATA_PRD_EOT);
f140f0f1
KL		 2004}
2005
2006static unsigned int nv_swncq_issue_atacmd(struct ata_port *ap,
2007 struct ata_queued_cmd *qc)
2008{
2009 struct nv_swncq_port_priv *pp = ap->private_data;
2010
2011 if (qc == NULL)
2012 return 0;
2013
2014 DPRINTK("Enter\n");
2015
4e5b6260
JA		 2016 writel((1 << qc->hw_tag), pp->sactive_block);
2017 pp->last_issue_tag = qc->hw_tag;
2018 pp->dhfis_bits &= ~(1 << qc->hw_tag);
2019 pp->dmafis_bits &= ~(1 << qc->hw_tag);
2020 pp->qc_active |= (0x1 << qc->hw_tag);
f140f0f1 2021
5682ed33
TH		 2022 ap->ops->sff_tf_load(ap, &qc->tf); /* load tf registers */
2023 ap->ops->sff_exec_command(ap, &qc->tf);
f140f0f1 2024
4e5b6260 2025 DPRINTK("Issued tag %u\n", qc->hw_tag);
f140f0f1
KL		 2026
2027 return 0;
2028}
2029
2030static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc)
2031{
2032 struct ata_port *ap = qc->ap;
2033 struct nv_swncq_port_priv *pp = ap->private_data;
2034
2035 if (qc->tf.protocol != ATA_PROT_NCQ)
360ff783 2036 return ata_bmdma_qc_issue(qc);
f140f0f1
KL		 2037
2038 DPRINTK("Enter\n");
2039
2040 if (!pp->qc_active)
2041 nv_swncq_issue_atacmd(ap, qc);
2042 else
2043 nv_swncq_qc_to_dq(ap, qc); /* add qc to defer queue */
2044
2045 return 0;
2046}
2047
2048static void nv_swncq_hotplug(struct ata_port *ap, u32 fis)
2049{
2050 u32 serror;
2051 struct ata_eh_info *ehi = &ap->link.eh_info;
2052
2053 ata_ehi_clear_desc(ehi);
2054
2055 /* AHCI needs SError cleared; otherwise, it might lock up */
2056 sata_scr_read(&ap->link, SCR_ERROR, &serror);
2057 sata_scr_write(&ap->link, SCR_ERROR, serror);
2058
2059 /* analyze @irq_stat */
2060 if (fis & NV_SWNCQ_IRQ_ADDED)
2061 ata_ehi_push_desc(ehi, "hot plug");
2062 else if (fis & NV_SWNCQ_IRQ_REMOVED)
2063 ata_ehi_push_desc(ehi, "hot unplug");
2064
2065 ata_ehi_hotplugged(ehi);
2066
2067 /* okay, let's hand over to EH */
2068 ehi->serror |= serror;
2069
2070 ata_port_freeze(ap);
2071}
2072
2073static int nv_swncq_sdbfis(struct ata_port *ap)
2074{
2075 struct ata_queued_cmd *qc;
2076 struct nv_swncq_port_priv *pp = ap->private_data;
2077 struct ata_eh_info *ehi = &ap->link.eh_info;
2078 u32 sactive;
f140f0f1 2079 u32 done_mask;
f140f0f1
KL		 2080 u8 host_stat;
2081 u8 lack_dhfis = 0;
2082
2083 host_stat = ap->ops->bmdma_status(ap);
2084 if (unlikely(host_stat & ATA_DMA_ERR)) {
25985edc 2085 /* error when transferring data to/from memory */
f140f0f1
KL		 2086 ata_ehi_clear_desc(ehi);
2087 ata_ehi_push_desc(ehi, "BMDMA stat 0x%x", host_stat);
2088 ehi->err_mask |= AC_ERR_HOST_BUS;
cf480626 2089 ehi->action |= ATA_EH_RESET;
f140f0f1
KL		 2090 return -EINVAL;
2091 }
2092
5682ed33 2093 ap->ops->sff_irq_clear(ap);
f140f0f1
KL		 2094 __ata_bmdma_stop(ap);
2095
2096 sactive = readl(pp->sactive_block);
2097 done_mask = pp->qc_active ^ sactive;
2098
1aadf5c3
TH		 2099 pp->qc_active &= ~done_mask;
2100 pp->dhfis_bits &= ~done_mask;
2101 pp->dmafis_bits &= ~done_mask;
2102 pp->sdbfis_bits |= done_mask;
2103 ata_qc_complete_multiple(ap, ap->qc_active ^ done_mask);
f140f0f1
KL		 2104
2105 if (!ap->qc_active) {
2106 DPRINTK("over\n");
2107 nv_swncq_pp_reinit(ap);
752e386c 2108 return 0;
f140f0f1
KL		 2109 }
2110
2111 if (pp->qc_active & pp->dhfis_bits)
752e386c 2112 return 0;
f140f0f1
KL		 2113
2114 if ((pp->ncq_flags & ncq_saw_backout) ||
2115 (pp->qc_active ^ pp->dhfis_bits))
752e386c 2116 /* if the controller can't get a device to host register FIS,
f140f0f1
KL		 2117 * The driver needs to reissue the new command.
2118 */
2119 lack_dhfis = 1;
2120
2121 DPRINTK("id 0x%x QC: qc_active 0x%x,"
2122 "SWNCQ:qc_active 0x%X defer_bits %X "
2123 "dhfis 0x%X dmafis 0x%X last_issue_tag %x\n",
2124 ap->print_id, ap->qc_active, pp->qc_active,
2125 pp->defer_queue.defer_bits, pp->dhfis_bits,
2126 pp->dmafis_bits, pp->last_issue_tag);
2127
2128 nv_swncq_fis_reinit(ap);
2129
2130 if (lack_dhfis) {
2131 qc = ata_qc_from_tag(ap, pp->last_issue_tag);
2132 nv_swncq_issue_atacmd(ap, qc);
752e386c 2133 return 0;
f140f0f1
KL		 2134 }
2135
2136 if (pp->defer_queue.defer_bits) {
2137 /* send deferral queue command */
2138 qc = nv_swncq_qc_from_dq(ap);
2139 WARN_ON(qc == NULL);
2140 nv_swncq_issue_atacmd(ap, qc);
2141 }
2142
752e386c 2143 return 0;
f140f0f1
KL		 2144}
2145
2146static inline u32 nv_swncq_tag(struct ata_port *ap)
2147{
2148 struct nv_swncq_port_priv *pp = ap->private_data;
2149 u32 tag;
2150
2151 tag = readb(pp->tag_block) >> 2;
2152 return (tag & 0x1f);
2153}
2154
752e386c 2155static void nv_swncq_dmafis(struct ata_port *ap)
f140f0f1
KL		 2156{
2157 struct ata_queued_cmd *qc;
2158 unsigned int rw;
2159 u8 dmactl;
2160 u32 tag;
2161 struct nv_swncq_port_priv *pp = ap->private_data;
2162
2163 __ata_bmdma_stop(ap);
2164 tag = nv_swncq_tag(ap);
2165
2166 DPRINTK("dma setup tag 0x%x\n", tag);
2167 qc = ata_qc_from_tag(ap, tag);
2168
2169 if (unlikely(!qc))
752e386c 2170 return;
f140f0f1
KL		 2171
2172 rw = qc->tf.flags & ATA_TFLAG_WRITE;
2173
2174 /* load PRD table addr. */
4e5b6260 2175 iowrite32(pp->prd_dma + ATA_PRD_TBL_SZ * qc->hw_tag,
f140f0f1
KL		 2176 ap->ioaddr.bmdma_addr + ATA_DMA_TABLE_OFS);
2177
2178 /* specify data direction, triple-check start bit is clear */
2179 dmactl = ioread8(ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
2180 dmactl &= ~ATA_DMA_WR;
2181 if (!rw)
2182 dmactl |= ATA_DMA_WR;
2183
2184 iowrite8(dmactl | ATA_DMA_START, ap->ioaddr.bmdma_addr + ATA_DMA_CMD);
f140f0f1
KL		 2185}
2186
2187static void nv_swncq_host_interrupt(struct ata_port *ap, u16 fis)
2188{
2189 struct nv_swncq_port_priv *pp = ap->private_data;
2190 struct ata_queued_cmd *qc;
2191 struct ata_eh_info *ehi = &ap->link.eh_info;
2192 u32 serror;
2193 u8 ata_stat;
f140f0f1 2194
5682ed33 2195 ata_stat = ap->ops->sff_check_status(ap);
f140f0f1
KL		 2196 nv_swncq_irq_clear(ap, fis);
2197 if (!fis)
2198 return;
2199
2200 if (ap->pflags & ATA_PFLAG_FROZEN)
2201 return;
2202
2203 if (fis & NV_SWNCQ_IRQ_HOTPLUG) {
2204 nv_swncq_hotplug(ap, fis);
2205 return;
2206 }
2207
2208 if (!pp->qc_active)
2209 return;
2210
82ef04fb 2211 if (ap->ops->scr_read(&ap->link, SCR_ERROR, &serror))
f140f0f1 2212 return;
82ef04fb 2213 ap->ops->scr_write(&ap->link, SCR_ERROR, serror);
f140f0f1
KL		 2214
2215 if (ata_stat & ATA_ERR) {
2216 ata_ehi_clear_desc(ehi);
2217 ata_ehi_push_desc(ehi, "Ata error. fis:0x%X", fis);
2218 ehi->err_mask |= AC_ERR_DEV;
2219 ehi->serror |= serror;
cf480626 2220 ehi->action |= ATA_EH_RESET;
f140f0f1
KL		 2221 ata_port_freeze(ap);
2222 return;
2223 }
2224
2225 if (fis & NV_SWNCQ_IRQ_BACKOUT) {
2226 /* If the IRQ is backout, driver must issue
2227 * the new command again some time later.
2228 */
2229 pp->ncq_flags |= ncq_saw_backout;
2230 }
2231
2232 if (fis & NV_SWNCQ_IRQ_SDBFIS) {
2233 pp->ncq_flags |= ncq_saw_sdb;
2234 DPRINTK("id 0x%x SWNCQ: qc_active 0x%X "
2235 "dhfis 0x%X dmafis 0x%X sactive 0x%X\n",
2236 ap->print_id, pp->qc_active, pp->dhfis_bits,
2237 pp->dmafis_bits, readl(pp->sactive_block));
752e386c 2238 if (nv_swncq_sdbfis(ap) < 0)
f140f0f1
KL		 2239 goto irq_error;
2240 }
2241
2242 if (fis & NV_SWNCQ_IRQ_DHREGFIS) {
2243 /* The interrupt indicates the new command
2244 * was transmitted correctly to the drive.
2245 */
2246 pp->dhfis_bits |= (0x1 << pp->last_issue_tag);
2247 pp->ncq_flags |= ncq_saw_d2h;
2248 if (pp->ncq_flags & (ncq_saw_sdb | ncq_saw_backout)) {
2249 ata_ehi_push_desc(ehi, "illegal fis transaction");
2250 ehi->err_mask |= AC_ERR_HSM;
cf480626 2251 ehi->action |= ATA_EH_RESET;
f140f0f1
KL		 2252 goto irq_error;
2253 }
2254
2255 if (!(fis & NV_SWNCQ_IRQ_DMASETUP) &&
2256 !(pp->ncq_flags & ncq_saw_dmas)) {
5682ed33 2257 ata_stat = ap->ops->sff_check_status(ap);
f140f0f1
KL		 2258 if (ata_stat & ATA_BUSY)
2259 goto irq_exit;
2260
2261 if (pp->defer_queue.defer_bits) {
2262 DPRINTK("send next command\n");
2263 qc = nv_swncq_qc_from_dq(ap);
2264 nv_swncq_issue_atacmd(ap, qc);
2265 }
2266 }
2267 }
2268
2269 if (fis & NV_SWNCQ_IRQ_DMASETUP) {
2270 /* program the dma controller with appropriate PRD buffers
2271 * and start the DMA transfer for requested command.
2272 */
2273 pp->dmafis_bits |= (0x1 << nv_swncq_tag(ap));
2274 pp->ncq_flags |= ncq_saw_dmas;
752e386c 2275 nv_swncq_dmafis(ap);
f140f0f1
KL		 2276 }
2277
2278irq_exit:
2279 return;
2280irq_error:
2281 ata_ehi_push_desc(ehi, "fis:0x%x", fis);
2282 ata_port_freeze(ap);
2283 return;
2284}
2285
2286static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance)
2287{
2288 struct ata_host *host = dev_instance;
2289 unsigned int i;
2290 unsigned int handled = 0;
2291 unsigned long flags;
2292 u32 irq_stat;
2293
2294 spin_lock_irqsave(&host->lock, flags);
2295
2296 irq_stat = readl(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_MCP55);
2297
2298 for (i = 0; i < host->n_ports; i++) {
2299 struct ata_port *ap = host->ports[i];
2300
3e4ec344
TH		 2301 if (ap->link.sactive) {
2302 nv_swncq_host_interrupt(ap, (u16)irq_stat);
2303 handled = 1;
2304 } else {
2305 if (irq_stat) /* reserve Hotplug */
2306 nv_swncq_irq_clear(ap, 0xfff0);
f140f0f1 2307
3e4ec344 2308 handled += nv_host_intr(ap, (u8)irq_stat);
f140f0f1
KL		 2309 }
2310 irq_stat >>= NV_INT_PORT_SHIFT_MCP55;
2311 }
2312
2313 spin_unlock_irqrestore(&host->lock, flags);
2314
2315 return IRQ_RETVAL(handled);
2316}
2317
5796d1c4 2318static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
1da177e4 2319{
1626aeb8 2320 const struct ata_port_info *ppi[] = { NULL, NULL };
95947193 2321 struct nv_pi_priv *ipriv;
9a829ccf 2322 struct ata_host *host;
cdf56bcf 2323 struct nv_host_priv *hpriv;
1da177e4
LT		 2324 int rc;
2325 u32 bar;
0d5ff566 2326 void __iomem *base;
fbbb262d 2327 unsigned long type = ent->driver_data;
1da177e4
LT		 2328
2329 // Make sure this is a SATA controller by counting the number of bars
2330 // (NVIDIA SATA controllers will always have six bars). Otherwise,
2331 // it's an IDE controller and we ignore it.
c9c13ba4 2332 for (bar = 0; bar < PCI_STD_NUM_BARS; bar++)
1da177e4
LT		 2333 if (pci_resource_start(pdev, bar) == 0)
2334 return -ENODEV;
2335
06296a1e 2336 ata_print_version_once(&pdev->dev, DRV_VERSION);
1da177e4 2337
24dc5f33 2338 rc = pcim_enable_device(pdev);
1da177e4 2339 if (rc)
24dc5f33 2340 return rc;
1da177e4 2341
9a829ccf 2342 /* determine type and allocate host */
f140f0f1 2343 if (type == CK804 && adma_enabled) {
a44fec1f 2344 dev_notice(&pdev->dev, "Using ADMA mode\n");
fbbb262d 2345 type = ADMA;
2d775708 2346 } else if (type == MCP5x && swncq_enabled) {
a44fec1f 2347 dev_notice(&pdev->dev, "Using SWNCQ mode\n");
2d775708 2348 type = SWNCQ;
360737a9
JG		 2349 }
2350
1626aeb8 2351 ppi[0] = &nv_port_info[type];
95947193 2352 ipriv = ppi[0]->private_data;
1c5afdf7 2353 rc = ata_pci_bmdma_prepare_host(pdev, ppi, &host);
9a829ccf
TH		 2354 if (rc)
2355 return rc;
1da177e4 2356
24dc5f33 2357 hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL);
cdf56bcf 2358 if (!hpriv)
24dc5f33 2359 return -ENOMEM;
9a829ccf
TH		 2360 hpriv->type = type;
2361 host->private_data = hpriv;
cdf56bcf 2362
9a829ccf
TH		 2363 /* request and iomap NV_MMIO_BAR */
2364 rc = pcim_iomap_regions(pdev, 1 << NV_MMIO_BAR, DRV_NAME);
2365 if (rc)
2366 return rc;
1da177e4 2367
9a829ccf
TH		 2368 /* configure SCR access */
2369 base = host->iomap[NV_MMIO_BAR];
2370 host->ports[0]->ioaddr.scr_addr = base + NV_PORT0_SCR_REG_OFFSET;
2371 host->ports[1]->ioaddr.scr_addr = base + NV_PORT1_SCR_REG_OFFSET;
1da177e4 2372
ada364e8 2373 /* enable SATA space for CK804 */
fbbb262d 2374 if (type >= CK804) {
ada364e8
TH		 2375 u8 regval;
2376
2377 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2378 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2379 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2380 }
2381
9a829ccf 2382 /* init ADMA */
fbbb262d 2383 if (type == ADMA) {
9a829ccf 2384 rc = nv_adma_host_init(host);
fbbb262d 2385 if (rc)
24dc5f33 2386 return rc;
360737a9 2387 } else if (type == SWNCQ)
f140f0f1 2388 nv_swncq_host_init(host);
fbbb262d 2389
51c89499 2390 if (msi_enabled) {
a44fec1f 2391 dev_notice(&pdev->dev, "Using MSI\n");
51c89499
TV		 2392 pci_enable_msi(pdev);
2393 }
2394
9a829ccf 2395 pci_set_master(pdev);
95cc2c70 2396 return ata_pci_sff_activate_host(host, ipriv->irq_handler, ipriv->sht);
1da177e4
LT		 2397}
2398
58eb8cd5 2399#ifdef CONFIG_PM_SLEEP
cdf56bcf
RH		 2400static int nv_pci_device_resume(struct pci_dev *pdev)
2401{
0a86e1c8 2402 struct ata_host *host = pci_get_drvdata(pdev);
cdf56bcf 2403 struct nv_host_priv *hpriv = host->private_data;
ce053fa8 2404 int rc;
cdf56bcf 2405
ce053fa8 2406 rc = ata_pci_device_do_resume(pdev);
b447916e 2407 if (rc)
ce053fa8 2408 return rc;
cdf56bcf
RH		 2409
2410 if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
b447916e 2411 if (hpriv->type >= CK804) {
cdf56bcf
RH		 2412 u8 regval;
2413
2414 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2415 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2416 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
2417 }
b447916e 2418 if (hpriv->type == ADMA) {
cdf56bcf
RH		 2419 u32 tmp32;
2420 struct nv_adma_port_priv *pp;
2421 /* enable/disable ADMA on the ports appropriately */
2422 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2423
2424 pp = host->ports[0]->private_data;
b447916e 2425 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
cdf56bcf 2426 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
5796d1c4 2427 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
cdf56bcf
RH		 2428 else
2429 tmp32 |= (NV_MCP_SATA_CFG_20_PORT0_EN |
5796d1c4 2430 NV_MCP_SATA_CFG_20_PORT0_PWB_EN);
cdf56bcf 2431 pp = host->ports[1]->private_data;
b447916e 2432 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)
cdf56bcf 2433 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT1_EN |
5796d1c4 2434 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
cdf56bcf
RH		 2435 else
2436 tmp32 |= (NV_MCP_SATA_CFG_20_PORT1_EN |
5796d1c4 2437 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
cdf56bcf
RH		 2438
2439 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2440 }
2441 }
2442
2443 ata_host_resume(host);
2444
2445 return 0;
2446}
438ac6d5 2447#endif
cdf56bcf 2448
cca3974e 2449static void nv_ck804_host_stop(struct ata_host *host)
ada364e8 2450{
cca3974e 2451 struct pci_dev *pdev = to_pci_dev(host->dev);
ada364e8
TH		 2452 u8 regval;
2453
2454 /* disable SATA space for CK804 */
2455 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
2456 regval &= ~NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
2457 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
ada364e8
TH		 2458}
2459
fbbb262d
RH		 2460static void nv_adma_host_stop(struct ata_host *host)
2461{
2462 struct pci_dev *pdev = to_pci_dev(host->dev);
fbbb262d
RH		 2463 u32 tmp32;
2464
fbbb262d
RH		 2465 /* disable ADMA on the ports */
2466 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
2467 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN |
2468 NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
2469 NV_MCP_SATA_CFG_20_PORT1_EN |
2470 NV_MCP_SATA_CFG_20_PORT1_PWB_EN);
2471
2472 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);
2473
2474 nv_ck804_host_stop(host);
2475}
2476
2fc75da0 2477module_pci_driver(nv_pci_driver);
1da177e4 2478
fbbb262d 2479module_param_named(adma, adma_enabled, bool, 0444);
55f784c8 2480MODULE_PARM_DESC(adma, "Enable use of ADMA (Default: false)");
f140f0f1 2481module_param_named(swncq, swncq_enabled, bool, 0444);
d21279f4 2482MODULE_PARM_DESC(swncq, "Enable use of SWNCQ (Default: true)");
51c89499
TV		 2483module_param_named(msi, msi_enabled, bool, 0444);
2484MODULE_PARM_DESC(msi, "Enable use of MSI (Default: false)");
A mirror of Linus' kernel repository