sys/src/9/pc/sdiahci.c

   1 /*
   2  * intel/amd ahci sata controller
   3  * copyright © 2007-10 coraid, inc.
   4  */
   5
   6 #include "u.h"
   7 #include "../port/lib.h"
   8 #include "mem.h"
   9 #include "dat.h"
  10 #include "fns.h"
  11 #include "io.h"
  12 #include "../port/error.h"
  13 #include "../port/sd.h"
  14 #include <fis.h>
  15 #include "ahci.h"
  16 #include "../port/led.h"
  17
  18 #pragma varargck        type    "T"     int
  19 #define dprint(...)     if(debug)       print(__VA_ARGS__); else USED(debug)
  20 #define idprint(...)    if(prid)        print(__VA_ARGS__); else USED(prid)
  21 #define aprint(...)     if(datapi)      print(__VA_ARGS__); else USED(datapi)
  22 #define ledprint(...)   if(dled)        print(__VA_ARGS__); else USED(dled)
  23 #define Pciwaddrh(a)    0
  24 #define Tname(c)        tname[(c)->type]
  25 #define Ticks           MACHP(0)->ticks
  26 #define MS2TK(t)        (((ulong)(t)*HZ)/1000)
  27
  28 enum {
  29         NCtlr   = 4,
  30         NCtlrdrv= 32,
  31         NDrive  = NCtlr*NCtlrdrv,
  32
  33         Fahdrs  = 4,
  34
  35         Read    = 0,
  36         Write,
  37
  38         Eesb    = 1<<0, /* must have (Eesb & Emtype) == 0 */
  39 };
  40
  41 /* pci space configuration */
  42 enum {
  43         Pmap    = 0x90,
  44         Ppcs    = 0x91,
  45         Prev    = 0xa8,
  46 };
  47
  48 enum {
  49         Tesb,
  50         Tich,
  51         Tsb600,
  52         Tjmicron,
  53         Tahci,
  54 };
  55
  56 static char *tname[] = {
  57         "63xxesb",
  58         "ich",
  59         "sb600",
  60         "jmicron",
  61         "ahci",
  62 };
  63
  64 enum {
  65         Dnull,
  66         Dmissing,
  67         Dnew,
  68         Dready,
  69         Derror,
  70         Dreset,
  71         Doffline,
  72         Dportreset,
  73         Dlast,
  74 };
  75
  76 static char *diskstates[Dlast] = {
  77         "null",
  78         "missing",
  79         "new",
  80         "ready",
  81         "error",
  82         "reset",
  83         "offline",
  84         "portreset",
  85 };
  86
  87 extern SDifc sdiahciifc;
  88 typedef struct Ctlr Ctlr;
  89
  90 enum {
  91         DMautoneg,
  92         DMsatai,
  93         DMsataii,
  94         DMsataiii,
  95         DMlast,
  96 };
  97
  98 static char *modes[DMlast] = {
  99         "auto",
 100         "satai",
 101         "sataii",
 102         "sataiii",
 103 };
 104
 105 typedef struct Htab Htab;
 106 struct Htab {
 107         ulong   bit;
 108         char    *name;
 109 };
 110
 111 typedef struct {
 112         Lock;
 113
 114         Ctlr    *ctlr;
 115         SDunit  *unit;
 116         char    name[10];
 117         Aport   *port;
 118         Aportm  portm;
 119         Aportc  portc;  /* redundant ptr to port and portm. */
 120         Ledport;
 121
 122         uchar   drivechange;
 123         uchar   nodma;
 124         uchar   state;
 125
 126         uvlong  sectors;
 127         uint    secsize;
 128         ulong   totick;
 129         ulong   lastseen;
 130         uint    wait;
 131         uchar   mode;
 132         uchar   active;
 133
 134         char    serial[20+1];
 135         char    firmware[8+1];
 136         char    model[40+1];
 137         uvlong  wwn;
 138
 139         ushort  info[0x200];
 140
 141         /*
 142          * ahci allows non-sequential ports.
 143          * to avoid this hassle, we let
 144          * driveno      ctlr*NCtlrdrv + unit
 145          * portno       nth available port
 146          */
 147         uint    driveno;
 148         uint    portno;
 149 } Drive;
 150
 151 struct Ctlr {
 152         Lock;
 153
 154         int     type;
 155         int     enabled;
 156         SDev    *sdev;
 157         Pcidev  *pci;
 158
 159         uchar   *mmio;
 160         ulong   *lmmio;
 161         Ahba    *hba;
 162         Aenc;
 163         uint    enctype;
 164
 165         Drive   rawdrive[NCtlrdrv];
 166         Drive*  drive[NCtlrdrv];
 167         int     ndrive;
 168
 169         uint    missirq;
 170 };
 171
 172 static  Ctlr    iactlr[NCtlr];
 173 static  SDev    sdevs[NCtlr];
 174 static  int     niactlr;
 175
 176 static  Drive   *iadrive[NDrive];
 177 static  int     niadrive;
 178
 179 static  int     debug;
 180 static  int     prid = 1;
 181 static  int     datapi;
 182 static  int     dled;
 183
 184 static char stab[] = {
 185 [0]     'i', 'm',
 186 [8]     't', 'c', 'p', 'e',
 187 [16]    'N', 'I', 'W', 'B', 'D', 'C', 'H', 'S', 'T', 'F', 'X'
 188 };
 189
 190 static void
 191 serrstr(ulong r, char *s, char *e)
 192 {
 193         int i;
 194
 195         e -= 3;
 196         for(i = 0; i < nelem(stab) && s < e; i++)
 197                 if(r & (1<<i) && stab[i]){
 198                         *s++ = stab[i];
 199                         if(SerrBad & (1<<i))
 200                                 *s++ = '*';
 201                 }
 202         *s = 0;
 203 }
 204
 205 static char ntab[] = "0123456789abcdef";
 206
 207 static void
 208 preg(uchar *reg, int n)
 209 {
 210         char buf[25*3+1], *e;
 211         int i;
 212
 213         e = buf;
 214         for(i = 0; i < n; i++){
 215                 *e++ = ntab[reg[i] >> 4];
 216                 *e++ = ntab[reg[i] & 0xf];
 217                 *e++ = ' ';
 218         }
 219         *e++ = '\n';
 220         *e = 0;
 221         dprint(buf);
 222 }
 223
 224 static void
 225 dreg(char *s, Aport *p)
 226 {
 227         dprint("%stask=%lux; cmd=%lux; ci=%lux; is=%lux\n",
 228                 s, p->task, p->cmd, p->ci, p->isr);
 229 }
 230
 231 static void
 232 esleep(int ms)
 233 {
 234         if(waserror())
 235                 return;
 236         tsleep(&up->sleep, return0, 0, ms);
 237         poperror();
 238 }
 239
 240 typedef struct {
 241         Aport   *p;
 242         int     i;
 243 } Asleep;
 244
 245 static int
 246 ahciclear(void *v)
 247 {
 248         Asleep *s;
 249
 250         s = v;
 251         return (s->p->ci & s->i) == 0;
 252 }
 253
 254 static void
 255 aesleep(Aportm *m, Asleep *a, int ms)
 256 {
 257         if(waserror())
 258                 return;
 259         tsleep(m, ahciclear, a, ms);
 260         poperror();
 261 }
 262
 263 static int
 264 ahciwait(Aportc *c, int ms)
 265 {
 266         Aport *p;
 267         Asleep as;
 268
 269         p = c->p;
 270         p->ci = 1;
 271         as.p = p;
 272         as.i = 1;
 273         aesleep(c->m, &as, ms);
 274         if((p->task & 1) == 0 && p->ci == 0)
 275                 return 0;
 276         dreg("ahciwait fail/timeout ", c->p);
 277         return -1;
 278 }
 279
 280 static Alist*
 281 mkalist(Aportm *m, uint flags, uchar *data, int len)
 282 {
 283         Actab *t;
 284         Alist *l;
 285         Aprdt *p;
 286
 287         t = m->ctab;
 288         if(data && len > 0){
 289                 p = &t->prdt;
 290                 p->dba = PCIWADDR(data);
 291                 p->dbahi = Pciwaddrh(data);
 292                 p->count = 1<<31 | len - 2 | 1;
 293                 flags |= 1<<16;
 294         }
 295         l = m->list;
 296         l->flags = flags | 0x5;
 297         l->len = 0;
 298         l->ctab = PCIWADDR(t);
 299         l->ctabhi = Pciwaddrh(t);
 300         return l;
 301 }
 302
 303 static int
 304 nop(Aportc *pc)
 305 {
 306         uchar *c;
 307
 308         if((pc->m->feat & Dnop) == 0)
 309                 return -1;
 310         c = pc->m->ctab->cfis;
 311         nopfis(pc->m, c, 0);
 312         mkalist(pc->m, Lwrite, 0, 0);
 313         return ahciwait(pc, 3*1000);
 314 }
 315
 316 static int
 317 setfeatures(Aportc *pc, uchar f, uint w)
 318 {
 319         uchar *c;
 320
 321         c = pc->m->ctab->cfis;
 322         featfis(pc->m, c, f);
 323         mkalist(pc->m, Lwrite, 0, 0);
 324         return ahciwait(pc, w);
 325 }
 326
 327 /*
 328  * ata 7, required for sata, requires that all devices "support"
 329  * udma mode 5,   however sata:pata bridges allow older devices
 330  * which may not.  the innodisk satadom, for example allows
 331  * only udma mode 2.  on the assumption that actual udma is
 332  * taking place on these bridges, we set the highest udma mode
 333  * available, or pio if there is no udma mode available.
 334  */
 335 static int
 336 settxmode(Aportc *pc, uchar f)
 337 {
 338         uchar *c;
 339
 340         c = pc->m->ctab->cfis;
 341         if(txmodefis(pc->m, c, f) == -1)
 342                 return 0;
 343         mkalist(pc->m, Lwrite, 0, 0);
 344         return ahciwait(pc, 3*1000);
 345 }
 346
 347 static void
 348 asleep(int ms)
 349 {
 350         if(up == nil || !islo())
 351                 delay(ms);
 352         else
 353                 esleep(ms);
 354 }
 355
 356 static void
 357 ahciportreset(Aportc *c, uint mode)
 358 {
 359         ulong *cmd, i;
 360         Aport *p;
 361
 362         p = c->p;
 363         cmd = &p->cmd;
 364         *cmd &= ~(Afre|Ast);
 365         for(i = 0; i < 500; i += 25){
 366                 if((*cmd & Acr) == 0)
 367                         break;
 368                 asleep(25);
 369         }
 370         if((*cmd & Apwr) != Apwr)
 371                 *cmd |= Apwr;
 372         p->sctl = 3*Aipm | 0*Aspd | Adet;
 373         delay(1);
 374         p->sctl = 3*Aipm | mode*Aspd;
 375 }
 376
 377 static int
 378 ahciflushcache(Aportc *pc)
 379 {
 380         uchar *c;
 381
 382         c = pc->m->ctab->cfis;
 383         flushcachefis(pc->m, c);
 384         mkalist(pc->m, Lwrite, 0, 0);
 385
 386         if(ahciwait(pc, 60000) == -1 || pc->p->task & (1|32)){
 387                 dprint("ahciflushcache fail [task %lux]\n", pc->p->task);
 388 //              preg(pc->m->fis.r, 20);
 389                 return -1;
 390         }
 391         return 0;
 392 }
 393
 394 static int
 395 ahciidentify0(Aportc *pc, void *id)
 396 {
 397         uchar *c;
 398         Actab *t;
 399
 400         t = pc->m->ctab;
 401         c = t->cfis;
 402         memset(id, 0, 0x200);
 403         identifyfis(pc->m, c);
 404         mkalist(pc->m, 0, id, 0x200);
 405         return ahciwait(pc, 3*1000);
 406 }
 407
 408 static vlong
 409 ahciidentify(Aportc *pc, ushort *id, uint *ss, char *d)
 410 {
 411         int i, n;
 412         vlong s;
 413         Aportm *m;
 414
 415         m = pc->m;
 416         for(i = 0;; i++){
 417                 if(i > 5 || ahciidentify0(pc, id) != 0)
 418                         return -1;
 419                 n = idpuis(id);
 420                 if(n & Pspinup && setfeatures(pc, 7, 20*1000) == -1)
 421                         print("%s: puis spinup fail\n", d);
 422                 if(n & Pidready)
 423                         break;
 424                 print("%s: puis waiting\n", d);
 425         }
 426         s = idfeat(m, id);
 427         *ss = idss(m, id);
 428         if(s == -1 || (m->feat&Dlba) == 0){
 429                 if((m->feat&Dlba) == 0)
 430                         dprint("%s: no lba support\n", d);
 431                 return -1;
 432         }
 433         return s;
 434 }
 435
 436 static int
 437 ahciquiet(Aport *a)
 438 {
 439         ulong *p, i;
 440
 441         p = &a->cmd;
 442         *p &= ~Ast;
 443         for(i = 0; i < 500; i += 50){
 444                 if((*p & Acr) == 0)
 445                         goto stop;
 446                 asleep(50);
 447         }
 448         return -1;
 449 stop:
 450         if((a->task & (ASdrq|ASbsy)) == 0){
 451                 *p |= Ast;
 452                 return 0;
 453         }
 454
 455         *p |= Aclo;
 456         for(i = 0; i < 500; i += 50){
 457                 if((*p & Aclo) == 0)
 458                         goto stop1;
 459                 asleep(50);
 460         }
 461         return -1;
 462 stop1:
 463         /* extra check */
 464         dprint("ahci: clo clear [task %lux]\n", a->task);
 465         if(a->task & ASbsy)
 466                 return -1;
 467         *p |= Afre | Ast;
 468         return 0;
 469 }
 470
 471 static int
 472 ahcicomreset(Aportc *pc)
 473 {
 474         uchar *c;
 475
 476         dreg("comreset ", pc->p);
 477         if(ahciquiet(pc->p) == -1){
 478                 dprint("ahci: ahciquiet failed\n");
 479                 return -1;
 480         }
 481         dreg("comreset ", pc->p);
 482
 483         c = pc->m->ctab->cfis;
 484         nopfis(pc->m, c, 1);
 485         mkalist(pc->m, Lclear | Lreset, 0, 0);
 486         if(ahciwait(pc, 500) == -1){
 487                 dprint("ahci: comreset1 failed\n");
 488                 return -1;
 489         }
 490         microdelay(250);
 491         dreg("comreset ", pc->p);
 492
 493         nopfis(pc->m, c, 0);
 494         mkalist(pc->m, Lwrite, 0, 0);
 495         if(ahciwait(pc, 150) == -1){
 496                 dprint("ahci: comreset2 failed\n");
 497                 return -1;
 498         }
 499         dreg("comreset ", pc->p);
 500         return 0;
 501 }
 502
 503 static int
 504 ahciidle(Aport *port)
 505 {
 506         ulong *p, i, r;
 507
 508         p = &port->cmd;
 509         if((*p & Arun) == 0)
 510                 return 0;
 511         *p &= ~Ast;
 512         r = 0;
 513         for(i = 0; i < 500; i += 25){
 514                 if((*p & Acr) == 0)
 515                         goto stop;
 516                 asleep(25);
 517         }
 518         r = -1;
 519 stop:
 520         if((*p & Afre) == 0)
 521                 return r;
 522         *p &= ~Afre;
 523         for(i = 0; i < 500; i += 25){
 524                 if((*p & Afre) == 0)
 525                         return 0;
 526                 asleep(25);
 527         }
 528         return -1;
 529 }
 530
 531 /*
 532  * §6.2.2.1  first part; comreset handled by reset disk.
 533  *      - remainder is handled by configdisk.
 534  *      - ahcirecover is a quick recovery from a failed command.
 535  */
 536 static int
 537 ahciswreset(Aportc *pc)
 538 {
 539         int i;
 540
 541         i = ahciidle(pc->p);
 542         pc->p->cmd |= Afre;
 543         if(i == -1)
 544                 return -1;
 545         if(pc->p->task & (ASdrq|ASbsy))
 546                 return -1;
 547         return 0;
 548 }
 549
 550 static int
 551 ahcirecover(Aportc *pc)
 552 {
 553         ahciswreset(pc);
 554         pc->p->cmd |= Ast;
 555         if(settxmode(pc, pc->m->udma) == -1)
 556                 return -1;
 557         return 0;
 558 }
 559
 560 static void*
 561 malign(int size, int align)
 562 {
 563         void *v;
 564
 565         v = xspanalloc(size, align, 0);
 566         memset(v, 0, size);
 567         return v;
 568 }
 569
 570 static void
 571 setupfis(Afis *f)
 572 {
 573         f->base = malign(0x100, 0x100);
 574         f->d = f->base + 0;
 575         f->p = f->base + 0x20;
 576         f->r = f->base + 0x40;
 577         f->u = f->base + 0x60;
 578         f->devicebits = (ulong*)(f->base + 0x58);
 579 }
 580
 581 static void
 582 ahciwakeup(Aportc *c, uint mode)
 583 {
 584         ushort s;
 585         Aport *p;
 586
 587         p = c->p;
 588         s = p->sstatus;
 589         if((s & Isleepy) == 0)
 590                 return;
 591         if((s & Smask) != Spresent){
 592                 dprint("ahci: slumbering drive missing [ss %.3ux]\n", s);
 593                 return;
 594         }
 595         ahciportreset(c, mode);
 596         dprint("ahci: wake %.3ux -> %.3lux\n", s, c->p->sstatus);
 597 }
 598
 599 static int
 600 ahciconfigdrive(Ahba *h, Aportc *c, int mode)
 601 {
 602         Aportm *m;
 603         Aport *p;
 604         int i;
 605
 606         p = c->p;
 607         m = c->m;
 608
 609         if(m->list == 0){
 610                 setupfis(&m->fis);
 611                 m->list = malign(sizeof *m->list, 1024);
 612                 m->ctab = malign(sizeof *m->ctab, 128);
 613         }
 614
 615         if(ahciidle(p) == -1){
 616                 dprint("ahci: port not idle\n");
 617                 return -1;
 618         }
 619
 620         p->list = PCIWADDR(m->list);
 621         p->listhi = Pciwaddrh(m->list);
 622         p->fis = PCIWADDR(m->fis.base);
 623         p->fishi = Pciwaddrh(m->fis.base);
 624
 625         p->cmd |= Afre;
 626
 627         if((p->cmd & Apwr) != Apwr)
 628                 p->cmd |= Apwr;
 629
 630         if((h->cap & Hss) != 0){
 631                 dprint("ahci: spin up ... [%.3lux]\n", p->sstatus);
 632                 for(i = 0; i < 1400; i += 50){
 633                         if((p->sstatus & Sbist) != 0)
 634                                 break;
 635                         if((p->sstatus & Smask) == Sphylink)
 636                                 break;
 637                         asleep(50);
 638                 }
 639         }
 640
 641         if((p->sstatus & SSmask) == (Isleepy | Spresent))
 642                 ahciwakeup(c, mode);
 643
 644         p->serror = SerrAll;
 645         p->ie = IEM;
 646
 647         /* we will get called again once phylink has been established */
 648         if((p->sstatus & Smask) != Sphylink)
 649                 return 0;
 650
 651         /* disable power managment sequence from book. */
 652         p->sctl = 3*Aipm | mode*Aspd | 0*Adet;
 653         p->cmd &= ~Aalpe;
 654
 655         p->cmd |= Afre | Ast;
 656
 657         return 0;
 658 }
 659
 660 static int
 661 ahcienable(Ahba *h)
 662 {
 663         h->ghc |= Hie;
 664         return 0;
 665 }
 666
 667 static int
 668 ahcidisable(Ahba *h)
 669 {
 670         h->ghc &= ~Hie;
 671         return 0;
 672 }
 673
 674 static int
 675 countbits(ulong u)
 676 {
 677         int i, n;
 678
 679         n = 0;
 680         for(i = 0; i < 32; i++)
 681                 if(u & (1<<i))
 682                         n++;
 683         return n;
 684 }
 685
 686 static int
 687 ahciconf(Ctlr *c)
 688 {
 689         uint u;
 690         Ahba *h;
 691
 692         h = c->hba = (Ahba*)c->mmio;
 693         u = h->cap;
 694
 695         if((u & Ham) == 0)
 696                 h->ghc |= Hae;
 697
 698         return countbits(h->pi);
 699 }
 700
 701 static int
 702 ahcihandoff(Ahba *h)
 703 {
 704         int wait;
 705
 706         if((h->cap2 & Boh) == 0)
 707                 return 0;
 708         h->bios |= Oos;
 709         for(wait = 0; wait < 2000; wait += 100){
 710                 if((h->bios & Bos) == 0)
 711                         return 0;
 712                 delay(100);
 713         }
 714         iprint("ahci: bios handoff timed out\n");
 715         return -1;
 716 }
 717
 718 static int
 719 ahcihbareset(Ahba *h)
 720 {
 721         int wait;
 722
 723         h->ghc |= Hhr;
 724         for(wait = 0; wait < 1000; wait += 100){
 725                 if((h->ghc & Hhr) == 0)
 726                         return 0;
 727                 delay(100);
 728         }
 729         return -1;
 730 }
 731
 732 static char*
 733 dnam(Drive *d)
 734 {
 735         char *s;
 736
 737         s = d->name;
 738         if(d->unit && d->unit->name)
 739                 s = d->unit->name;
 740         return s;
 741 }
 742
 743 static int
 744 identify(Drive *d)
 745 {
 746         uchar oserial[21];
 747         ushort *id;
 748         vlong osectors, s;
 749         SDunit *u;
 750
 751         id = d->info;
 752         s = ahciidentify(&d->portc, id, &d->secsize, dnam(d));
 753         if(s == -1)
 754                 return -1;
 755         osectors = d->sectors;
 756         memmove(oserial, d->serial, sizeof d->serial);
 757
 758         d->sectors = s;
 759
 760         idmove(d->serial, id+10, 20);
 761         idmove(d->firmware, id+23, 8);
 762         idmove(d->model, id+27, 40);
 763         d->wwn = idwwn(d->portc.m, id);
 764
 765         u = d->unit;
 766         memset(u->inquiry, 0, sizeof u->inquiry);
 767         u->inquiry[2] = 2;
 768         u->inquiry[3] = 2;
 769         u->inquiry[4] = sizeof u->inquiry - 4;
 770         memmove(u->inquiry+8, d->model, 40);
 771
 772         if(osectors != s || memcmp(oserial, d->serial, sizeof oserial)){
 773                 d->drivechange = 1;
 774                 d->nodma = 0;
 775                 u->sectors = 0;
 776         }
 777         return 0;
 778 }
 779
 780 static void
 781 clearci(Aport *p)
 782 {
 783         if(p->cmd & Ast){
 784                 p->cmd &= ~Ast;
 785                 p->cmd |=  Ast;
 786         }
 787 }
 788
 789 static int
 790 ignoreahdrs(Drive *d)
 791 {
 792         return d->portm.feat & Datapi && d->ctlr->type == Tsb600;
 793 }
 794
 795 static void
 796 updatedrive(Drive *d)
 797 {
 798         ulong f, cause, serr, s0, pr, ewake;
 799         Aport *p;
 800         static ulong last;
 801
 802         pr = 1;
 803         ewake = 0;
 804         f = 0;
 805         p = d->port;
 806         cause = p->isr;
 807         if(d->ctlr->type == Tjmicron)
 808                 cause &= ~Aifs;
 809         serr = p->serror;
 810         p->isr = cause;
 811
 812         if(p->ci == 0){
 813                 f |= Fdone;
 814                 pr = 0;
 815         }else if(cause & Adps){
 816                 pr = 0;
 817         }else if(cause & Atfes){
 818                 f |= Ferror;
 819                 ewake = 1;
 820                 pr = 0;
 821         }
 822         if(cause & Ifatal){
 823                 ewake = 1;
 824                 dprint("%s: fatal\n", dnam(d));
 825         }
 826         if(cause & Adhrs){
 827                 if(p->task & 33){
 828                         if(ignoreahdrs(d) && serr & ErrE)
 829                                 f |= Fahdrs;
 830                         dprint("%s: Adhrs cause %lux serr %lux task %lux\n",
 831                                 dnam(d), cause, serr, p->task);
 832                         f |= Ferror;
 833                         ewake = 1;
 834                 }
 835                 pr = 0;
 836         }
 837         if(p->task & 1 && last != cause)
 838                 dprint("%s: err ca %lux serr %lux task %lux sstat %.3lux\n",
 839                         dnam(d), cause, serr, p->task, p->sstatus);
 840         if(pr)
 841                 dprint("%s: upd %lux ta %lux\n", dnam(d), cause, p->task);
 842
 843         if(cause & (Aprcs|Aifs)){
 844                 s0 = d->state;
 845                 switch(p->sstatus & Smask){
 846                 case Smissing:
 847                         d->state = Dmissing;
 848                         break;
 849                 case Spresent:
 850                         if((p->sstatus & Imask) == Islumber)
 851                                 d->state = Dnew;
 852                         else
 853                                 d->state = Derror;
 854                         break;
 855                 case Sphylink:
 856                         /* power mgnt crap for suprise removal */
 857                         p->ie |= Aprcs|Apcs;    /* is this required? */
 858                         d->state = Dreset;
 859                         break;
 860                 case Sbist:
 861                         d->state = Doffline;
 862                         break;
 863                 }
 864                 dprint("%s: updatedrive: %s → %s [ss %.3lux]\n",
 865                         dnam(d), diskstates[s0], diskstates[d->state], p->sstatus);
 866                 if(s0 == Dready && d->state != Dready)
 867                         dprint("%s: pulled\n", dnam(d));
 868                 if(d->state != Dready)
 869                         f |= Ferror;
 870                 if(d->state != Dready || p->ci)
 871                         ewake = 1;
 872         }
 873         p->serror = serr;
 874         if(ewake)
 875                 clearci(p);
 876         if(f){
 877                 d->portm.flag = f;
 878                 wakeup(&d->portm);
 879         }
 880         last = cause;
 881 }
 882
 883 static void
 884 dstatus(Drive *d, int s)
 885 {
 886         dprint("%s: dstatus: %s → %s from pc=%p\n", dnam(d),
 887                 diskstates[d->state], diskstates[s], getcallerpc(&d));
 888
 889         ilock(d);
 890         d->state = s;
 891         iunlock(d);
 892 }
 893
 894 static void
 895 configdrive(Drive *d)
 896 {
 897         if(ahciconfigdrive(d->ctlr->hba, &d->portc, d->mode) == -1){
 898                 dstatus(d, Dportreset);
 899                 return;
 900         }
 901
 902         ilock(d);
 903         switch(d->port->sstatus & Smask){
 904         default:
 905         case Smissing:
 906                 d->state = Dmissing;
 907                 break;
 908         case Spresent:
 909                 if(d->state == Dnull)
 910                         d->state = Dportreset;
 911                 break;
 912         case Sphylink:
 913                 if(d->state == Dready)
 914                         break;
 915                 d->wait = 0;
 916                 d->state = Dnew;
 917                 break;
 918         case Sbist:
 919                 d->state = Doffline;
 920                 break;
 921         }
 922         iunlock(d);
 923
 924         dprint("%s: configdrive: %s\n", dnam(d), diskstates[d->state]);
 925 }
 926
 927 static void
 928 resetdisk(Drive *d)
 929 {
 930         uint state, det, stat;
 931         Aport *p;
 932
 933         p = d->port;
 934         det = p->sctl & 7;
 935         stat = p->sstatus & Smask;
 936         state = (p->cmd>>28) & 0xf;
 937         dprint("%s: resetdisk [icc %ux; det %.3ux; sdet %.3ux]\n", dnam(d), state, det, stat);
 938
 939         ilock(d);
 940         if(d->state != Dready && d->state != Dnew)
 941                 d->portm.flag |= Ferror;
 942         if(stat != Sphylink)
 943                 d->state = Dportreset;
 944         else
 945                 d->state = Dreset;
 946         clearci(p);                     /* satisfy sleep condition. */
 947         wakeup(&d->portm);
 948         iunlock(d);
 949
 950         if(stat != Sphylink)
 951                 return;
 952
 953         qlock(&d->portm);
 954         if(p->cmd&Ast && ahciswreset(&d->portc) == -1)
 955                 dstatus(d, Dportreset); /* get a bigger stick. */
 956         else
 957                 configdrive(d);
 958         qunlock(&d->portm);
 959 }
 960
 961 static int
 962 newdrive(Drive *d)
 963 {
 964         char *s;
 965         Aportc *c;
 966         Aportm *m;
 967
 968         c = &d->portc;
 969         m = &d->portm;
 970
 971         qlock(c->m);
 972         setfissig(m, c->p->sig);
 973         if(identify(d) == -1){
 974                 dprint("%s: identify failure\n", dnam(d));
 975                 goto lose;
 976         }
 977         if(settxmode(c, m->udma) == -1){
 978                 dprint("%s: can't set udma mode\n", dnam(d));
 979                 goto lose;
 980         }
 981         if(m->feat & Dpower && setfeatures(c, 0x85, 3*1000) == -1){
 982                 dprint("%s: can't disable apm\n", dnam(d));
 983                 m->feat &= ~Dpower;
 984                 if(ahcirecover(c) == -1)
 985                         goto lose;
 986         }
 987         dstatus(d, Dready);
 988         qunlock(c->m);
 989
 990         s = "";
 991         if(m->feat & Dllba)
 992                 s = "L";
 993         idprint("%s: %sLBA %,lld sectors\n", dnam(d), s, d->sectors);
 994         idprint("  %s %s %s %s\n", d->model, d->firmware, d->serial,
 995                 d->drivechange? "[newdrive]": "");
 996         return 0;
 997
 998 lose:
 999         idprint("%s: can't be initialized\n", dnam(d));
1000         dstatus(d, Dnull);
1001         qunlock(c->m);
1002         return -1;
1003 }
1004
1005 enum {
1006         Nms             = 256,
1007         Mcomrwait       =  1*1024/Nms - 1,
1008         Mphywait        =  2*1024/Nms - 1,
1009         Midwait         = 16*1024/Nms - 1,
1010 };
1011
1012 static void
1013 hangck(Drive *d)
1014 {
1015         if(d->active && d->totick != 0 && (long)(Ticks - d->totick) > 0){
1016                 dprint("%s: drive hung [task %lux; ci %lux; serr %lux]%s\n",
1017                         dnam(d), d->port->task, d->port->ci, d->port->serror,
1018                         d->nodma == 0 ? "; disabling dma" : "");
1019                 d->nodma = 1;
1020                 d->state = Dreset;
1021         }
1022 }
1023
1024 static ushort olds[NCtlr*NCtlrdrv];
1025
1026 static void
1027 doportreset(Drive *d)
1028 {
1029         qlock(&d->portm);
1030         ahciportreset(&d->portc, d->mode);
1031         qunlock(&d->portm);
1032
1033         dprint("ahci: portreset: %s [task %lux; ss %.3lux]\n",
1034                 diskstates[d->state], d->port->task, d->port->sstatus);
1035 }
1036
1037 /* drive must be locked */
1038 static void
1039 statechange(Drive *d)
1040 {
1041         switch(d->state){
1042         case Dnull:
1043         case Doffline:
1044                 if(d->unit)
1045                 if(d->unit->sectors != 0){
1046                         d->sectors = 0;
1047                         d->drivechange = 1;
1048                 }
1049         case Dready:
1050                 d->wait = 0;
1051         }
1052 }
1053
1054 static uint
1055 maxmode(Ctlr *c)
1056 {
1057         return (c->hba->cap & 0xf*Hiss)/Hiss;
1058 }
1059
1060 static void iainterrupt(Ureg*, void *);
1061
1062 static void
1063 checkdrive(Drive *d, int i)
1064 {
1065         ushort s, sig;
1066
1067         if(d->ctlr->enabled == 0)
1068                 return;
1069         if(d->driveno == 0)
1070                 iainterrupt(0, d->ctlr);        /* check for missed irq's */
1071
1072         ilock(d);
1073         s = d->port->sstatus;
1074         if(s)
1075                 d->lastseen = Ticks;
1076         if(s != olds[i]){
1077                 dprint("%s: status: %.3ux -> %.3ux: %s\n",
1078                         dnam(d), olds[i], s, diskstates[d->state]);
1079                 olds[i] = s;
1080                 d->wait = 0;
1081         }
1082         hangck(d);
1083         switch(d->state){
1084         case Dnull:
1085         case Dready:
1086                 break;
1087         case Dmissing:
1088         case Dnew:
1089                 switch(s & (Iactive|Smask)){
1090                 case Spresent:
1091                         ahciwakeup(&d->portc, d->mode);
1092                 case Smissing:
1093                         break;
1094                 default:
1095                         dprint("%s: unknown status %.3ux\n", dnam(d), s);
1096                         /* fall through */
1097                 case Iactive:           /* active, no device */
1098                         if(++d->wait&Mphywait)
1099                                 break;
1100 reset:
1101                         if(d->mode == 0)
1102                                 d->mode = maxmode(d->ctlr);
1103                         else
1104                                 d->mode--;
1105                         if(d->mode == DMautoneg){
1106                                 d->wait = 0;
1107                                 d->state = Dportreset;
1108                                 goto portreset;
1109                         }
1110                         dprint("%s: reset; new mode %s\n", dnam(d),
1111                                 modes[d->mode]);
1112                         iunlock(d);
1113                         resetdisk(d);
1114                         ilock(d);
1115                         break;
1116                 case Iactive | Sphylink:
1117                         if(d->unit == nil)
1118                                 break;
1119                         if((++d->wait&Midwait) == 0){
1120                                 dprint("%s: slow reset [task %lux; ss %.3ux; wait %d]\n",
1121                                         dnam(d), d->port->task, s, d->wait);
1122                                 goto reset;
1123                         }
1124                         s = (uchar)d->port->task;
1125                         sig = d->port->sig >> 16;
1126                         if(s == 0x7f || s&ASbsy ||
1127                             (sig != 0xeb14 && (s & ASdrdy) == 0))
1128                                 break;
1129                         iunlock(d);
1130                         newdrive(d);
1131                         ilock(d);
1132                         break;
1133                 }
1134                 break;
1135         case Doffline:
1136                 if(d->wait++ & Mcomrwait)
1137                         break;
1138         case Derror:
1139         case Dreset:
1140                 dprint("%s: reset [%s]: mode %d; status %.3ux\n",
1141                         dnam(d), diskstates[d->state], d->mode, s);
1142                 iunlock(d);
1143                 resetdisk(d);
1144                 ilock(d);
1145                 break;
1146         case Dportreset:
1147 portreset:
1148                 if(d->wait++ & Mcomrwait)
1149                         break;
1150                 if(d->wait > Mcomrwait && (s & Iactive) == 0){
1151                         d->state = Dnull;       /* stuck in portreset */
1152                         break;
1153                 }
1154                 dprint("%s: portreset [%s]: mode %d; status %.3ux\n",
1155                         dnam(d), diskstates[d->state], d->mode, s);
1156                 d->portm.flag |= Ferror;
1157                 clearci(d->port);
1158                 wakeup(&d->portm);
1159                 if((s & Smask) == Smissing){
1160                         d->state = Dmissing;
1161                         break;
1162                 }
1163                 iunlock(d);
1164                 doportreset(d);
1165                 ilock(d);
1166                 break;
1167         }
1168         statechange(d);
1169         iunlock(d);
1170 }
1171
1172 static void
1173 satakproc(void*)
1174 {
1175         int i;
1176
1177         while(waserror())
1178                 ;
1179         for(;;){
1180                 tsleep(&up->sleep, return0, 0, Nms);
1181                 for(i = 0; i < niadrive; i++)
1182                         checkdrive(iadrive[i], i);
1183         }
1184 }
1185
1186 static void
1187 iainterrupt(Ureg *u, void *a)
1188 {
1189         int i;
1190         ulong cause, m;
1191         Ctlr *c;
1192         Drive *d;
1193
1194         c = a;
1195         ilock(c);
1196         cause = c->hba->isr;
1197         for(i = 0; cause; i++){
1198                 m = 1 << i;
1199                 if((cause & m) == 0)
1200                         continue;
1201                 cause &= ~m;
1202                 d = c->rawdrive + i;
1203                 ilock(d);
1204                 if(d->port != nil && d->port->isr && c->hba->pi & m)
1205                         updatedrive(d);
1206                 c->hba->isr = m;
1207                 iunlock(d);
1208         }
1209         if(u == 0 && i > 0)
1210                 c->missirq++;
1211         iunlock(c);
1212 }
1213
1214 static int
1215 ahciencreset(Ctlr *c)
1216 {
1217         Ahba *h;
1218
1219         if(c->enctype == Eesb)
1220                 return 0;
1221         h = c->hba;
1222         h->emctl |= Emrst;
1223         while(h->emctl & Emrst)
1224                 delay(1);
1225         return 0;
1226 }
1227
1228 /*
1229  * from the standard: (http://en.wikipedia.org/wiki/IBPI)
1230  * rebuild is preferred as locate+fail; alternate 1hz fail
1231  * we're going to assume no locate led.
1232  */
1233
1234 enum {
1235         Ledsleep        = 125,          /* 8hz */
1236
1237         N0      = Ledon*Aled,
1238         L0      = Ledon*Aled | Ledon*Locled,
1239         L1      = Ledon*Aled | Ledoff*Locled,
1240         R0      = Ledon*Aled | Ledon*Locled |   Ledon*Errled,
1241         R1      = Ledon*Aled |                  Ledoff*Errled,
1242         S0      = Ledon*Aled |  Ledon*Locled /*|        Ledon*Errled*/, /* botch */
1243         S1      = Ledon*Aled |                  Ledoff*Errled,
1244         P0      = Ledon*Aled |                  Ledon*Errled,
1245         P1      = Ledon*Aled |                  Ledoff*Errled,
1246         F0      = Ledon*Aled |                  Ledon*Errled,
1247         C0      = Ledon*Aled | Ledon*Locled,
1248         C1      = Ledon*Aled | Ledoff*Locled,
1249
1250 };
1251
1252 //static ushort led3[Ibpilast*8] = {
1253 //[Ibpinone*8]  0,      0,      0,      0,      0,      0,      0,      0,
1254 //[Ibpinormal*8]        N0,     N0,     N0,     N0,     N0,     N0,     N0,     N0,
1255 //[Ibpirebuild*8]       R0,     R0,     R0,     R0,     R1,     R1,     R1,     R1,
1256 //[Ibpilocate*8]        L0,     L1,     L0,     L1,     L0,     L1,     L0,     L1,
1257 //[Ibpispare*8] S0,     S1,     S0,     S1,     S1,     S1,     S1,     S1,
1258 //[Ibpipfa*8]   P0,     P1,     P0,     P1,     P1,     P1,     P1,     P1,     /* first 1 sec */
1259 //[Ibpifail*8]  F0,     F0,     F0,     F0,     F0,     F0,     F0,     F0,
1260 //[Ibpicritarray*8]     C0,     C0,     C0,     C0,     C1,     C1,     C1,     C1,
1261 //[Ibpifailarray*8]     C0,     C1,     C0,     C1,     C0,     C1,     C0,     C1,
1262 //};
1263
1264 static ushort led2[Ibpilast*8] = {
1265 [Ibpinone*8]    0,      0,      0,      0,      0,      0,      0,      0,
1266 [Ibpinormal*8]  N0,     N0,     N0,     N0,     N0,     N0,     N0,     N0,
1267 [Ibpirebuild*8] R0,     R0,     R0,     R0,     R1,     R1,     R1,     R1,
1268 [Ibpilocate*8]  L0,     L0,     L0,     L0,     L0,     L0,     L0,     L0,
1269 [Ibpispare*8]   S0,     S0,     S0,     S0,     S1,     S1,     S1,     S1,
1270 [Ibpipfa*8]     P0,     P1,     P0,     P1,     P1,     P1,     P1,     P1,     /* first 1 sec */
1271 [Ibpifail*8]    F0,     F0,     F0,     F0,     F0,     F0,     F0,     F0,
1272 [Ibpicritarray*8]       C0,     C0,     C0,     C0,     C1,     C1,     C1,     C1,
1273 [Ibpifailarray*8]       C0,     C1,     C0,     C1,     C0,     C1,     C0,     C1,
1274 };
1275
1276 static int
1277 ledstate(Ledport *p, uint seq)
1278 {
1279         ushort i;
1280
1281         if(p->led == Ibpipfa && seq%32 >= 8)
1282                 i = P1;
1283         else
1284                 i = led2[8*p->led + seq%8];
1285         if(i != p->ledbits){
1286                 p->ledbits = i;
1287                 ledprint("ledstate %,.011ub %ud\n", p->ledbits, seq);
1288                 return 1;
1289         }
1290         return 0;
1291 }
1292
1293 static int
1294 blink(Drive *d, ulong t)
1295 {
1296         Ahba *h;
1297         Ctlr *c;
1298         Aledmsg msg;
1299
1300         if(ledstate(d, t) == 0)
1301                 return 0;
1302         c = d->ctlr;
1303         h = c->hba;
1304         /* ensure last message has been transmitted */
1305         while(h->emctl & Tmsg)
1306                 microdelay(1);
1307         switch(c->enctype){
1308         default:
1309                 panic("%s: bad led type %d", dnam(d), c->enctype);
1310         case Elmt:
1311                 memset(&msg, 0, sizeof msg);
1312                 msg.type = Mled;
1313                 msg.dsize = 0;
1314                 msg.msize = sizeof msg - 4;
1315                 msg.led[0] = d->ledbits;
1316                 msg.led[1] = d->ledbits>>8;
1317                 msg.pm = 0;
1318                 msg.hba = d->driveno;
1319                 memmove(c->enctx, &msg, sizeof msg);
1320                 break;
1321         }
1322         h->emctl |= Tmsg;
1323         return 1;
1324 }
1325
1326 enum {
1327         Esbdrv0 = 4,            /* start pos in bits */
1328         Esbiota = 3,            /* shift in bits */
1329         Esbact  = 1,
1330         Esbloc  = 2,
1331         Esberr  = 4,
1332 };
1333
1334 uint
1335 esbbits(uint s)
1336 {
1337         uint i, e;                              /* except after c */
1338
1339         e = 0;
1340         for(i = 0; i < 3; i++)
1341                 e |= ((s>>3*i & 7) != 0)<<i;
1342         return e;
1343 }
1344
1345 static int
1346 blinkesb(Ctlr *c, ulong t)
1347 {
1348         uint i, s, u[32/4];
1349         uvlong v;
1350         Drive *d;
1351
1352         s = 0;
1353         for(i = 0; i < c->ndrive; i++){
1354                 d = c->drive[i];
1355                 s |= ledstate(d, t);            /* no port mapping */
1356         }
1357         if(s == 0)
1358                 return 0;
1359         memset(u, 0, sizeof u);
1360         for(i = 0; i < c->ndrive; i++){
1361                 d = c->drive[i];
1362                 s = Esbdrv0 + Esbiota*i;
1363                 v = esbbits(d->ledbits) * (1ull << s%32);
1364                 u[s/32 + 0] |= v;
1365                 u[s/32 + 1] |= v>>32;
1366         }
1367         for(i = 0; i < c->encsz; i++)
1368                 c->enctx[i] = u[i];
1369         return 1;
1370 }
1371
1372 static long
1373 ahciledr(SDunit *u, Chan *ch, void *a, long n, vlong off)
1374 {
1375         Ctlr *c;
1376         Drive *d;
1377
1378         c = u->dev->ctlr;
1379         d = c->drive[u->subno];
1380         return ledr(d, ch, a, n, off);
1381 }
1382
1383 static long
1384 ahciledw(SDunit *u, Chan *ch, void *a, long n, vlong off)
1385 {
1386         Ctlr *c;
1387         Drive *d;
1388
1389         c = u->dev->ctlr;
1390         d = c->drive[u->subno];
1391         return ledw(d, ch, a, n, off);
1392 }
1393
1394 static void
1395 ledkproc(void*)
1396 {
1397         uchar map[NCtlr];
1398         uint i, j, t0, t1;
1399         Ctlr *c;
1400         Drive *d;
1401
1402         j = 0;
1403         memset(map, 0, sizeof map);
1404         for(i = 0; i < niactlr; i++)
1405                 if(iactlr[i].enctype != 0){
1406                         ahciencreset(iactlr + i);
1407                         map[i] = 1;
1408                         j++;
1409                 }
1410         if(j == 0)
1411                 pexit("no work", 1);
1412         for(i = 0; i < niadrive; i++){
1413                 iadrive[i]->nled = 3;           /* hardcoded */
1414                 if(iadrive[i]->ctlr->enctype == Eesb)
1415                         iadrive[i]->nled = 3;
1416                 iadrive[i]->ledbits = -1;
1417         }
1418         for(i = 0; ; i++){
1419                 t0 = Ticks;
1420                 for(j = 0; j < niadrive; ){
1421                         c = iadrive[j]->ctlr;
1422                         if(map[j] == 0)
1423                                 j += c->enctype;
1424                         else if(c->enctype == Eesb){
1425                                 blinkesb(c, i);
1426                                 j += c->ndrive;
1427                         }else{
1428                                 d = iadrive[j++];
1429                                 blink(d, i);
1430                         }
1431                 }
1432                 t1 = Ticks;
1433                 esleep(Ledsleep - TK2MS(t1 - t0));
1434         }
1435 }
1436
1437 static int
1438 waitready(Drive *d)
1439 {
1440         ulong s, i, δ;
1441
1442         for(i = 0;; i += 250){
1443                 if(d->state == Dreset || d->state == Dportreset || d->state == Dnew)
1444                         return 1;
1445                 ilock(d);
1446                 s = d->port->sstatus;
1447                 if(d->state == Dready && (s & Smask) == Sphylink){
1448                         iunlock(d);
1449                         return 0;
1450                 }
1451                 δ = Ticks - d->lastseen;
1452                 if(d->state == Dnull || δ > 10*1000)
1453                         break;
1454                 if((s & Imask) == 0 && δ > 1500)
1455                         break;
1456                 if(i >= 15*1000){
1457                         d->state = Doffline;
1458                         iunlock(d);
1459                         print("%s: not responding; offline\n", dnam(d));
1460                         return -1;
1461                 }
1462                 iunlock(d);
1463                 esleep(250);
1464         }
1465         iunlock(d);
1466         return -1;
1467 }
1468
1469 static int
1470 iaverify(SDunit *u)
1471 {
1472         Ctlr *c;
1473         Drive *d;
1474
1475         c = u->dev->ctlr;
1476         d = c->drive[u->subno];
1477         ilock(c);
1478         ilock(d);
1479         if(d->unit == nil){
1480                 d->unit = u;
1481                 if(c->enctype != 0)
1482                         sdaddfile(u, "led", 0644, eve, ahciledr, ahciledw);
1483         }
1484         iunlock(d);
1485         iunlock(c);
1486         checkdrive(d, d->driveno);              /* c->d0 + d->driveno */
1487         return 1;
1488 }
1489
1490 static int
1491 iaonline(SDunit *u)
1492 {
1493         int r;
1494         Ctlr *c;
1495         Drive *d;
1496
1497         c = u->dev->ctlr;
1498         d = c->drive[u->subno];
1499
1500         while(d->state != Dmissing && waitready(d) == 1)
1501                 esleep(1);
1502
1503         dprint("%s: iaonline: %s\n", dnam(d), diskstates[d->state]);
1504
1505         ilock(d);
1506         if(d->portm.feat & Datapi){
1507                 r = d->drivechange;
1508                 d->drivechange = 0;
1509                 iunlock(d);
1510                 if(r != 0)
1511                         scsiverify(u);
1512                 return scsionline(u);
1513         }
1514         r = 0;
1515         if(d->drivechange){
1516                 d->drivechange = 0;
1517                 r = 2;
1518         }else if(d->state == Dready)
1519                 r = 1;
1520         if(r){
1521                 u->sectors = d->sectors;
1522                 u->secsize = d->secsize;
1523         }
1524         iunlock(d);
1525
1526         return r;
1527 }
1528
1529 static int
1530 iaenable(SDev *s)
1531 {
1532         char name[32];
1533         Ctlr *c;
1534         static int once;
1535
1536         c = s->ctlr;
1537         ilock(c);
1538         if(!c->enabled){
1539                 if(once == 0)
1540                         kproc("iasata", satakproc, 0);
1541                 if(c->ndrive == 0)
1542                         panic("iaenable: zero s->ctlr->ndrive");
1543                 pcisetbme(c->pci);
1544                 snprint(name, sizeof name, "%s (%s)", s->name, s->ifc->name);
1545                 intrenable(c->pci->intl, iainterrupt, c, c->pci->tbdf, name);
1546                 /* supposed to squelch leftover interrupts here. */
1547                 ahcienable(c->hba);
1548                 c->enabled = 1;
1549                 if(++once == niactlr)
1550                         kproc("ialed", ledkproc, 0);
1551         }
1552         iunlock(c);
1553         return 1;
1554 }
1555
1556 static int
1557 iadisable(SDev *s)
1558 {
1559         char name[32];
1560         Ctlr *c;
1561
1562         c = s->ctlr;
1563         ilock(c);
1564         ahcidisable(c->hba);
1565         snprint(name, sizeof name, "%s (%s)", s->name, s->ifc->name);
1566         intrdisable(c->pci->intl, iainterrupt, c, c->pci->tbdf, name);
1567         c->enabled = 0;
1568         iunlock(c);
1569         return 1;
1570 }
1571
1572 static Alist*
1573 ahcibuild(Drive *d, int rw, void *data, int nsect, vlong lba)
1574 {
1575         uchar *c;
1576         uint flags;
1577         Aportm *m;
1578
1579         m = &d->portm;
1580         c = m->ctab->cfis;
1581         rwfis(m, c, rw, nsect, lba);
1582         flags = Lpref;
1583         if(rw == SDwrite)
1584                 flags |= Lwrite;
1585         return mkalist(m, flags, data, nsect * d->secsize);
1586 }
1587
1588 static Alist*
1589 ahcibuildpkt(Drive *d, SDreq *r, void *data, int n)
1590 {
1591         uint flags;
1592         Aportm *m;
1593         uchar *c;
1594         Actab *t;
1595
1596         m = &d->portm;
1597         t = m->ctab;
1598         c = t->cfis;
1599
1600         atapirwfis(m, c, r->cmd, r->clen, 0x2000);
1601         if((n & 15) != 0 || d->nodma)
1602                 c[Ffeat] &= ~1; /* use pio */
1603         else if(c[Ffeat] & 1 && d->info[62] & (1<<15))  /* dma direction */
1604                 c[Ffeat] = (c[Ffeat] & ~(1<<2)) | ((r->write == 0) << 2);
1605         flags = Lpref | Latapi;
1606         if(r->write != 0 && data)
1607                 flags |= Lwrite;
1608         return mkalist(m, flags, data, n);
1609 }
1610
1611 static Alist*
1612 ahcibuildfis(Drive *d, SDreq *r, void *data, uint n)
1613 {
1614         uint flags;
1615         uchar *c;
1616         Aportm *m;
1617
1618         if((r->ataproto & Pprotom) == Ppkt)
1619                 return ahcibuildpkt(d, r, data, n);
1620
1621         m = &d->portm;
1622         c = m->ctab->cfis;
1623         memmove(c, r->cmd, r->clen);
1624         flags = Lpref;
1625         if(r->write || n == 0)
1626                 flags |= Lwrite;
1627         return mkalist(m, flags, data, n);
1628 }
1629
1630 static int
1631 lockready(Drive *d)
1632 {
1633         int i;
1634
1635         qlock(&d->portm);
1636         while ((i = waitready(d)) == 1) {
1637                 qunlock(&d->portm);
1638                 esleep(1);
1639                 qlock(&d->portm);
1640         }
1641         return i;
1642 }
1643
1644 static int
1645 flushcache(Drive *d)
1646 {
1647         int i;
1648
1649         i = -1;
1650         if(lockready(d) == 0)
1651                 i = ahciflushcache(&d->portc);
1652         qunlock(&d->portm);
1653         return i;
1654 }
1655
1656 static int
1657 io(Drive *d, uint proto, int to, int interrupt)
1658 {
1659         uint task, flag, stat, rv;
1660         Aport *p;
1661         Asleep as;
1662
1663         switch(waitready(d)){
1664         case -1:
1665                 return SDeio;
1666         case 1:
1667                 return SDretry;
1668         }
1669
1670         ilock(d);
1671         d->portm.flag = 0;
1672         iunlock(d);
1673         p = d->port;
1674         p->ci = 1;
1675
1676         as.p = p;
1677         as.i = 1;
1678         d->totick = 0;
1679         if(to > 0)
1680                 d->totick = Ticks + MS2TK(to) | 1;      /* fix fencepost */
1681         d->active++;
1682
1683         while(waserror())
1684                 if(interrupt){
1685                         d->active--;
1686                         d->port->ci = 0;
1687                         if(ahcicomreset(&d->portc) == -1)
1688                                 dstatus(d, Dreset);
1689                         return SDtimeout;
1690                 }
1691
1692         sleep(&d->portm, ahciclear, &as);
1693         poperror();
1694
1695         d->active--;
1696         ilock(d);
1697         stat = d->state;
1698         flag = d->portm.flag;
1699         task = d->port->task;
1700         iunlock(d);
1701
1702         rv = SDok;
1703         if(proto & Ppkt && stat == Dready){
1704                 rv = task >> 8 + 4 & 0xf;
1705                 flag &= ~Fahdrs;
1706                 flag |= Fdone;
1707         }else if(task & (Efatal<<8) || task & (ASbsy|ASdrq) && stat == Dready){
1708                 d->port->ci = 0;
1709                 ahcirecover(&d->portc);
1710                 task = d->port->task;
1711                 flag &= ~Fdone;         /* either an error or do-over */
1712         }
1713         if(flag == 0){
1714                 print("%s: retry\n", dnam(d));
1715                 return SDretry;
1716         }
1717         if(flag & (Fahdrs | Ferror)){
1718                 if((task & Eidnf) == 0)
1719                         print("%s: i/o error %ux\n", dnam(d), task);
1720                 return SDcheck;
1721         }
1722         return rv;
1723 }
1724
1725 static int
1726 iariopkt(SDreq *r, Drive *d)
1727 {
1728         int try, to;
1729         uchar *cmd;
1730         Alist *l;
1731
1732         cmd = r->cmd;
1733         aprint("%s: %.2ux %.2ux %c %d %p\n", dnam(d), cmd[0], cmd[2],
1734                 "rw"[r->write], r->dlen, r->data);
1735
1736         r->rlen = 0;
1737
1738         /*
1739          * prevent iaonline() to hang forever by timing out
1740          * inquiry and capacity commands after 5 seconds.
1741          */
1742         to = 30*1000;
1743         switch(cmd[0]){
1744         case 0x9e: if(cmd[1] != 0x10) break;
1745         case 0x25:
1746         case 0x12:
1747                 to = 5*1000;
1748                 break;
1749         }
1750
1751         for(try = 0; try < 10; try++){
1752                 qlock(&d->portm);
1753                 l = ahcibuildpkt(d, r, r->data, r->dlen);
1754                 r->status = io(d, Ppkt, to, 0);
1755                 switch(r->status){
1756                 case SDeio:
1757                         qunlock(&d->portm);
1758                         return SDeio;
1759                 case SDretry:
1760                         qunlock(&d->portm);
1761                         continue;
1762                 }
1763                 r->rlen = l->len;
1764                 qunlock(&d->portm);
1765                 return SDok;
1766         }
1767         print("%s: bad disk\n", dnam(d));
1768         return r->status = SDcheck;
1769 }
1770
1771 static long
1772 ahcibio(SDunit *u, int lun, int write, void *a, long count, uvlong lba)
1773 {
1774         int n, rw, try, status, max;
1775         uchar *data;
1776         Ctlr *c;
1777         Drive *d;
1778
1779         c = u->dev->ctlr;
1780         d = c->drive[u->subno];
1781         if(d->portm.feat & Datapi)
1782                 return scsibio(u, lun, write, a, count, lba);
1783
1784         max = 128;
1785         if(d->portm.feat & Dllba){
1786                 max = 8192;             /* ahci maximum */
1787                 if(c->type == Tsb600)
1788                         max = 255;      /* errata */
1789         }
1790         rw = write? SDwrite: SDread;
1791         data = a;
1792         dprint("%s: bio: %llud %c %lud %p\n",
1793                 dnam(d), lba, "rw"[rw], count, data);
1794
1795         for(try = 0; try < 10;){
1796                 n = count;
1797                 if(n > max)
1798                         n = max;
1799                 qlock(&d->portm);
1800                 ahcibuild(d, rw, data, n, lba);
1801                 status = io(d, Pdma, 5000, 0);
1802                 qunlock(&d->portm);
1803                 switch(status){
1804                 case SDeio:
1805                         return -1;
1806                 case SDretry:
1807                         try++;
1808                         continue;
1809                 }
1810                 try = 0;
1811                 count -= n;
1812                 lba += n;
1813                 data += n * u->secsize;
1814                 if(count == 0)
1815                         return data - (uchar*)a;
1816         }
1817         print("%s: bad disk\n", dnam(d));
1818         return -1;
1819 }
1820
1821 static int
1822 iario(SDreq *r)
1823 {
1824         int i, n, count, rw;
1825         uchar *cmd;
1826         uvlong lba;
1827         Ctlr *c;
1828         Drive *d;
1829         SDunit *u;
1830
1831         u = r->unit;
1832         c = u->dev->ctlr;
1833         d = c->drive[u->subno];
1834         if(d->portm.feat & Datapi)
1835                 return iariopkt(r, d);
1836         cmd = r->cmd;
1837
1838         if(cmd[0] == 0x35 || cmd[0] == 0x91){
1839                 if(flushcache(d) == 0)
1840                         return sdsetsense(r, SDok, 0, 0, 0);
1841                 return sdsetsense(r, SDcheck, 3, 0xc, 2);
1842         }
1843
1844         if((i = sdfakescsi(r)) != SDnostatus){
1845                 r->status = i;
1846                 return i;
1847         }
1848
1849         if((i = sdfakescsirw(r, &lba, &count, &rw)) != SDnostatus)
1850                 return i;
1851         n = ahcibio(u, r->lun, r->write, r->data, count, lba);
1852         if(n == -1)
1853                 return SDeio;
1854         r->rlen = n;
1855         return SDok;
1856 }
1857
1858 static uchar bogusrfis[16] = {
1859 [Ftype]         0x34,
1860 [Fioport]       0x40,
1861 [Fstatus]       0x50,
1862 [Fdev]          0xa0,
1863 };
1864
1865 static void
1866 sdr0(Drive *d)
1867 {
1868         uchar *c;
1869
1870         c = d->portm.fis.r;
1871         memmove(c, bogusrfis, sizeof bogusrfis);
1872         coherence();
1873 }
1874
1875 static int
1876 sdr(SDreq *r, Drive *d, int st)
1877 {
1878         uchar *c;
1879         uint t;
1880
1881         if((r->ataproto & Pprotom) == Ppkt){
1882                 t = d->port->task;
1883                 if(t & ASerr)
1884                         st = t >> 8 + 4 & 0xf;
1885         }
1886         c = d->portm.fis.r;
1887         memmove(r->cmd, c, 16);
1888         r->status = st;
1889         if(st == SDcheck)
1890                 st = SDok;
1891         return st;
1892 }
1893
1894 static int
1895 fisreqchk(Sfis *f, SDreq *r)
1896 {
1897         if((r->ataproto & Pprotom) == Ppkt)
1898                 return SDnostatus;
1899         /*
1900          * handle oob requests;
1901          *    restrict & sanitize commands
1902          */
1903         if(r->clen != 16)
1904                 error(Eio);
1905         if(r->cmd[0] == 0xf0){
1906                 sigtofis(f, r->cmd);
1907                 r->status = SDok;
1908                 return SDok;
1909         }
1910         r->cmd[0] = 0x27;
1911         r->cmd[1] = 0x80;
1912         r->cmd[7] |= 0xa0;
1913         return SDnostatus;
1914 }
1915
1916 static int
1917 iaataio(SDreq *r)
1918 {
1919         int try;
1920         Ctlr *c;
1921         Drive *d;
1922         SDunit *u;
1923         Alist *l;
1924
1925         u = r->unit;
1926         c = u->dev->ctlr;
1927         d = c->drive[u->subno];
1928
1929         if((r->status = fisreqchk(&d->portm, r)) != SDnostatus)
1930                 return r->status;
1931         r->rlen = 0;
1932         sdr0(d);
1933         for(try = 0; try < 10; try++){
1934                 qlock(&d->portm);
1935                 l = ahcibuildfis(d, r, r->data, r->dlen);
1936                 r->status = io(d, r->ataproto & Pprotom, -1, 1);
1937                 switch(r->status){
1938                 case SDtimeout:
1939                         qunlock(&d->portm);
1940                         return sdsetsense(r, SDcheck, 11, 0, 6);
1941                 case SDeio:
1942                         qunlock(&d->portm);
1943                         return SDeio;
1944                 case SDretry:
1945                         qunlock(&d->portm);
1946                         continue;
1947                 }
1948                 r->rlen = (r->ataproto & Pprotom) == Ppkt ? l->len : r->dlen;
1949                 try = sdr(r, d, r->status);
1950                 qunlock(&d->portm);
1951                 return try;
1952         }
1953         print("%s: bad disk\n", dnam(d));
1954         return r->status = SDeio;
1955 }
1956
1957 enum{
1958         Ghc     = 0x04/4,       /* global host control */
1959         Pi      = 0x0c/4,       /* ports implemented */
1960         Cmddec  = 1<<15,        /* enable command block decode */
1961
1962         /* Ghc bits */
1963         Ahcien  = 1<<31,        /* ahci enable */
1964 };
1965
1966 static void
1967 iasetupahci(Ctlr *c)
1968 {
1969         if(c->type != Tich)
1970                 return;
1971
1972         pcicfgw16(c->pci, 0x40, pcicfgr16(c->pci, 0x40) & ~Cmddec);
1973         pcicfgw16(c->pci, 0x42, pcicfgr16(c->pci, 0x42) & ~Cmddec);
1974
1975         c->lmmio[Ghc] |= Ahcien;
1976         c->lmmio[Pi] = (1 << 6) - 1;    /* 6 ports (supposedly ro pi reg) */
1977
1978         /* enable ahci mode; from ich9 datasheet */
1979         pcicfgw16(c->pci, 0x90, 1<<6 | 1<<5);
1980
1981         /* configure drives 0-5 as ahci sata  (c.f. errata) */
1982         pcicfgw16(c->pci, 0x92, pcicfgr16(c->pci, 0x92) | 0xf);
1983 }
1984
1985 static void
1986 sbsetupahci(Pcidev *p)
1987 {
1988         print("sbsetupahci: tweaking %.4ux ccru %.2ux ccrp %.2ux\n",
1989                 p->did, p->ccru, p->ccrp);
1990         pcicfgw8(p, 0x40, pcicfgr8(p, 0x40) | 1);
1991         pcicfgw8(p, PciCCRu, 6);
1992         pcicfgw8(p, PciCCRp, 1);
1993         p->ccru = 6;
1994         p->ccrp = 1;
1995 }
1996
1997 static int
1998 esbenc(Ctlr *c)
1999 {
2000         c->encsz = 1;
2001         c->enctx = (ulong*)(c->mmio + 0xa0);
2002         c->enctype = Eesb;
2003         c->enctx[0] = 0;
2004         return 0;
2005 }
2006
2007 static int
2008 ahciencinit(Ctlr *c)
2009 {
2010         ulong type, sz, o, *bar;
2011         Ahba *h;
2012
2013         h = c->hba;
2014         if(c->type == Tesb)
2015                 return esbenc(c);
2016         if((h->cap & Hems) == 0)
2017                 return -1;
2018         type = h->emctl & Emtype;
2019         switch(type){
2020         case Esgpio:
2021         case Eses2:
2022         case Esafte:
2023                 return -1;
2024         case Elmt:
2025                 break;
2026         default:
2027                 return -1;
2028         }
2029
2030         sz = h->emloc & 0xffff;
2031         o = h->emloc>>16;
2032         if(sz == 0 || o == 0)
2033                 return -1;
2034         bar = c->lmmio;
2035         dprint("size = %.4lux; loc = %.4lux*4\n", sz, o);
2036         c->encsz = sz;
2037         c->enctx = bar + o;
2038         if((h->emctl & Xonly) == 0){
2039                 if(h->emctl & Smb)
2040                         c->encrx = bar + o;
2041                 else
2042                         c->encrx = bar + o*2;
2043         }
2044         c->enctype = type;
2045         return 0;
2046 }
2047
2048 static int
2049 didtype(Pcidev *p)
2050 {
2051         int type;
2052
2053         type = Tahci;
2054         switch(p->vid){
2055         default:
2056                 return -1;
2057         case 0x8086:
2058                 if((p->did & 0xffff) == 0x1e02)
2059                         return Tich;            /* c210 */
2060                 if((p->did & 0xffff) == 0x8c02)
2061                         return Tich;            /* c220 */
2062                 if((p->did & 0xffff) == 0x24d1)
2063                         return Tich;            /* 82801eb/er */
2064                 if((p->did & 0xffff) == 0x2653)
2065                         return Tich;            /* 82801fbm */
2066                 if((p->did & 0xfffc) == 0x2680)
2067                         return Tesb;
2068                 if((p->did & 0xfffb) == 0x27c1)
2069                         return Tich;            /* 82801g[bh]m */
2070                 if((p->did & 0xffff) == 0x2822)
2071                         return Tich;            /* 82801 SATA RAID */
2072                 if((p->did & 0xffff) == 0x2821)
2073                         return Tich;            /* 82801h[roh] */
2074                 if((p->did & 0xfffe) == 0x2824)
2075                         return Tich;            /* 82801h[b] */
2076                 if((p->did & 0xfeff) == 0x2829)
2077                         return Tich;            /* ich8 */
2078                 if((p->did & 0xfffe) == 0x2922)
2079                         return Tich;            /* ich9 */
2080                 if((p->did & 0xffff) == 0x3a02)
2081                         return Tich;            /* 82801jd/do */
2082                 if((p->did & 0xfefe) == 0x3a22)
2083                         return Tich;            /* ich10, pch */
2084                 if((p->did & 0xfff7) == 0x3b28)
2085                         return Tich;            /* pchm */
2086                 if((p->did & 0xfffe) == 0x3b22)
2087                         return Tich;            /* pch */
2088                 break;
2089         case 0x1002:
2090                 if(p->ccru == 1 || p->ccrp != 1)
2091                 if(p->did == 0x4380 || p->did == 0x4390)
2092                         sbsetupahci(p);
2093                 type = Tsb600;
2094                 break;
2095         case 0x1106:
2096                 /*
2097                  * unconfirmed report that the programming
2098                  * interface is set incorrectly.
2099                  */
2100                 if(p->did == 0x3349)
2101                         return Tahci;
2102                 break;
2103         case 0x1022:
2104                 /* Hudson SATA Controller [AHCI mode] */
2105                 if(p->did == 0x7801)
2106                         return Tahci;
2107                 break;
2108         case 0x10de:
2109         case 0x1039:
2110         case 0x1b4b:
2111         case 0x11ab:
2112                 break;
2113         case 0x197b:
2114         case 0x10b9:
2115                 type = Tjmicron;
2116                 break;
2117         }
2118         if(p->ccrb == Pcibcstore && p->ccru == 6 && p->ccrp == 1)
2119                 return type;
2120         return -1;
2121 }
2122
2123 static SDev*
2124 iapnp(void)
2125 {
2126         int i, n, nunit, type;
2127         uintptr io;
2128         Ctlr *c;
2129         Drive *d;
2130         Pcidev *p;
2131         SDev *s;
2132         static int done;
2133
2134         if(done)
2135                 return nil;
2136         done = 1;
2137
2138         if(getconf("*noahci") != nil)
2139                 return nil;
2140
2141         if(getconf("*ahcidebug") != nil){
2142                 debug = 1;
2143                 datapi = 1;
2144         }
2145
2146         memset(olds, 0xff, sizeof olds);
2147         p = nil;
2148         while((p = pcimatch(p, 0, 0)) != nil){
2149                 if((type = didtype(p)) == -1)
2150                         continue;
2151                 io = p->mem[Abar].bar;
2152                 if(io == 0 || (io & 1) != 0 || p->mem[Abar].size < 0x180)
2153                         continue;
2154                 io &= ~0xf;
2155                 if(niactlr == NCtlr){
2156                         print("iapnp: %s: too many controllers\n", tname[type]);
2157                         break;
2158                 }
2159                 c = iactlr + niactlr;
2160                 s = sdevs + niactlr;
2161                 memset(c, 0, sizeof *c);
2162                 memset(s, 0, sizeof *s);
2163                 c->mmio = vmap(io, p->mem[Abar].size);
2164                 if(c->mmio == 0){
2165                         print("%s: address %#p in use did %.4ux\n",
2166                                 Tname(c), io, p->did);
2167                         continue;
2168                 }
2169                 c->lmmio = (ulong*)c->mmio;
2170                 c->pci = p;
2171                 c->type = type;
2172
2173                 s->ifc = &sdiahciifc;
2174                 s->idno = 'E';
2175                 s->ctlr = c;
2176                 c->sdev = s;
2177
2178                 ahcihandoff((Ahba*)c->mmio);
2179                 if(p->vid == 0x8086)
2180                         iasetupahci(c);
2181                 nunit = ahciconf(c);
2182                 if(nunit < 1){
2183                         vunmap(c->mmio, p->mem[Abar].size);
2184                         continue;
2185                 }
2186                 c->ndrive = s->nunit = nunit;
2187
2188                 /* map the drives -- they don't all need to be enabled. */
2189                 memset(c->rawdrive, 0, sizeof c->rawdrive);
2190                 n = 0;
2191                 for(i = 0; i < NCtlrdrv; i++){
2192                         d = c->rawdrive + i;
2193                         d->portno = i;
2194                         d->driveno = -1;
2195                         d->sectors = 0;
2196                         d->serial[0] = ' ';
2197                         d->ctlr = c;
2198                         if((c->hba->pi & 1<<i) == 0)
2199                                 continue;
2200                         io = 0x100 + 0x80*i;
2201                         if((io + 0x80) > p->mem[Abar].size)
2202                                 continue;
2203                         d->port = (Aport*)(c->mmio + io);
2204                         d->portc.p = d->port;
2205                         d->portc.m = &d->portm;
2206                         d->driveno = n++;
2207                         snprint(d->name, sizeof d->name, "iahci%d.%d", niactlr, i);
2208                         c->drive[d->driveno] = d;
2209                         iadrive[niadrive + d->driveno] = d;
2210                 }
2211                 for(i = 0; i < n; i++){
2212                         c->drive[i]->mode = DMautoneg;
2213                         configdrive(c->drive[i]);
2214                 }
2215                 ahciencinit(c);
2216
2217                 niadrive += n;
2218                 niactlr++;
2219                 sdadddevs(s);
2220                 i = (c->hba->cap >> 21) & 1;
2221                 print("#S/%s: %s: sata-%s with %d ports\n", s->name,
2222                         Tname(c), "I\0II" + i*2, nunit);
2223         }
2224         return nil;
2225 }
2226
2227 static Htab ctab[] = {
2228         Aasp,   "asp",
2229         Aalpe , "alpe ",
2230         Adlae,  "dlae",
2231         Aatapi, "atapi",
2232         Apste,  "pste",
2233         Afbsc,  "fbsc",
2234         Aesp,   "esp",
2235         Acpd,   "cpd",
2236         Ampsp,  "mpsp",
2237         Ahpcp,  "hpcp",
2238         Apma,   "pma",
2239         Acps,   "cps",
2240         Acr,    "cr",
2241         Afr,    "fr",
2242         Ampss,  "mpss",
2243         Apod,   "pod",
2244         Asud,   "sud",
2245         Ast,    "st",
2246 };
2247
2248 static char*
2249 capfmt(char *p, char *e, Htab *t, int n, ulong cap)
2250 {
2251         uint i;
2252
2253         *p = 0;
2254         for(i = 0; i < n; i++)
2255                 if(cap & t[i].bit)
2256                         p = seprint(p, e, "%s ", t[i].name);
2257         return p;
2258 }
2259
2260 static int
2261 iarctl(SDunit *u, char *p, int l)
2262 {
2263         char buf[32], *e, *op;
2264         Aport *o;
2265         Ctlr *c;
2266         Drive *d;
2267
2268         if((c = u->dev->ctlr) == nil)
2269                 return 0;
2270         d = c->drive[u->subno];
2271         o = d->port;
2272
2273         e = p+l;
2274         op = p;
2275         if(d->state == Dready){
2276                 p = seprint(p, e, "model\t%s\n", d->model);
2277                 p = seprint(p, e, "serial\t%s\n", d->serial);
2278                 p = seprint(p, e, "firm\t%s\n", d->firmware);
2279                 if(d->wwn != 0)
2280                         p = seprint(p, e, "wwn\t%ullx\n", d->wwn);
2281                 p = seprint(p, e, "flag\t");
2282                 p = pflag(p, e, &d->portm);
2283                 p = seprint(p, e, "udma\t%d\n", d->portm.udma);
2284         }else
2285                 p = seprint(p, e, "no disk present [%s]\n", diskstates[d->state]);
2286         serrstr(o->serror, buf, buf + sizeof buf - 1);
2287         p = seprint(p, e, "reg\ttask %lux cmd %lux serr %lux %s ci %lux is %lux "
2288                 "sig %lux sstatus %.3lux\n", o->task, o->cmd, o->serror, buf,
2289                 o->ci, o->isr, o->sig, o->sstatus);
2290         p = seprint(p, e, "cmd\t");
2291         p = capfmt(p, e, ctab, nelem(ctab), o->cmd);
2292         p = seprint(p, e, "\n");
2293         p = seprint(p, e, "mode\t%s %s\n", modes[d->mode], modes[maxmode(c)]);
2294         p = seprint(p, e, "geometry %llud %d\n", d->sectors, d->secsize);
2295         p = seprint(p, e, "alignment %d %d\n",
2296                 d->secsize<<d->portm.physshift, d->portm.physalign);
2297         p = seprint(p, e, "missirq\t%ud\n", c->missirq);
2298         return p - op;
2299 }
2300
2301 static void
2302 forcemode(Drive *d, char *mode)
2303 {
2304         int i;
2305
2306         for(i = 0; i < nelem(modes); i++)
2307                 if(strcmp(mode, modes[i]) == 0)
2308                         break;
2309         if(i == nelem(modes))
2310                 i = 0;
2311         ilock(d);
2312         d->mode = i;
2313         iunlock(d);
2314 }
2315
2316 static void
2317 forcestate(Drive *d, char *state)
2318 {
2319         int i;
2320
2321         for(i = 0; i < nelem(diskstates); i++)
2322                 if(strcmp(state, diskstates[i]) == 0)
2323                         break;
2324         if(i == nelem(diskstates))
2325                 error(Ebadctl);
2326         dstatus(d, i);
2327 }
2328
2329 static int
2330 runsettxmode(Drive *d, char *s)
2331 {
2332         int i;
2333         Aportc *c;
2334         Aportm *m;
2335
2336         c = &d->portc;
2337         m = &d->portm;
2338
2339         i = 1;
2340         if(lockready(d) == 0){
2341                 m->udma = atoi(s);
2342                 if(settxmode(c, m->udma) == 0)
2343                         i = 0;
2344         }
2345         qunlock(m);
2346         return i;
2347 }
2348
2349
2350 static int
2351 iawctl(SDunit *u, Cmdbuf *cmd)
2352 {
2353         char **f;
2354         Ctlr *c;
2355         Drive *d;
2356
2357         c = u->dev->ctlr;
2358         d = c->drive[u->subno];
2359         f = cmd->f;
2360
2361         if(strcmp(f[0], "mode") == 0)
2362                 forcemode(d, f[1]? f[1]: "satai");
2363         else if(strcmp(f[0], "state") == 0)
2364                 forcestate(d, f[1]? f[1]: "null");
2365         else if(strcmp(f[0], "txmode") == 0){
2366                 if(runsettxmode(d, f[1]? f[1]: "0"))
2367                         cmderror(cmd, "bad txmode / stuck port");
2368         }else
2369                 cmderror(cmd, Ebadctl);
2370         return 0;
2371 }
2372
2373 static char *
2374 portr(char *p, char *e, uint x)
2375 {
2376         int i, a;
2377
2378         p[0] = 0;
2379         a = -1;
2380         for(i = 0; i < 32; i++){
2381                 if((x & (1<<i)) == 0){
2382                         if(a != -1 && i - 1 != a)
2383                                 p = seprint(p, e, "-%d", i - 1);
2384                         a = -1;
2385                         continue;
2386                 }
2387                 if(a == -1){
2388                         if(i > 0)
2389                                 p = seprint(p, e, ", ");
2390                         p = seprint(p, e, "%d", a = i);
2391                 }
2392         }
2393         if(a != -1 && i - 1 != a)
2394                 p = seprint(p, e, "-%d", i - 1);
2395         return p;
2396 }
2397
2398 static Htab htab[] = {
2399         H64a,   "64a",
2400         Hncq,   "ncq",
2401         Hsntf,  "ntf",
2402         Hmps,   "mps",
2403         Hss,    "ss",
2404         Halp,   "alp",
2405         Hal,    "led",
2406         Hclo,   "clo",
2407         Ham,    "am",
2408         Hpm,    "pm",
2409         Hfbs,   "fbs",
2410         Hpmb,   "pmb",
2411         Hssc,   "slum",
2412         Hpsc,   "pslum",
2413         Hcccs,  "coal",
2414         Hems,   "ems",
2415         Hxs,    "xs",
2416 };
2417
2418 static Htab htab2[] = {
2419         Apts,   "apts",
2420         Nvmp,   "nvmp",
2421         Boh,    "boh",
2422 };
2423
2424 static Htab emtab[] = {
2425         Pm,     "pm",
2426         Alhd,   "alhd",
2427         Xonly,  "xonly",
2428         Smb,    "smb",
2429         Esgpio, "esgpio",
2430         Eses2,  "eses2",
2431         Esafte, "esafte",
2432         Elmt,   "elmt",
2433 };
2434
2435 static char*
2436 iartopctl(SDev *s, char *p, char *e)
2437 {
2438         char pr[25];
2439         ulong cap;
2440         Ahba *h;
2441         Ctlr *c;
2442
2443         c = s->ctlr;
2444         h = c->hba;
2445         cap = h->cap;
2446         p = seprint(p, e, "sd%c ahci %s port %#p: ", s->idno, Tname(c), h);
2447         p = capfmt(p, e, htab, nelem(htab), cap);
2448         p = capfmt(p, e, htab2, nelem(htab2), h->cap2);
2449         p = capfmt(p, e, emtab, nelem(emtab), h->emctl);
2450         portr(pr, pr + sizeof pr, h->pi);
2451         return seprint(p, e,
2452                 "iss %ld ncs %ld np %ld ghc %lux isr %lux pi %lux %s ver %lux\n",
2453                 (cap>>20) & 0xf, (cap>>8) & 0x1f, 1 + (cap & 0x1f),
2454                 h->ghc, h->isr, h->pi, pr, h->ver);
2455 }
2456
2457 static int
2458 iawtopctl(SDev *, Cmdbuf *cmd)
2459 {
2460         int *v;
2461         char **f;
2462
2463         f = cmd->f;
2464         v = 0;
2465
2466         if(strcmp(f[0], "debug") == 0)
2467                 v = &debug;
2468         else if(strcmp(f[0], "idprint") == 0)
2469                 v = &prid;
2470         else if(strcmp(f[0], "aprint") == 0)
2471                 v = &datapi;
2472         else if(strcmp(f[0], "ledprint") == 0)
2473                 v = &dled;
2474         else
2475                 cmderror(cmd, Ebadctl);
2476
2477         switch(cmd->nf){
2478         default:
2479                 cmderror(cmd, Ebadarg);
2480         case 1:
2481                 *v ^= 1;
2482                 break;
2483         case 2:
2484                 if(f[1])
2485                         *v = strcmp(f[1], "on") == 0;
2486                 else
2487                         *v ^= 1;
2488                 break;
2489         }
2490         return 0;
2491 }
2492
2493 SDifc sdiahciifc = {
2494         "ahci",
2495
2496         iapnp,
2497         nil,            /* legacy */
2498         iaenable,
2499         iadisable,
2500
2501         iaverify,
2502         iaonline,
2503         iario,
2504         iarctl,
2505         iawctl,
2506
2507         ahcibio,
2508         nil,            /* probe */
2509         nil,            /* clear */
2510         iartopctl,
2511         iawtopctl,
2512         iaataio,
2513 };