sys/src/9/pc/sdiahci.c

   1 /*
   2  * intel/amd ahci sata controller
   3  * copyright © 2007-8 coraid, inc.
   4  */
   5
   6 #include "u.h"
   7 #include "../port/lib.h"
   8 #include "mem.h"
   9 #include "dat.h"
  10 #include "fns.h"
  11 #include "io.h"
  12 #include "../port/error.h"
  13 #include "../port/sd.h"
  14 #include "ahci.h"
  15
  16 #define dprint(...)     if(debug)       iprint(__VA_ARGS__); else USED(debug)
  17 #define idprint(...)    if(prid)        iprint(__VA_ARGS__);  else USED(prid)
  18 #define aprint(...)     if(datapi)      iprint(__VA_ARGS__);  else USED(datapi)
  19 #define Tname(c)        tname[(c)->type]
  20
  21 enum {
  22         NCtlr   = 4,
  23         NCtlrdrv= 32,
  24         NDrive  = NCtlr*NCtlrdrv,
  25
  26         Read    = 0,
  27         Write,
  28 };
  29
  30 /* pci space configuration */
  31 enum {
  32         Pmap    = 0x90,
  33         Ppcs    = 0x91,
  34         Prev    = 0xa8,
  35 };
  36
  37 enum {
  38         Tesb,
  39         Tich,
  40         Tsb600,
  41         Tunk,
  42 };
  43
  44 #define Intel(x)        ((x)->pci->vid == 0x8086)
  45
  46 static char *tname[] = {
  47         "63xxesb",
  48         "ich",
  49         "sb600",
  50         "unk",
  51 };
  52
  53 enum {
  54         Dnull,
  55         Dmissing,
  56         Dnew,
  57         Dready,
  58         Derror,
  59         Dreset,
  60         Doffline,
  61         Dportreset,
  62         Dlast,
  63 };
  64
  65 static char *diskstates[Dlast] = {
  66         "null",
  67         "missing",
  68         "new",
  69         "ready",
  70         "error",
  71         "reset",
  72         "offline",
  73         "portreset",
  74 };
  75
  76 extern SDifc sdiahciifc;
  77 typedef struct Ctlr Ctlr;
  78
  79 enum {
  80         DMautoneg,
  81         DMsatai,
  82         DMsataii,
  83 };
  84
  85 static char *modename[] = {
  86         "auto",
  87         "satai",
  88         "sataii",
  89 };
  90
  91 static char *flagname[] = {
  92         "llba",
  93         "smart",
  94         "power",
  95         "nop",
  96         "atapi",
  97         "atapi16",
  98 };
  99
 100 typedef struct {
 101         Lock;
 102
 103         Ctlr    *ctlr;
 104         SDunit  *unit;
 105         char    name[10];
 106         Aport   *port;
 107         Aportm  portm;
 108         Aportc  portc;          /* redundant ptr to port and portm */
 109
 110         uchar   mediachange;
 111         uchar   state;
 112         uchar   smartrs;
 113
 114         uvlong  sectors;
 115         ulong   secsize;
 116         ulong   intick;         /* start tick of current transfer */
 117         ulong   lastseen;
 118         int     wait;
 119         uchar   mode;           /* DMautoneg, satai or sataii */
 120         uchar   active;
 121
 122         char    serial[20+1];
 123         char    firmware[8+1];
 124         char    model[40+1];
 125
 126         ushort  info[0x200];
 127
 128         int     driveno;        /* ctlr*NCtlrdrv + unit */
 129         /* controller port # != driveno when not all ports are enabled */
 130         int     portno;
 131 } Drive;
 132
 133 struct Ctlr {
 134         Lock;
 135
 136         int     type;
 137         int     enabled;
 138         SDev    *sdev;
 139         Pcidev  *pci;
 140
 141         uchar   *mmio;
 142         ulong   *lmmio;
 143         Ahba    *hba;
 144
 145         Drive   rawdrive[NCtlrdrv];
 146         Drive*  drive[NCtlrdrv];
 147         int     ndrive;
 148         int     mport;
 149 };
 150
 151 static  Ctlr    iactlr[NCtlr];
 152 static  SDev    sdevs[NCtlr];
 153 static  int     niactlr;
 154
 155 static  Drive   *iadrive[NDrive];
 156 static  int     niadrive;
 157
 158 /* these are fiddled in iawtopctl() */
 159 static  int     debug;
 160 static  int     prid = 1;
 161 static  int     datapi;
 162
 163 static char stab[] = {
 164 [0]     'i', 'm',
 165 [8]     't', 'c', 'p', 'e',
 166 [16]    'N', 'I', 'W', 'B', 'D', 'C', 'H', 'S', 'T', 'F', 'X'
 167 };
 168
 169 static void
 170 serrstr(ulong r, char *s, char *e)
 171 {
 172         int i;
 173
 174         e -= 3;
 175         for(i = 0; i < nelem(stab) && s < e; i++)
 176                 if(r & (1<<i) && stab[i]){
 177                         *s++ = stab[i];
 178                         if(SerrBad & (1<<i))
 179                                 *s++ = '*';
 180                 }
 181         *s = 0;
 182 }
 183
 184 static char ntab[] = "0123456789abcdef";
 185
 186 static void
 187 preg(uchar *reg, int n)
 188 {
 189         int i;
 190         char buf[25*3+1], *e;
 191
 192         e = buf;
 193         for(i = 0; i < n; i++){
 194                 *e++ = ntab[reg[i]>>4];
 195                 *e++ = ntab[reg[i]&0xf];
 196                 *e++ = ' ';
 197         }
 198         *e++ = '\n';
 199         *e = 0;
 200         dprint(buf);
 201 }
 202
 203 static void
 204 dreg(char *s, Aport *p)
 205 {
 206         dprint("ahci: %stask=%lux; cmd=%lux; ci=%lux; is=%lux\n",
 207                 s, p->task, p->cmd, p->ci, p->isr);
 208 }
 209
 210 static void
 211 esleep(int ms)
 212 {
 213         if(waserror())
 214                 return;
 215         tsleep(&up->sleep, return0, 0, ms);
 216         poperror();
 217 }
 218
 219 typedef struct {
 220         Aport   *p;
 221         int     i;
 222 }Asleep;
 223
 224 static int
 225 ahciclear(void *v)
 226 {
 227         Asleep *s;
 228
 229         s = v;
 230         return (s->p->ci & s->i) == 0;
 231 }
 232
 233 static void
 234 aesleep(Aportm *m, Asleep *a, int ms)
 235 {
 236         if(waserror())
 237                 return;
 238         tsleep(m, ahciclear, a, ms);
 239         poperror();
 240 }
 241
 242 static int
 243 ahciwait(Aportc *c, int ms)
 244 {
 245         Asleep as;
 246         Aport *p;
 247
 248         p = c->p;
 249         p->ci = 1;
 250         as.p = p;
 251         as.i = 1;
 252         aesleep(c->m, &as, ms);
 253         if((p->task&1) == 0 && p->ci == 0)
 254                 return 0;
 255         dreg("ahciwait timeout ", c->p);
 256         return -1;
 257 }
 258
 259 static int
 260 nop(Aportc *pc)
 261 {
 262         uchar *c;
 263         Actab *t;
 264         Alist *l;
 265
 266         if((pc->m->feat & Dnop) == 0)
 267                 return -1;
 268
 269         t = pc->m->ctab;
 270         c = t->cfis;
 271
 272         memset(c, 0, 0x20);
 273         c[0] = 0x27;
 274         c[1] = 0x80;
 275         c[2] = 0x00;
 276         c[7] = 0xa0;            /* obsolete device bits */
 277
 278         l = pc->m->list;
 279         l->flags = Lwrite | 0x5;
 280         l->len = 0;
 281         l->ctab = PCIWADDR(t);
 282         l->ctabhi = 0;
 283
 284         return ahciwait(pc, 3*1000);
 285 }
 286
 287 static int
 288 setfeatures(Aportc *pc, uchar f)
 289 {
 290         uchar *c;
 291         Actab *t;
 292         Alist *l;
 293
 294         t = pc->m->ctab;
 295         c = t->cfis;
 296
 297         memset(c, 0, 0x20);
 298         c[0] = 0x27;
 299         c[1] = 0x80;
 300         c[2] = 0xef;
 301         c[3] = f;
 302         c[7] = 0xa0;            /* obsolete device bits */
 303
 304         l = pc->m->list;
 305         l->flags = Lwrite | 0x5;
 306         l->len = 0;
 307         l->ctab = PCIWADDR(t);
 308         l->ctabhi = 0;
 309
 310         return ahciwait(pc, 3*1000);
 311 }
 312
 313 static int
 314 setudmamode(Aportc *pc, uchar f)
 315 {
 316         uchar *c;
 317         Actab *t;
 318         Alist *l;
 319
 320         /* hack */
 321         if((pc->p->sig >> 16) == 0xeb14)
 322                 return 0;
 323
 324         t = pc->m->ctab;
 325         c = t->cfis;
 326
 327         memset(c, 0, 0x20);
 328         c[0] = 0x27;
 329         c[1] = 0x80;
 330         c[2] = 0xef;
 331         c[3] = 3;               /* set transfer mode */
 332         c[7] = 0xa0;            /* obsolete device bits */
 333         c[12] = 0x40 | f;       /* sector count */
 334
 335         l = pc->m->list;
 336         l->flags = Lwrite | 0x5;
 337         l->len = 0;
 338         l->ctab = PCIWADDR(t);
 339         l->ctabhi = 0;
 340
 341         return ahciwait(pc, 3*1000);
 342 }
 343
 344 static void
 345 asleep(int ms)
 346 {
 347         if(up == nil)
 348                 delay(ms);
 349         else
 350                 esleep(ms);
 351 }
 352
 353 static int
 354 ahciportreset(Aportc *c)
 355 {
 356         ulong *cmd, i;
 357         Aport *p;
 358
 359         p = c->p;
 360         cmd = &p->cmd;
 361         *cmd &= ~(Afre|Ast);
 362         for(i = 0; i < 500; i += 25){
 363                 if((*cmd&Acr) == 0)
 364                         break;
 365                 asleep(25);
 366         }
 367         p->sctl = 1|(p->sctl&~7);
 368         delay(1);
 369         p->sctl &= ~7;
 370         return 0;
 371 }
 372
 373 static int
 374 smart(Aportc *pc, int n)
 375 {
 376         uchar *c;
 377         Actab *t;
 378         Alist *l;
 379
 380         if((pc->m->feat&Dsmart) == 0)
 381                 return -1;
 382
 383         t = pc->m->ctab;
 384         c = t->cfis;
 385
 386         memset(c, 0, 0x20);
 387         c[0] = 0x27;
 388         c[1] = 0x80;
 389         c[2] = 0xb0;
 390         c[3] = 0xd8 + n;        /* able smart */
 391         c[5] = 0x4f;
 392         c[6] = 0xc2;
 393         c[7] = 0xa0;
 394
 395         l = pc->m->list;
 396         l->flags = Lwrite | 0x5;
 397         l->len = 0;
 398         l->ctab = PCIWADDR(t);
 399         l->ctabhi = 0;
 400
 401         if(ahciwait(pc, 1000) == -1 || pc->p->task & (1|32)){
 402                 dprint("ahci: smart fail %lux\n", pc->p->task);
 403 //              preg(pc->m->fis.r, 20);
 404                 return -1;
 405         }
 406         if(n)
 407                 return 0;
 408         return 1;
 409 }
 410
 411 static int
 412 smartrs(Aportc *pc)
 413 {
 414         uchar *c;
 415         Actab *t;
 416         Alist *l;
 417
 418         t = pc->m->ctab;
 419         c = t->cfis;
 420
 421         memset(c, 0, 0x20);
 422         c[0] = 0x27;
 423         c[1] = 0x80;
 424         c[2] = 0xb0;
 425         c[3] = 0xda;            /* return smart status */
 426         c[5] = 0x4f;
 427         c[6] = 0xc2;
 428         c[7] = 0xa0;
 429
 430         l = pc->m->list;
 431         l->flags = Lwrite | 0x5;
 432         l->len = 0;
 433         l->ctab = PCIWADDR(t);
 434         l->ctabhi = 0;
 435
 436         c = pc->m->fis.r;
 437         if(ahciwait(pc, 1000) == -1 || pc->p->task & (1|32)){
 438                 dprint("ahci: smart fail %lux\n", pc->p->task);
 439                 preg(c, 20);
 440                 return -1;
 441         }
 442         if(c[5] == 0x4f && c[6] == 0xc2)
 443                 return 1;
 444         return 0;
 445 }
 446
 447 static int
 448 ahciflushcache(Aportc *pc)
 449 {
 450         uchar *c, llba;
 451         Actab *t;
 452         Alist *l;
 453         static uchar tab[2] = {0xe7, 0xea};
 454
 455         llba = pc->m->feat&Dllba? 1: 0;
 456         t = pc->m->ctab;
 457         c = t->cfis;
 458
 459         memset(c, 0, 0x20);
 460         c[0] = 0x27;
 461         c[1] = 0x80;
 462         c[2] = tab[llba];
 463         c[7] = 0xa0;
 464
 465         l = pc->m->list;
 466         l->flags = Lwrite | 0x5;
 467         l->len = 0;
 468         l->ctab = PCIWADDR(t);
 469         l->ctabhi = 0;
 470
 471         if(ahciwait(pc, 60000) == -1 || pc->p->task & (1|32)){
 472                 dprint("ahciflushcache: fail %lux\n", pc->p->task);
 473 //              preg( pc->m->fis.r, 20);
 474                 return -1;
 475         }
 476         return 0;
 477 }
 478
 479 static ushort
 480 gbit16(void *a)
 481 {
 482         uchar *i;
 483
 484         i = a;
 485         return i[1]<<8 | i[0];
 486 }
 487
 488 static ulong
 489 gbit32(void *a)
 490 {
 491         ulong j;
 492         uchar *i;
 493
 494         i = a;
 495         j  = i[3] << 24;
 496         j |= i[2] << 16;
 497         j |= i[1] << 8;
 498         j |= i[0];
 499         return j;
 500 }
 501
 502 static uvlong
 503 gbit64(void *a)
 504 {
 505         uchar *i;
 506
 507         i = a;
 508         return (uvlong)gbit32(i+4) << 32 | gbit32(a);
 509 }
 510
 511 static int
 512 ahciidentify0(Aportc *pc, void *id, int atapi)
 513 {
 514         uchar *c;
 515         Actab *t;
 516         Alist *l;
 517         Aprdt *p;
 518         static uchar tab[] = { 0xec, 0xa1, };
 519
 520         t = pc->m->ctab;
 521         c = t->cfis;
 522
 523         memset(c, 0, 0x20);
 524         c[0] = 0x27;
 525         c[1] = 0x80;
 526         c[2] = tab[atapi];
 527         c[7] = 0xa0;            /* obsolete device bits */
 528
 529         l = pc->m->list;
 530         l->flags = 1<<16 | 0x5;
 531         l->len = 0;
 532         l->ctab = PCIWADDR(t);
 533         l->ctabhi = 0;
 534
 535         memset(id, 0, 0x100);
 536         p = &t->prdt;
 537         p->dba = PCIWADDR(id);
 538         p->dbahi = 0;
 539         p->count = 1<<31 | (0x200-2) | 1;
 540
 541         return ahciwait(pc, 3*1000);
 542 }
 543
 544 static vlong
 545 ahciidentify(Aportc *pc, ushort *id)
 546 {
 547         int i, sig;
 548         vlong s;
 549         Aportm *m;
 550
 551         m = pc->m;
 552         m->feat = 0;
 553         m->smart = 0;
 554         i = 0;
 555         sig = pc->p->sig >> 16;
 556         if(sig == 0xeb14){
 557                 m->feat |= Datapi;
 558                 i = 1;
 559         }
 560         if(ahciidentify0(pc, id, i) == -1)
 561                 return -1;
 562
 563         i = gbit16(id+83) | gbit16(id+86);
 564         if(i & (1<<10)){
 565                 m->feat |= Dllba;
 566                 s = gbit64(id+100);
 567         }else
 568                 s = gbit32(id+60);
 569
 570         if(m->feat&Datapi){
 571                 i = gbit16(id+0);
 572                 if(i&1)
 573                         m->feat |= Datapi16;
 574         }
 575
 576         i = gbit16(id+83);
 577         if((i>>14) == 1) {
 578                 if(i & (1<<3))
 579                         m->feat |= Dpower;
 580                 i = gbit16(id+82);
 581                 if(i & 1)
 582                         m->feat |= Dsmart;
 583                 if(i & (1<<14))
 584                         m->feat |= Dnop;
 585         }
 586         return s;
 587 }
 588
 589 static int
 590 ahciquiet(Aport *a)
 591 {
 592         ulong *p, i;
 593
 594         p = &a->cmd;
 595         *p &= ~Ast;
 596         for(i = 0; i < 500; i += 50){
 597                 if((*p & Acr) == 0)
 598                         goto stop;
 599                 asleep(50);
 600         }
 601         return -1;
 602 stop:
 603         if((a->task & (ASdrq|ASbsy)) == 0){
 604                 *p |= Ast;
 605                 return 0;
 606         }
 607
 608         *p |= Aclo;
 609         for(i = 0; i < 500; i += 50){
 610                 if((*p & Aclo) == 0)
 611                         goto stop1;
 612                 asleep(50);
 613         }
 614         return -1;
 615 stop1:
 616         /* extra check */
 617         dprint("ahci: clo clear %lx\n", a->task);
 618         if(a->task & ASbsy)
 619                 return -1;
 620         *p |= Ast;
 621         return 0;
 622 }
 623
 624 static int
 625 ahcicomreset(Aportc *pc)
 626 {
 627         uchar *c;
 628         Actab *t;
 629         Alist *l;
 630
 631         dprint("ahcicomreset\n");
 632         dreg("ahci: comreset ", pc->p);
 633         if(ahciquiet(pc->p) == -1){
 634                 dprint("ahciquiet failed\n");
 635                 return -1;
 636         }
 637         dreg("comreset ", pc->p);
 638
 639         t = pc->m->ctab;
 640         c = t->cfis;
 641
 642         memset(c, 0, 0x20);
 643         c[0] = 0x27;
 644         c[1] = 0x00;
 645         c[7] = 0xa0;            /* obsolete device bits */
 646         c[15] = 1<<2;           /* srst */
 647
 648         l = pc->m->list;
 649         l->flags = Lclear | Lreset | 0x5;
 650         l->len = 0;
 651         l->ctab = PCIWADDR(t);
 652         l->ctabhi = 0;
 653
 654         if(ahciwait(pc, 500) == -1){
 655                 dprint("ahcicomreset: first command failed\n");
 656                 return -1;
 657         }
 658         microdelay(250);
 659         dreg("comreset ", pc->p);
 660
 661         memset(c, 0, 0x20);
 662         c[0] = 0x27;
 663         c[1] = 0x00;
 664         c[7] = 0xa0;            /* obsolete device bits */
 665
 666         l = pc->m->list;
 667         l->flags = Lwrite | 0x5;
 668         l->len = 0;
 669         l->ctab = PCIWADDR(t);
 670         l->ctabhi = 0;
 671
 672         if(ahciwait(pc, 150) == -1){
 673                 dprint("ahcicomreset: second command failed\n");
 674                 return -1;
 675         }
 676         dreg("comreset ", pc->p);
 677         return 0;
 678 }
 679
 680 static int
 681 ahciidle(Aport *port)
 682 {
 683         ulong *p, i, r;
 684
 685         p = &port->cmd;
 686         if((*p & Arun) == 0)
 687                 return 0;
 688         *p &= ~Ast;
 689         r = 0;
 690         for(i = 0; i < 500; i += 25){
 691                 if((*p & Acr) == 0)
 692                         goto stop;
 693                 asleep(25);
 694         }
 695         r = -1;
 696 stop:
 697         if((*p & Afre) == 0)
 698                 return r;
 699         *p &= ~Afre;
 700         for(i = 0; i < 500; i += 25){
 701                 if((*p & Afre) == 0)
 702                         return 0;
 703                 asleep(25);
 704         }
 705         return -1;
 706 }
 707
 708 /*
 709  * § 6.2.2.1  first part; comreset handled by reset disk.
 710  *      - remainder is handled by configdisk.
 711  *      - ahcirecover is a quick recovery from a failed command.
 712  */
 713 static int
 714 ahciswreset(Aportc *pc)
 715 {
 716         int i;
 717
 718         i = ahciidle(pc->p);
 719         pc->p->cmd |= Afre;
 720         if(i == -1)
 721                 return -1;
 722         if(pc->p->task & (ASdrq|ASbsy))
 723                 return -1;
 724         return 0;
 725 }
 726
 727 static int
 728 ahcirecover(Aportc *pc)
 729 {
 730         ahciswreset(pc);
 731         pc->p->cmd |= Ast;
 732         if(setudmamode(pc, 5) == -1)
 733                 return -1;
 734         return 0;
 735 }
 736
 737 static void*
 738 malign(int size, int align)
 739 {
 740         void *v;
 741
 742         v = xspanalloc(size, align, 0);
 743         memset(v, 0, size);
 744         return v;
 745 }
 746
 747 static void
 748 setupfis(Afis *f)
 749 {
 750         f->base = malign(0x100, 0x100);
 751         f->d = f->base + 0;
 752         f->p = f->base + 0x20;
 753         f->r = f->base + 0x40;
 754         f->u = f->base + 0x60;
 755         f->devicebits = (ulong*)(f->base + 0x58);
 756 }
 757
 758 static void
 759 ahciwakeup(Aport *p)
 760 {
 761         ushort s;
 762
 763         s = p->sstatus;
 764         if((s & 0xF00) != 0x600)
 765                 return;
 766         if((s & 7) != 1){               /* not (device, no phy) */
 767                 iprint("ahci: slumbering drive unwakable %ux\n", s);
 768                 return;
 769         }
 770         p->sctl = 3*Aipm | 0*Aspd | Adet;
 771         delay(1);
 772         p->sctl &= ~7;
 773 //      iprint("ahci: wake %ux -> %ux\n", s, p->sstatus);
 774 }
 775
 776 static int
 777 ahciconfigdrive(Ahba *h, Aportc *c, int mode)
 778 {
 779         Aportm *m;
 780         Aport *p;
 781
 782         p = c->p;
 783         m = c->m;
 784
 785         if(m->list == 0){
 786                 setupfis(&m->fis);
 787                 m->list = malign(sizeof *m->list, 1024);
 788                 m->ctab = malign(sizeof *m->ctab, 128);
 789         }
 790
 791         if(p->sstatus & 3 && h->cap & Hsss){
 792                 /* device connected & staggered spin-up */
 793                 dprint("ahci: configdrive:  spinning up ... [%lux]\n",
 794                         p->sstatus);
 795                 p->cmd |= Apod|Asud;
 796                 asleep(1400);
 797         }
 798
 799         p->serror = SerrAll;
 800
 801         p->list = PCIWADDR(m->list);
 802         p->listhi = 0;
 803         p->fis = PCIWADDR(m->fis.base);
 804         p->fishi = 0;
 805         p->cmd |= Afre|Ast;
 806
 807         if((p->sstatus & 0xF0F) == 0x601) /* drive coming up in slumbering? */
 808                 ahciwakeup(p);
 809
 810         /* disable power managment sequence from book. */
 811         p->sctl = (3*Aipm) | (mode*Aspd) | (0*Adet);
 812         p->cmd &= ~Aalpe;
 813
 814         p->ie = IEM;
 815
 816         return 0;
 817 }
 818
 819 static int
 820 ahcienable(Ahba *h)
 821 {
 822         h->ghc |= Hie;
 823         return 0;
 824 }
 825
 826 static int
 827 ahcidisable(Ahba *h)
 828 {
 829         h->ghc &= ~Hie;
 830         return 0;
 831 }
 832
 833 static int
 834 countbits(ulong u)
 835 {
 836         int i, n;
 837
 838         n = 0;
 839         for(i = 0; i < 32; i++)
 840                 if(u & (1<<i))
 841                         n++;
 842         return n;
 843 }
 844
 845 static int
 846 ahciconf(Ctlr *ctlr)
 847 {
 848         Ahba *h;
 849         ulong u;
 850
 851         h = ctlr->hba = (Ahba*)ctlr->mmio;
 852         u = h->cap;
 853
 854         if((u&Hsam) == 0)
 855                 h->ghc |= Hae;
 856
 857         print("#S/sd%c: ahci %s port %#p: sss %ld ncs %ld coal %ld "
 858                 "mports %ld led %ld clo %ld ems %ld\n",
 859                 ctlr->sdev->idno, tname[ctlr->type], h,
 860                 (u>>27) & 1, (u>>8) & 0x1f, (u>>7) & 1, u & 0x1f, (u>>25) & 1,
 861                 (u>>24) & 1, (u>>6) & 1);
 862         return countbits(h->pi);
 863 }
 864
 865 static int
 866 ahcihbareset(Ahba *h)
 867 {
 868         int wait;
 869
 870         h->ghc |= 1;
 871         for(wait = 0; wait < 1000; wait += 100){
 872                 if(h->ghc == 0)
 873                         return 0;
 874                 delay(100);
 875         }
 876         return -1;
 877 }
 878
 879 static void
 880 idmove(char *p, ushort *a, int n)
 881 {
 882         int i;
 883         char *op, *e;
 884
 885         op = p;
 886         for(i = 0; i < n/2; i++){
 887                 *p++ = a[i] >> 8;
 888                 *p++ = a[i];
 889         }
 890         *p = 0;
 891         while(p > op && *--p == ' ')
 892                 *p = 0;
 893         e = p;
 894         for (p = op; *p == ' '; p++)
 895                 ;
 896         memmove(op, p, n - (e - p));
 897 }
 898
 899 static int
 900 identify(Drive *d)
 901 {
 902         ushort *id;
 903         vlong osectors, s;
 904         uchar oserial[21];
 905         SDunit *u;
 906
 907         id = d->info;
 908         s = ahciidentify(&d->portc, id);
 909         if(s == -1){
 910                 d->state = Derror;
 911                 return -1;
 912         }
 913         osectors = d->sectors;
 914         memmove(oserial, d->serial, sizeof d->serial);
 915
 916         u = d->unit;
 917         d->sectors = s;
 918         d->secsize = u->secsize;
 919         if(d->secsize == 0)
 920                 d->secsize = 512;               /* default */
 921         d->smartrs = 0;
 922
 923         idmove(d->serial, id+10, 20);
 924         idmove(d->firmware, id+23, 8);
 925         idmove(d->model, id+27, 40);
 926
 927         memset(u->inquiry, 0, sizeof u->inquiry);
 928         u->inquiry[2] = 2;
 929         u->inquiry[3] = 2;
 930         u->inquiry[4] = sizeof u->inquiry - 4;
 931         memmove(u->inquiry+8, d->model, 40);
 932
 933         if(osectors != s || memcmp(oserial, d->serial, sizeof oserial) != 0){
 934                 d->mediachange = 1;
 935                 u->sectors = 0;
 936         }
 937         return 0;
 938 }
 939
 940 static void
 941 clearci(Aport *p)
 942 {
 943         if(p->cmd & Ast) {
 944                 p->cmd &= ~Ast;
 945                 p->cmd |=  Ast;
 946         }
 947 }
 948
 949 static void
 950 updatedrive(Drive *d)
 951 {
 952         ulong cause, serr, s0, pr, ewake;
 953         char *name;
 954         Aport *p;
 955         static ulong last;
 956
 957         pr = 1;
 958         ewake = 0;
 959         p = d->port;
 960         cause = p->isr;
 961         serr = p->serror;
 962         p->isr = cause;
 963         name = "??";
 964         if(d->unit && d->unit->name)
 965                 name = d->unit->name;
 966
 967         if(p->ci == 0){
 968                 d->portm.flag |= Fdone;
 969                 wakeup(&d->portm);
 970                 pr = 0;
 971         }else if(cause & Adps)
 972                 pr = 0;
 973         if(cause & Ifatal){
 974                 ewake = 1;
 975                 dprint("ahci: updatedrive: fatal\n");
 976         }
 977         if(cause & Adhrs){
 978                 if(p->task & (1<<5|1)){
 979                         dprint("ahci: Adhrs cause %lux serr %lux task %lux\n",
 980                                 cause, serr, p->task);
 981                         d->portm.flag |= Ferror;
 982                         ewake = 1;
 983                 }
 984                 pr = 0;
 985         }
 986         if(p->task & 1 && last != cause)
 987                 dprint("%s: err ca %lux serr %lux task %lux sstat %lux\n",
 988                         name, cause, serr, p->task, p->sstatus);
 989         if(pr)
 990                 dprint("%s: upd %lux ta %lux\n", name, cause, p->task);
 991
 992         if(cause & (Aprcs|Aifs)){
 993                 s0 = d->state;
 994                 switch(p->sstatus & 7){
 995                 case 0:                         /* no device */
 996                         d->state = Dmissing;
 997                         break;
 998                 case 1:                         /* device but no phy comm. */
 999                         if((p->sstatus & 0xF00) == 0x600)
1000                                 d->state = Dnew; /* slumbering */
1001                         else
1002                                 d->state = Derror;
1003                         break;
1004                 case 3:                         /* device & phy comm. estab. */
1005                         /* power mgnt crap for surprise removal */
1006                         p->ie |= Aprcs|Apcs;    /* is this required? */
1007                         d->state = Dreset;
1008                         break;
1009                 case 4:                         /* phy off-line */
1010                         d->state = Doffline;
1011                         break;
1012                 }
1013                 dprint("%s: %s → %s [Apcrs] %lux\n", name,
1014                         diskstates[s0], diskstates[d->state], p->sstatus);
1015                 /* print pulled message here. */
1016                 if(s0 == Dready && d->state != Dready)
1017                         idprint("%s: pulled\n", name);
1018                 if(d->state != Dready)
1019                         d->portm.flag |= Ferror;
1020                 ewake = 1;
1021         }
1022         p->serror = serr;
1023         if(ewake){
1024                 clearci(p);
1025                 wakeup(&d->portm);
1026         }
1027         last = cause;
1028 }
1029
1030 static void
1031 pstatus(Drive *d, ulong s)
1032 {
1033         /*
1034          * bogus code because the first interrupt is currently dropped.
1035          * likely my fault.  serror may be cleared at the wrong time.
1036          */
1037         switch(s){
1038         case 0:                 /* no device */
1039                 d->state = Dmissing;
1040                 break;
1041         case 1:                 /* device but no phy. comm. */
1042                 break;
1043         case 2:                 /* should this be missing?  need testcase. */
1044                 dprint("ahci: pstatus 2\n");
1045                 /* fallthrough */
1046         case 3:                 /* device & phy. comm. */
1047                 d->wait = 0;
1048                 d->state = Dnew;
1049                 break;
1050         case 4:                 /* offline */
1051                 d->state = Doffline;
1052                 break;
1053         case 6:                 /* ? not sure this makes sense. TODO */
1054                 d->state = Dnew;
1055                 break;
1056         }
1057 }
1058
1059 static int
1060 configdrive(Drive *d)
1061 {
1062         if(ahciconfigdrive(d->ctlr->hba, &d->portc, d->mode) == -1)
1063                 return -1;
1064         ilock(d);
1065         pstatus(d, d->port->sstatus & 7);
1066         iunlock(d);
1067         return 0;
1068 }
1069
1070 static void
1071 resetdisk(Drive *d)
1072 {
1073         uint state, det, stat;
1074         Aport *p;
1075
1076         p = d->port;
1077         det = p->sctl & 7;
1078         stat = p->sstatus & 7;
1079         state = (p->cmd>>28) & 0xf;
1080         dprint("ahci: resetdisk: icc %ux  det %d sdet %d\n", state, det, stat);
1081
1082         ilock(d);
1083         state = d->state;
1084         if(d->state != Dready || d->state != Dnew)
1085                 d->portm.flag |= Ferror;
1086         clearci(p);                     /* satisfy sleep condition. */
1087         wakeup(&d->portm);
1088         if(stat != 3){          /* device absent or phy not communicating? */
1089                 d->state = Dportreset;
1090                 iunlock(d);
1091                 return;
1092         }
1093         d->state = Derror;
1094         iunlock(d);
1095
1096         qlock(&d->portm);
1097         if(p->cmd&Ast && ahciswreset(&d->portc) == -1){
1098                 ilock(d);
1099                 d->state = Dportreset;  /* get a bigger stick. */
1100                 iunlock(d);
1101         } else {
1102                 ilock(d);
1103                 d->state = Dmissing;
1104                 iunlock(d);
1105
1106                 configdrive(d);
1107         }
1108         dprint("ahci: resetdisk: %s → %s\n",
1109                 diskstates[state], diskstates[d->state]);
1110         qunlock(&d->portm);
1111 }
1112
1113 static int
1114 newdrive(Drive *d)
1115 {
1116         char *name;
1117         Aportc *c;
1118         Aportm *m;
1119
1120         c = &d->portc;
1121         m = &d->portm;
1122
1123         name = d->unit->name;
1124         if(name == 0)
1125                 name = "??";
1126
1127         if(d->port->task == 0x80)
1128                 return -1;
1129         qlock(c->m);
1130         if(setudmamode(c, 5) == -1){
1131                 dprint("%s: can't set udma mode\n", name);
1132                 goto lose;
1133         }
1134         if(identify(d) == -1){
1135                 dprint("%s: identify failure\n", name);
1136                 goto lose;
1137         }
1138         if(m->feat & Dpower && setfeatures(c, 0x85) == -1){
1139                 m->feat &= ~Dpower;
1140                 if(ahcirecover(c) == -1)
1141                         goto lose;
1142         }
1143
1144         ilock(d);
1145         d->state = Dready;
1146         iunlock(d);
1147
1148         qunlock(c->m);
1149
1150         idprint("%s: %sLBA %,llud sectors: %s %s %s %s\n", d->unit->name,
1151                 (m->feat & Dllba? "L": ""), d->sectors, d->model, d->firmware,
1152                 d->serial, d->mediachange? "[mediachange]": "");
1153         return 0;
1154
1155 lose:
1156         idprint("%s: can't be initialized\n", d->unit->name);
1157         ilock(d);
1158         d->state = Dnull;
1159         iunlock(d);
1160         qunlock(c->m);
1161         return -1;
1162 }
1163
1164 enum {
1165         Nms             = 256,
1166         Mphywait        =  2*1024/Nms - 1,
1167         Midwait         = 16*1024/Nms - 1,
1168         Mcomrwait       = 64*1024/Nms - 1,
1169 };
1170
1171 static void
1172 westerndigitalhung(Drive *d)
1173 {
1174         if((d->portm.feat&Datapi) == 0 && d->active &&
1175             TK2MS(MACHP(0)->ticks - d->intick) > 5000){
1176                 dprint("%s: drive hung; resetting [%lux] ci %lx\n",
1177                         d->unit->name, d->port->task, d->port->ci);
1178                 d->state = Dreset;
1179         }
1180 }
1181
1182 static ushort olds[NCtlr*NCtlrdrv];
1183
1184 static int
1185 doportreset(Drive *d)
1186 {
1187         int i;
1188
1189         i = -1;
1190         qlock(&d->portm);
1191         if(ahciportreset(&d->portc) == -1)
1192                 dprint("ahci: doportreset: fails\n");
1193         else
1194                 i = 0;
1195         qunlock(&d->portm);
1196         dprint("ahci: doportreset: portreset → %s  [task %lux]\n",
1197                 diskstates[d->state], d->port->task);
1198         return i;
1199 }
1200
1201 /* drive must be locked */
1202 static void
1203 statechange(Drive *d)
1204 {
1205         switch(d->state){
1206         case Dnull:
1207         case Doffline:
1208                 if(d->unit->sectors != 0){
1209                         d->sectors = 0;
1210                         d->mediachange = 1;
1211                 }
1212                 /* fallthrough */
1213         case Dready:
1214                 d->wait = 0;
1215                 break;
1216         }
1217 }
1218
1219 static void
1220 checkdrive(Drive *d, int i)
1221 {
1222         ushort s;
1223         char *name;
1224
1225         if(d == nil) {
1226                 print("checkdrive: nil d\n");
1227                 return;
1228         }
1229         ilock(d);
1230         if(d->unit == nil || d->port == nil) {
1231                 if(0)
1232                         print("checkdrive: nil d->%s\n",
1233                                 d->unit == nil? "unit": "port");
1234                 iunlock(d);
1235                 return;
1236         }
1237         name = d->unit->name;
1238         s = d->port->sstatus;
1239         if(s)
1240                 d->lastseen = MACHP(0)->ticks;
1241         if(s != olds[i]){
1242                 dprint("%s: status: %04ux -> %04ux: %s\n",
1243                         name, olds[i], s, diskstates[d->state]);
1244                 olds[i] = s;
1245                 d->wait = 0;
1246         }
1247         westerndigitalhung(d);
1248         switch(d->state){
1249         case Dnull:
1250         case Dready:
1251                 break;
1252         case Dmissing:
1253         case Dnew:
1254                 switch(s & 0x107){
1255                 case 1:         /* no device (pm), device but no phy. comm. */
1256                         ahciwakeup(d->port);
1257                         /* fall through */
1258                 case 0:         /* no device */
1259                         break;
1260                 default:
1261                         dprint("%s: unknown status %04ux\n", name, s);
1262                         /* fall through */
1263                 case 0x100:             /* active, no device */
1264                         if(++d->wait&Mphywait)
1265                                 break;
1266 reset:
1267                         if(++d->mode > DMsataii)
1268                                 d->mode = 0;
1269                         if(d->mode == DMsatai){ /* we tried everything */
1270                                 d->state = Dportreset;
1271                                 goto portreset;
1272                         }
1273                         dprint("%s: reset; new mode %s\n", name,
1274                                 modename[d->mode]);
1275                         iunlock(d);
1276                         resetdisk(d);
1277                         ilock(d);
1278                         break;
1279                 case 0x103:             /* active, device, phy. comm. */
1280                         if((++d->wait&Midwait) == 0){
1281                                 dprint("%s: slow reset %04ux task=%lux; %d\n",
1282                                         name, s, d->port->task, d->wait);
1283                                 goto reset;
1284                         }
1285                         s = (uchar)d->port->task;
1286                         if(s == 0x7f || ((d->port->sig >> 16) != 0xeb14 &&
1287                             (s & ~0x17) != (1<<6)))
1288                                 break;
1289                         iunlock(d);
1290                         newdrive(d);
1291                         ilock(d);
1292                         break;
1293                 }
1294                 break;
1295         case Doffline:
1296                 if(d->wait++ & Mcomrwait)
1297                         break;
1298                 /* fallthrough */
1299         case Derror:
1300         case Dreset:
1301                 dprint("%s: reset [%s]: mode %d; status %04ux\n",
1302                         name, diskstates[d->state], d->mode, s);
1303                 iunlock(d);
1304                 resetdisk(d);
1305                 ilock(d);
1306                 break;
1307         case Dportreset:
1308 portreset:
1309                 if(d->wait++ & 0xff && (s & 0x100) == 0)
1310                         break;
1311                 /* device is active */
1312                 dprint("%s: portreset [%s]: mode %d; status %04ux\n",
1313                         name, diskstates[d->state], d->mode, s);
1314                 d->portm.flag |= Ferror;
1315                 clearci(d->port);
1316                 wakeup(&d->portm);
1317                 if((s & 7) == 0){       /* no device */
1318                         d->state = Dmissing;
1319                         break;
1320                 }
1321                 iunlock(d);
1322                 doportreset(d);
1323                 ilock(d);
1324                 break;
1325         }
1326         statechange(d);
1327         iunlock(d);
1328 }
1329
1330 static void
1331 satakproc(void*)
1332 {
1333         int i;
1334
1335         for(;;){
1336                 tsleep(&up->sleep, return0, 0, Nms);
1337                 for(i = 0; i < niadrive; i++)
1338                         checkdrive(iadrive[i], i);
1339         }
1340 }
1341
1342 static void
1343 iainterrupt(Ureg*, void *a)
1344 {
1345         int i;
1346         ulong cause, m;
1347         Ctlr *c;
1348         Drive *d;
1349
1350         c = a;
1351         ilock(c);
1352         cause = c->hba->isr;
1353         for(i = 0; i < c->mport; i++){
1354                 m = 1 << i;
1355                 if((cause & m) == 0)
1356                         continue;
1357                 d = c->rawdrive + i;
1358                 ilock(d);
1359                 if(d->port->isr && c->hba->pi & m)
1360                         updatedrive(d);
1361                 c->hba->isr = m;
1362                 iunlock(d);
1363         }
1364         iunlock(c);
1365 }
1366
1367 static int
1368 iaverify(SDunit *u)
1369 {
1370         Ctlr *c;
1371         Drive *d;
1372
1373         c = u->dev->ctlr;
1374         d = c->drive[u->subno];
1375         ilock(c);
1376         ilock(d);
1377         d->unit = u;
1378         iunlock(d);
1379         iunlock(c);
1380         checkdrive(d, d->driveno);              /* c->d0 + d->driveno */
1381         return 1;
1382 }
1383
1384 static int
1385 iaenable(SDev *s)
1386 {
1387         char name[32];
1388         Ctlr *c;
1389         static int once;
1390
1391         c = s->ctlr;
1392         ilock(c);
1393         if(!c->enabled) {
1394                 if(once == 0) {
1395                         once = 1;
1396                         kproc("iasata", satakproc, 0);
1397                 }
1398                 if(c->ndrive == 0)
1399                         panic("iaenable: zero s->ctlr->ndrive");
1400                 pcisetbme(c->pci);
1401                 snprint(name, sizeof name, "%s (%s)", s->name, s->ifc->name);
1402                 intrenable(c->pci->intl, iainterrupt, c, c->pci->tbdf, name);
1403                 /* supposed to squelch leftover interrupts here. */
1404                 ahcienable(c->hba);
1405                 c->enabled = 1;
1406         }
1407         iunlock(c);
1408         return 1;
1409 }
1410
1411 static int
1412 iadisable(SDev *s)
1413 {
1414         char name[32];
1415         Ctlr *c;
1416
1417         c = s->ctlr;
1418         ilock(c);
1419         ahcidisable(c->hba);
1420         snprint(name, sizeof name, "%s (%s)", s->name, s->ifc->name);
1421         intrdisable(c->pci->intl, iainterrupt, c, c->pci->tbdf, name);
1422         c->enabled = 0;
1423         iunlock(c);
1424         return 1;
1425 }
1426
1427 static int
1428 iaonline(SDunit *unit)
1429 {
1430         int r;
1431         Ctlr *c;
1432         Drive *d;
1433
1434         c = unit->dev->ctlr;
1435         d = c->drive[unit->subno];
1436         r = 0;
1437
1438         if(d->portm.feat & Datapi && d->mediachange){
1439                 r = scsionline(unit);
1440                 if(r > 0)
1441                         d->mediachange = 0;
1442                 return r;
1443         }
1444
1445         ilock(d);
1446         if(d->mediachange){
1447                 r = 2;
1448                 d->mediachange = 0;
1449                 /* devsd resets this after online is called; why? */
1450                 unit->sectors = d->sectors;
1451                 unit->secsize = 512;            /* default size */
1452         } else if(d->state == Dready)
1453                 r = 1;
1454         iunlock(d);
1455         return r;
1456 }
1457
1458 /* returns locked list! */
1459 static Alist*
1460 ahcibuild(Drive *d, uchar *cmd, void *data, int n, vlong lba)
1461 {
1462         uchar *c, acmd, dir, llba;
1463         Alist *l;
1464         Actab *t;
1465         Aportm *m;
1466         Aprdt *p;
1467         static uchar tab[2][2] = { 0xc8, 0x25, 0xca, 0x35, };
1468
1469         m = &d->portm;
1470         dir = *cmd != 0x28;
1471         llba = m->feat&Dllba? 1: 0;
1472         acmd = tab[dir][llba];
1473         qlock(m);
1474         l = m->list;
1475         t = m->ctab;
1476         c = t->cfis;
1477
1478         c[0] = 0x27;
1479         c[1] = 0x80;
1480         c[2] = acmd;
1481         c[3] = 0;
1482
1483         c[4] = lba;             /* sector               lba low 7:0 */
1484         c[5] = lba >> 8;        /* cylinder low         lba mid 15:8 */
1485         c[6] = lba >> 16;       /* cylinder hi          lba hi  23:16 */
1486         c[7] = 0xa0 | 0x40;     /* obsolete device bits + lba */
1487         if(llba == 0)
1488                 c[7] |= (lba>>24) & 7;
1489
1490         c[8] = lba >> 24;       /* sector (exp)         lba     31:24 */
1491         c[9] = lba >> 32;       /* cylinder low (exp)   lba     39:32 */
1492         c[10] = lba >> 48;      /* cylinder hi (exp)    lba     48:40 */
1493         c[11] = 0;              /* features (exp); */
1494
1495         c[12] = n;              /* sector count */
1496         c[13] = n >> 8;         /* sector count (exp) */
1497         c[14] = 0;              /* r */
1498         c[15] = 0;              /* control */
1499
1500         *(ulong*)(c + 16) = 0;
1501
1502         l->flags = 1<<16 | Lpref | 0x5; /* Lpref ?? */
1503         if(dir == Write)
1504                 l->flags |= Lwrite;
1505         l->len = 0;
1506         l->ctab = PCIWADDR(t);
1507         l->ctabhi = 0;
1508
1509         p = &t->prdt;
1510         p->dba = PCIWADDR(data);
1511         p->dbahi = 0;
1512         if(d->unit == nil)
1513                 panic("ahcibuild: nil d->unit");
1514         p->count = 1<<31 | (d->unit->secsize*n - 2) | 1;
1515
1516         return l;
1517 }
1518
1519 static Alist*
1520 ahcibuildpkt(Aportm *m, SDreq *r, void *data, int n)
1521 {
1522         int fill, len;
1523         uchar *c;
1524         Alist *l;
1525         Actab *t;
1526         Aprdt *p;
1527
1528         qlock(m);
1529         l = m->list;
1530         t = m->ctab;
1531         c = t->cfis;
1532
1533         fill = m->feat&Datapi16? 16: 12;
1534         if((len = r->clen) > fill)
1535                 len = fill;
1536         memmove(t->atapi, r->cmd, len);
1537         memset(t->atapi+len, 0, fill-len);
1538
1539         c[0] = 0x27;
1540         c[1] = 0x80;
1541         c[2] = 0xa0;
1542         if(n != 0)
1543                 c[3] = 1;       /* dma */
1544         else
1545                 c[3] = 0;       /* features (exp); */
1546
1547         c[4] = 0;               /* sector               lba low 7:0 */
1548         c[5] = n;               /* cylinder low         lba mid 15:8 */
1549         c[6] = n >> 8;          /* cylinder hi          lba hi  23:16 */
1550         c[7] = 0xa0;            /* obsolete device bits */
1551
1552         *(ulong*)(c + 8) = 0;
1553         *(ulong*)(c + 12) = 0;
1554         *(ulong*)(c + 16) = 0;
1555
1556         l->flags = 1<<16 | Lpref | Latapi | 0x5;
1557         if(r->write != 0 && data)
1558                 l->flags |= Lwrite;
1559         l->len = 0;
1560         l->ctab = PCIWADDR(t);
1561         l->ctabhi = 0;
1562
1563         if(data == 0)
1564                 return l;
1565
1566         p = &t->prdt;
1567         p->dba = PCIWADDR(data);
1568         p->dbahi = 0;
1569         p->count = 1<<31 | (n - 2) | 1;
1570
1571         return l;
1572 }
1573
1574 static int
1575 waitready(Drive *d)
1576 {
1577         ulong s, i, δ;
1578
1579         for(i = 0; i < 15000; i += 250){
1580                 if(d->state == Dreset || d->state == Dportreset ||
1581                     d->state == Dnew)
1582                         return 1;
1583                 δ = MACHP(0)->ticks - d->lastseen;
1584                 if(d->state == Dnull || δ > 10*1000)
1585                         return -1;
1586                 ilock(d);
1587                 s = d->port->sstatus;
1588                 iunlock(d);
1589                 if((s & 0x700) == 0 && δ > 1500)
1590                         return -1;      /* no detect */
1591                 if(d->state == Dready && (s & 7) == 3)
1592                         return 0;       /* ready, present & phy. comm. */
1593                 esleep(250);
1594         }
1595         print("%s: not responding; offline\n", d->unit->name);
1596         ilock(d);
1597         d->state = Doffline;
1598         iunlock(d);
1599         return -1;
1600 }
1601
1602 static int
1603 lockready(Drive *d)
1604 {
1605         int i;
1606
1607         qlock(&d->portm);
1608         while ((i = waitready(d)) == 1) {
1609                 qunlock(&d->portm);
1610                 esleep(1);
1611                 qlock(&d->portm);
1612         }
1613         return i;
1614 }
1615
1616 static int
1617 flushcache(Drive *d)
1618 {
1619         int i;
1620
1621         i = -1;
1622         if(lockready(d) == 0)
1623                 i = ahciflushcache(&d->portc);
1624         qunlock(&d->portm);
1625         return i;
1626 }
1627
1628 static int
1629 iariopkt(SDreq *r, Drive *d)
1630 {
1631         int n, count, try, max, flag, task;
1632         char *name;
1633         uchar *cmd, *data;
1634         Aport *p;
1635         Asleep as;
1636
1637         cmd = r->cmd;
1638         name = d->unit->name;
1639         p = d->port;
1640
1641         aprint("ahci: iariopkt: %02ux %02ux %c %d %p\n",
1642                 cmd[0], cmd[2], "rw"[r->write], r->dlen, r->data);
1643         if(cmd[0] == 0x5a && (cmd[2] & 0x3f) == 0x3f)
1644                 return sdmodesense(r, cmd, d->info, sizeof d->info);
1645         r->rlen = 0;
1646         count = r->dlen;
1647         max = 65536;
1648
1649         try = 0;
1650 retry:
1651         data = r->data;
1652         n = count;
1653         if(n > max)
1654                 n = max;
1655         ahcibuildpkt(&d->portm, r, data, n);
1656         switch(waitready(d)){
1657         case -1:
1658                 qunlock(&d->portm);
1659                 return SDeio;
1660         case 1:
1661                 qunlock(&d->portm);
1662                 esleep(1);
1663                 goto retry;
1664         }
1665
1666         ilock(d);
1667         d->portm.flag = 0;
1668         iunlock(d);
1669         p->ci = 1;
1670
1671         as.p = p;
1672         as.i = 1;
1673         d->intick = MACHP(0)->ticks;
1674         d->active++;
1675
1676         while(waserror())
1677                 ;
1678         sleep(&d->portm, ahciclear, &as);
1679         poperror();
1680
1681         d->active--;
1682         ilock(d);
1683         flag = d->portm.flag;
1684         task = d->port->task;
1685         iunlock(d);
1686
1687         if(task & (Efatal<<8) || task & (ASbsy|ASdrq) && d->state == Dready){
1688                 d->port->ci = 0;
1689                 ahcirecover(&d->portc);
1690                 task = d->port->task;
1691                 flag &= ~Fdone;         /* either an error or do-over */
1692         }
1693         qunlock(&d->portm);
1694         if(flag == 0){
1695                 if(++try == 10){
1696                         print("%s: bad disk\n", name);
1697                         r->status = SDcheck;
1698                         return SDcheck;
1699                 }
1700                 print("%s: retry\n", name);
1701                 goto retry;
1702         }
1703         if(flag & Ferror){
1704                 if((task&Eidnf) == 0)
1705                         print("%s: i/o error %ux\n", name, task);
1706                 r->status = SDcheck;
1707                 return SDcheck;
1708         }
1709
1710         data += n;
1711
1712         r->rlen = data - (uchar*)r->data;
1713         r->status = SDok;
1714         return SDok;
1715 }
1716
1717 static int
1718 iario(SDreq *r)
1719 {
1720         int i, n, count, try, max, flag, task;
1721         vlong lba;
1722         char *name;
1723         uchar *cmd, *data;
1724         Aport *p;
1725         Asleep as;
1726         Ctlr *c;
1727         Drive *d;
1728         SDunit *unit;
1729
1730         unit = r->unit;
1731         c = unit->dev->ctlr;
1732         d = c->drive[unit->subno];
1733         if(d->portm.feat & Datapi)
1734                 return iariopkt(r, d);
1735         cmd = r->cmd;
1736         name = d->unit->name;
1737         p = d->port;
1738
1739         if(r->cmd[0] == 0x35 || r->cmd[0] == 0x91){
1740                 if(flushcache(d) == 0)
1741                         return sdsetsense(r, SDok, 0, 0, 0);
1742                 return sdsetsense(r, SDcheck, 3, 0xc, 2);
1743         }
1744
1745         if((i = sdfakescsi(r, d->info, sizeof d->info)) != SDnostatus){
1746                 r->status = i;
1747                 return i;
1748         }
1749
1750         if(*cmd != 0x28 && *cmd != 0x2a){
1751                 print("%s: bad cmd 0x%.2ux\n", name, cmd[0]);
1752                 r->status = SDcheck;
1753                 return SDcheck;
1754         }
1755
1756         lba   = cmd[2]<<24 | cmd[3]<<16 | cmd[4]<<8 | cmd[5];
1757         count = cmd[7]<<8 | cmd[8];
1758         if(r->data == nil)
1759                 return SDok;
1760         if(r->dlen < count * unit->secsize)
1761                 count = r->dlen / unit->secsize;
1762         max = 128;
1763
1764         try = 0;
1765 retry:
1766         data = r->data;
1767         while(count > 0){
1768                 n = count;
1769                 if(n > max)
1770                         n = max;
1771                 ahcibuild(d, cmd, data, n, lba);
1772                 switch(waitready(d)){
1773                 case -1:
1774                         qunlock(&d->portm);
1775                         return SDeio;
1776                 case 1:
1777                         qunlock(&d->portm);
1778                         esleep(1);
1779                         goto retry;
1780                 }
1781                 ilock(d);
1782                 d->portm.flag = 0;
1783                 iunlock(d);
1784                 p->ci = 1;
1785
1786                 as.p = p;
1787                 as.i = 1;
1788                 d->intick = MACHP(0)->ticks;
1789                 d->active++;
1790
1791                 while(waserror())
1792                         ;
1793                 sleep(&d->portm, ahciclear, &as);
1794                 poperror();
1795
1796                 d->active--;
1797                 ilock(d);
1798                 flag = d->portm.flag;
1799                 task = d->port->task;
1800                 iunlock(d);
1801
1802                 if(task & (Efatal<<8) ||
1803                     task & (ASbsy|ASdrq) && d->state == Dready){
1804                         d->port->ci = 0;
1805                         ahcirecover(&d->portc);
1806                         task = d->port->task;
1807                 }
1808                 qunlock(&d->portm);
1809                 if(flag == 0){
1810                         if(++try == 10){
1811                                 print("%s: bad disk\n", name);
1812                                 r->status = SDeio;
1813                                 return SDeio;
1814                         }
1815                         iprint("%s: retry %lld\n", name, lba);
1816                         goto retry;
1817                 }
1818                 if(flag & Ferror){
1819                         iprint("%s: i/o error %ux @%,lld\n", name, task, lba);
1820                         r->status = SDeio;
1821                         return SDeio;
1822                 }
1823
1824                 count -= n;
1825                 lba   += n;
1826                 data += n * unit->secsize;
1827         }
1828         r->rlen = data - (uchar*)r->data;
1829         r->status = SDok;
1830         return SDok;
1831 }
1832
1833 /*
1834  * configure drives 0-5 as ahci sata  (c.f. errata)
1835  */
1836 static int
1837 iaahcimode(Pcidev *p)
1838 {
1839         dprint("iaahcimode: %ux %ux %ux\n", pcicfgr8(p, 0x91), pcicfgr8(p, 92),
1840                 pcicfgr8(p, 93));
1841         pcicfgw16(p, 0x92, pcicfgr32(p, 0x92) | 0xf);   /* ports 0-3 */
1842 //      pcicfgw8(p, 0x93, pcicfgr32(p, 9x93) | 3);      /* ports 4-5 */
1843         return 0;
1844 }
1845
1846 static void
1847 iasetupahci(Ctlr *c)
1848 {
1849         /* disable cmd block decoding. */
1850         pcicfgw16(c->pci, 0x40, pcicfgr16(c->pci, 0x40) & ~(1<<15));
1851         pcicfgw16(c->pci, 0x42, pcicfgr16(c->pci, 0x42) & ~(1<<15));
1852
1853         c->lmmio[0x4/4] |= 1 << 31;     /* enable ahci mode (ghc register) */
1854         c->lmmio[0xc/4] = (1 << 6) - 1; /* 5 ports. (supposedly ro pi reg.) */
1855
1856         /* enable ahci mode; from ich9 datasheet */
1857         pcicfgw16(c->pci, 0x90, 1<<6 | 1<<5);
1858 }
1859
1860 static int
1861 didtype(Pcidev *p)
1862 {
1863         switch(p->vid){
1864         case 0x8086:
1865                 if((p->did & 0xfffc) == 0x2680)
1866                         return Tesb;
1867                 /*
1868                  * 0x27c4 is the intel 82801 in compatibility (not sata) mode.
1869                  */
1870                 if ((p->did & 0xfffb) == 0x27c1 ||      /* 82801g[bh]m ich7 */
1871                     p->did == 0x2821 ||                 /* 82801h[roh] */
1872                     (p->did & 0xfffe) == 0x2824 ||      /* 82801h[b] */
1873                     (p->did & 0xfeff) == 0x2829 ||      /* ich8/9m */
1874                     (p->did & 0xfffe) == 0x2922 ||      /* ich9 */
1875                     p->did == 0x3a02 ||                 /* 82801jd/do */
1876                     (p->did & 0xfefe) == 0x3a22 ||      /* ich10, pch */
1877                     (p->did & 0xfff7) == 0x3b28)        /* pchm */
1878                         return Tich;
1879                 break;
1880         case 0x1002:
1881                 if(p->did == 0x4380)
1882                         return Tsb600;
1883                 break;
1884         }
1885         if(p->ccrb == Pcibcstore && p->ccru == 6 && p->ccrp == 1)
1886                 return Tunk;
1887         return -1;
1888 }
1889
1890 static SDev*
1891 iapnp(void)
1892 {
1893         int i, n, nunit, type;
1894         ulong io;
1895         Ctlr *c;
1896         Drive *d;
1897         Pcidev *p;
1898         SDev *head, *tail, *s;
1899         static int done;
1900
1901         if(done++)
1902                 return nil;
1903
1904         memset(olds, 0xff, sizeof olds);
1905         p = nil;
1906         head = tail = nil;
1907 loop:
1908         while((p = pcimatch(p, 0, 0)) != nil){
1909                 type = didtype(p);
1910                 if (type == -1 || p->mem[Abar].bar == 0)
1911                         continue;
1912                 if(niactlr == NCtlr){
1913                         print("ahci: iapnp: %s: too many controllers\n",
1914                                 tname[type]);
1915                         break;
1916                 }
1917                 c = iactlr + niactlr;
1918                 s = sdevs  + niactlr;
1919                 memset(c, 0, sizeof *c);
1920                 memset(s, 0, sizeof *s);
1921                 io = p->mem[Abar].bar & ~0xf;
1922                 c->mmio = vmap(io, p->mem[Abar].size);
1923                 if(c->mmio == 0){
1924                         print("ahci: %s: address 0x%luX in use did=%x\n",
1925                                 Tname(c), io, p->did);
1926                         continue;
1927                 }
1928                 c->lmmio = (ulong*)c->mmio;
1929                 c->pci = p;
1930                 c->type = type;
1931
1932                 s->ifc = &sdiahciifc;
1933                 s->idno = 'E' + niactlr;
1934                 s->ctlr = c;
1935                 c->sdev = s;
1936
1937                 if(Intel(c) && p->did != 0x2681)
1938                         iasetupahci(c);
1939                 nunit = ahciconf(c);
1940 //              ahcihbareset((Ahba*)c->mmio);
1941                 if(Intel(c) && iaahcimode(p) == -1)
1942                         break;
1943                 if(nunit < 1){
1944                         vunmap(c->mmio, p->mem[Abar].size);
1945                         continue;
1946                 }
1947                 c->ndrive = s->nunit = nunit;
1948                 c->mport = c->hba->cap & ((1<<5)-1);
1949
1950                 i = (c->hba->cap >> 21) & 1;
1951                 print("#S/sd%c: %s: sata-%s with %d ports\n", s->idno,
1952                         Tname(c), "I\0II" + i*2, nunit);
1953
1954                 /* map the drives -- they don't all need to be enabled. */
1955                 memset(c->rawdrive, 0, sizeof c->rawdrive);
1956                 n = 0;
1957                 for(i = 0; i < NCtlrdrv; i++) {
1958                         d = c->rawdrive + i;
1959                         d->portno = i;
1960                         d->driveno = -1;
1961                         d->sectors = 0;
1962                         d->serial[0] = ' ';
1963                         d->ctlr = c;
1964                         if((c->hba->pi & (1<<i)) == 0)
1965                                 continue;
1966                         d->port = (Aport*)(c->mmio + 0x80*i + 0x100);
1967                         d->portc.p = d->port;
1968                         d->portc.m = &d->portm;
1969                         d->driveno = n++;
1970                         c->drive[d->driveno] = d;
1971                         iadrive[niadrive + d->driveno] = d;
1972                 }
1973                 for(i = 0; i < n; i++)
1974                         if(ahciidle(c->drive[i]->port) == -1){
1975                                 dprint("ahci: %s: port %d wedged; abort\n",
1976                                         Tname(c), i);
1977                                 goto loop;
1978                         }
1979                 for(i = 0; i < n; i++){
1980                         c->drive[i]->mode = DMsatai;
1981                         configdrive(c->drive[i]);
1982                 }
1983
1984                 niadrive += n;
1985                 niactlr++;
1986                 if(head)
1987                         tail->next = s;
1988                 else
1989                         head = s;
1990                 tail = s;
1991         }
1992         return head;
1993 }
1994
1995 static char* smarttab[] = {
1996         "unset",
1997         "error",
1998         "threshold exceeded",
1999         "normal"
2000 };
2001
2002 static char *
2003 pflag(char *s, char *e, uchar f)
2004 {
2005         uchar i;
2006
2007         for(i = 0; i < 8; i++)
2008                 if(f & (1 << i))
2009                         s = seprint(s, e, "%s ", flagname[i]);
2010         return seprint(s, e, "\n");
2011 }
2012
2013 static int
2014 iarctl(SDunit *u, char *p, int l)
2015 {
2016         char buf[32];
2017         char *e, *op;
2018         Aport *o;
2019         Ctlr *c;
2020         Drive *d;
2021
2022         c = u->dev->ctlr;
2023         if(c == nil) {
2024 print("iarctl: nil u->dev->ctlr\n");
2025                 return 0;
2026         }
2027         d = c->drive[u->subno];
2028         o = d->port;
2029
2030         e = p+l;
2031         op = p;
2032         if(d->state == Dready){
2033                 p = seprint(p, e, "model\t%s\n", d->model);
2034                 p = seprint(p, e, "serial\t%s\n", d->serial);
2035                 p = seprint(p, e, "firm\t%s\n", d->firmware);
2036                 if(d->smartrs == 0xff)
2037                         p = seprint(p, e, "smart\tenable error\n");
2038                 else if(d->smartrs == 0)
2039                         p = seprint(p, e, "smart\tdisabled\n");
2040                 else
2041                         p = seprint(p, e, "smart\t%s\n",
2042                                 smarttab[d->portm.smart]);
2043                 p = seprint(p, e, "flag\t");
2044                 p = pflag(p, e, d->portm.feat);
2045         }else
2046                 p = seprint(p, e, "no disk present [%s]\n", diskstates[d->state]);
2047         serrstr(o->serror, buf, buf + sizeof buf - 1);
2048         p = seprint(p, e, "reg\ttask %lux cmd %lux serr %lux %s ci %lux is %lux; "
2049                 "sig %lux sstatus %04lux\n", o->task, o->cmd, o->serror, buf,
2050                 o->ci, o->isr, o->sig, o->sstatus);
2051         if(d->unit == nil)
2052                 panic("iarctl: nil d->unit");
2053         p = seprint(p, e, "geometry %llud %lud\n", d->sectors, d->unit->secsize);
2054         return p - op;
2055 }
2056
2057 static void
2058 runflushcache(Drive *d)
2059 {
2060         long t0;
2061
2062         t0 = MACHP(0)->ticks;
2063         if(flushcache(d) != 0)
2064                 error(Eio);
2065         dprint("ahci: flush in %ld ms\n", MACHP(0)->ticks - t0);
2066 }
2067
2068 static void
2069 forcemode(Drive *d, char *mode)
2070 {
2071         int i;
2072
2073         for(i = 0; i < nelem(modename); i++)
2074                 if(strcmp(mode, modename[i]) == 0)
2075                         break;
2076         if(i == nelem(modename))
2077                 i = 0;
2078         ilock(d);
2079         d->mode = i;
2080         iunlock(d);
2081 }
2082
2083 static void
2084 runsmartable(Drive *d, int i)
2085 {
2086         if(waserror()){
2087                 qunlock(&d->portm);
2088                 d->smartrs = 0;
2089                 nexterror();
2090         }
2091         if(lockready(d) == -1)
2092                 error(Eio);
2093         d->smartrs = smart(&d->portc, i);
2094         d->portm.smart = 0;
2095         qunlock(&d->portm);
2096         poperror();
2097 }
2098
2099 static void
2100 forcestate(Drive *d, char *state)
2101 {
2102         int i;
2103
2104         for(i = 0; i < nelem(diskstates); i++)
2105                 if(strcmp(state, diskstates[i]) == 0)
2106                         break;
2107         if(i == nelem(diskstates))
2108                 error(Ebadctl);
2109         ilock(d);
2110         d->state = i;
2111         iunlock(d);
2112 }
2113
2114
2115 static int
2116 iawctl(SDunit *u, Cmdbuf *cmd)
2117 {
2118         char **f;
2119         Ctlr *c;
2120         Drive *d;
2121         uint i;
2122
2123         c = u->dev->ctlr;
2124         d = c->drive[u->subno];
2125         f = cmd->f;
2126
2127         if(strcmp(f[0], "flushcache") == 0)
2128                 runflushcache(d);
2129         else if(strcmp(f[0], "identify") ==  0){
2130                 i = strtoul(f[1]? f[1]: "0", 0, 0);
2131                 if(i > 0xff)
2132                         i = 0;
2133                 dprint("ahci: %04d %ux\n", i, d->info[i]);
2134         }else if(strcmp(f[0], "mode") == 0)
2135                 forcemode(d, f[1]? f[1]: "satai");
2136         else if(strcmp(f[0], "nop") == 0){
2137                 if((d->portm.feat & Dnop) == 0){
2138                         cmderror(cmd, "no drive support");
2139                         return -1;
2140                 }
2141                 if(waserror()){
2142                         qunlock(&d->portm);
2143                         nexterror();
2144                 }
2145                 if(lockready(d) == -1)
2146                         error(Eio);
2147                 nop(&d->portc);
2148                 qunlock(&d->portm);
2149                 poperror();
2150         }else if(strcmp(f[0], "reset") == 0)
2151                 forcestate(d, "reset");
2152         else if(strcmp(f[0], "smart") == 0){
2153                 if(d->smartrs == 0){
2154                         cmderror(cmd, "smart not enabled");
2155                         return -1;
2156                 }
2157                 if(waserror()){
2158                         qunlock(&d->portm);
2159                         d->smartrs = 0;
2160                         nexterror();
2161                 }
2162                 if(lockready(d) == -1)
2163                         error(Eio);
2164                 d->portm.smart = 2 + smartrs(&d->portc);
2165                 qunlock(&d->portm);
2166                 poperror();
2167         }else if(strcmp(f[0], "smartdisable") == 0)
2168                 runsmartable(d, 1);
2169         else if(strcmp(f[0], "smartenable") == 0)
2170                 runsmartable(d, 0);
2171         else if(strcmp(f[0], "state") == 0)
2172                 forcestate(d, f[1]? f[1]: "null");
2173         else{
2174                 cmderror(cmd, Ebadctl);
2175                 return -1;
2176         }
2177         return 0;
2178 }
2179
2180 static char *
2181 portr(char *p, char *e, uint x)
2182 {
2183         int i, a;
2184
2185         p[0] = 0;
2186         a = -1;
2187         for(i = 0; i < 32; i++){
2188                 if((x & (1<<i)) == 0){
2189                         if(a != -1 && i - 1 != a)
2190                                 p = seprint(p, e, "-%d", i - 1);
2191                         a = -1;
2192                         continue;
2193                 }
2194                 if(a == -1){
2195                         if(i > 0)
2196                                 p = seprint(p, e, ", ");
2197                         p = seprint(p, e, "%d", a = i);
2198                 }
2199         }
2200         if(a != -1 && i - 1 != a)
2201                 p = seprint(p, e, "-%d", i - 1);
2202         return p;
2203 }
2204
2205 /* must emit exactly one line per controller (sd(3)) */
2206 static char*
2207 iartopctl(SDev *sdev, char *p, char *e)
2208 {
2209         ulong cap;
2210         char pr[25];
2211         Ahba *hba;
2212         Ctlr *ctlr;
2213
2214 #define has(x, str) if(cap & (x)) p = seprint(p, e, "%s ", (str))
2215
2216         ctlr = sdev->ctlr;
2217         hba = ctlr->hba;
2218         p = seprint(p, e, "sd%c ahci port %#p: ", sdev->idno, hba);
2219         cap = hba->cap;
2220         has(Hs64a, "64a");
2221         has(Hsalp, "alp");
2222         has(Hsam, "am");
2223         has(Hsclo, "clo");
2224         has(Hcccs, "coal");
2225         has(Hems, "ems");
2226         has(Hsal, "led");
2227         has(Hsmps, "mps");
2228         has(Hsncq, "ncq");
2229         has(Hssntf, "ntf");
2230         has(Hspm, "pm");
2231         has(Hpsc, "pslum");
2232         has(Hssc, "slum");
2233         has(Hsss, "ss");
2234         has(Hsxs, "sxs");
2235         portr(pr, pr + sizeof pr, hba->pi);
2236         return seprint(p, e,
2237                 "iss %ld ncs %ld np %ld; ghc %lux isr %lux pi %lux %s ver %lux\n",
2238                 (cap>>20) & 0xf, (cap>>8) & 0x1f, 1 + (cap & 0x1f),
2239                 hba->ghc, hba->isr, hba->pi, pr, hba->ver);
2240 #undef has
2241 }
2242
2243 static int
2244 iawtopctl(SDev *, Cmdbuf *cmd)
2245 {
2246         int *v;
2247         char **f;
2248
2249         f = cmd->f;
2250         v = 0;
2251
2252         if (f[0] == nil)
2253                 return 0;
2254         if(strcmp(f[0], "debug") == 0)
2255                 v = &debug;
2256         else if(strcmp(f[0], "idprint") == 0)
2257                 v = &prid;
2258         else if(strcmp(f[0], "aprint") == 0)
2259                 v = &datapi;
2260         else
2261                 cmderror(cmd, Ebadctl);
2262
2263         switch(cmd->nf){
2264         default:
2265                 cmderror(cmd, Ebadarg);
2266         case 1:
2267                 *v ^= 1;
2268                 break;
2269         case 2:
2270                 if(f[1])
2271                         *v = strcmp(f[1], "on") == 0;
2272                 else
2273                         *v ^= 1;
2274                 break;
2275         }
2276         return 0;
2277 }
2278
2279 SDifc sdiahciifc = {
2280         "iahci",
2281
2282         iapnp,
2283         nil,            /* legacy */
2284         iaenable,
2285         iadisable,
2286
2287         iaverify,
2288         iaonline,
2289         iario,
2290         iarctl,
2291         iawctl,
2292
2293         scsibio,
2294         nil,            /* probe */
2295         nil,            /* clear */
2296         iartopctl,
2297         iawtopctl,
2298 };