sys/src/9/pc/sdnvme.c

   1 #include "u.h"
   2 #include "../port/lib.h"
   3 #include "mem.h"
   4 #include "dat.h"
   5 #include "fns.h"
   6 #include "io.h"
   7 #include "../port/pci.h"
   8 #include "ureg.h"
   9 #include "../port/error.h"
  10
  11 #include "../port/sd.h"
  12
  13 typedef struct WS WS;
  14 typedef struct CQ CQ;
  15 typedef struct SQ SQ;
  16 typedef struct Ctlr Ctlr;
  17
  18 struct WS
  19 {
  20         u32int  cdw0;
  21         ushort  status;
  22         Rendez  *sleep;
  23         WS      **link;
  24         SQ      *queue;
  25 };
  26
  27 struct CQ
  28 {
  29         u32int  head;
  30         u32int  mask;
  31         u32int  shift;
  32         u32int  *base;
  33         Ctlr    *ctlr;
  34 };
  35
  36 struct SQ
  37 {
  38         u32int  tail;
  39         u32int  mask;
  40         u32int  shift;
  41         u32int  *base;
  42         WS      **wait;
  43         Ctlr    *ctlr;
  44         Lock;
  45 };
  46
  47 struct Ctlr
  48 {
  49         QLock;
  50
  51         Lock    intr;
  52         u32int  ints;
  53         u32int  irqc[2];
  54
  55         Pcidev  *pci;
  56         u32int  *reg;
  57
  58         u64int  cap;
  59         uchar   *ident;
  60         u32int  *nsid;
  61         int     nnsid;
  62
  63         u32int  mps;            /* mps = 1<<mpsshift */
  64         u32int  mpsshift;
  65         u32int  dstrd;
  66
  67         u32int  nsq;
  68
  69         CQ      cq[1+1];
  70         SQ      sq[1+MAXMACH];
  71
  72         Ctlr    *next;
  73 };
  74
  75 /* controller registers */
  76 enum {
  77         Cap0,
  78         Cap1,
  79         Ver,
  80         IntMs,
  81         IntMc,
  82         CCfg,
  83
  84         CSts = 0x1C/4,
  85         Nssr,
  86         AQAttr,
  87         ASQBase0,
  88         ASQBase1,
  89         ACQBase0,
  90         ACQBase1,
  91
  92         DBell = 0x1000/4,
  93 };
  94
  95 static u32int*
  96 qcmd(WS *ws, Ctlr *ctlr, int adm, u32int opc, u32int nsid, void *mptr, void *data, ulong len)
  97 {
  98         u32int cid, *e;
  99         u64int pa;
 100         SQ *sq;
 101
 102         if(!adm){
 103         Retry:
 104                 splhi();
 105                 sq = &ctlr->sq[1+(m->machno % ctlr->nsq)];
 106                 if(conf.nmach > ctlr->nsq)
 107                         lock(sq);
 108         } else {
 109                 qlock(ctlr);
 110                 sq = &ctlr->sq[0];
 111         }
 112         ws->sleep = &up->sleep;
 113         ws->queue = sq;
 114         ws->link = &sq->wait[sq->tail & sq->mask];
 115         while(*ws->link != nil){
 116                 sched();
 117                 if(!adm){
 118                         /* should be very rare */
 119                         goto Retry;
 120                 }
 121         }
 122         *ws->link = ws;
 123
 124         e = &sq->base[((cid = sq->tail++) & sq->mask)<<4];
 125         e[0] = opc | cid<<16;
 126         e[1] = nsid;
 127         e[2] = 0;
 128         e[3] = 0;
 129         if(mptr != nil){
 130                 pa = PCIWADDR(mptr);
 131                 e[4] = pa;
 132                 e[5] = pa>>32;
 133         } else {
 134                 e[4] = 0;
 135                 e[5] = 0;
 136         }
 137         if(len > 0){
 138                 pa = PCIWADDR(data);
 139                 e[6] = pa;
 140                 e[7] = pa>>32;
 141                 if(len > ctlr->mps - (pa & ctlr->mps-1))
 142                         pa += ctlr->mps - (pa & ctlr->mps-1);
 143                 else
 144                         pa = 0;
 145         } else {
 146                 e[6] = 0;
 147                 e[7] = 0;
 148                 pa = 0;
 149         }
 150         e[8] = pa;
 151         e[9] = pa>>32;
 152         return e;
 153 }
 154
 155 static void
 156 nvmeintr(Ureg *, void *arg)
 157 {
 158         u32int phaseshift, *e;
 159         WS *ws, **wp;
 160         Ctlr *ctlr;
 161         SQ *sq;
 162         CQ *cq;
 163
 164         ctlr = arg;
 165         if(ctlr->ints == 0)
 166                 return;
 167
 168         ilock(&ctlr->intr);
 169         ctlr->reg[IntMs] = ctlr->ints;
 170         for(cq = &ctlr->cq[nelem(ctlr->cq)-1]; cq >= ctlr->cq; cq--){
 171                 if(cq->base == nil)
 172                         continue;
 173                 phaseshift = 16 - cq->shift;
 174                 for(;;){
 175                         e = &cq->base[(cq->head & cq->mask)<<2];
 176                         if(((e[3] ^ (cq->head << phaseshift)) & 0x10000) == 0)
 177                                 break;
 178
 179                         if(0) iprint("nvmeintr: cq%d [%.4ux] %.8ux %.8ux %.8ux %.8ux\n",
 180                                 (int)(cq - ctlr->cq), cq->head & cq->mask,
 181                                 e[0], e[1], e[2], e[3]);
 182
 183                         sq = &ctlr->sq[e[2] >> 16];
 184                         wp = &sq->wait[e[3] & sq->mask];
 185                         if((ws = *wp) != nil && ws->link == wp){
 186                                 Rendez *z = ws->sleep;
 187                                 ws->cdw0 = e[0];
 188                                 ws->status = e[3]>>17;
 189                                 *wp = nil;
 190                                 wakeup(z);
 191                         }
 192                         ctlr->reg[DBell + ((cq-ctlr->cq)*2+1 << ctlr->dstrd)] = ++cq->head & cq->mask;
 193                 }
 194         }
 195         ctlr->reg[IntMc] = ctlr->ints;
 196         iunlock(&ctlr->intr);
 197 }
 198
 199 static int
 200 wdone(void *arg)
 201 {
 202         WS *ws = arg;
 203         return *ws->link != ws;
 204 }
 205
 206 static u32int
 207 wcmd(WS *ws)
 208 {
 209         SQ *sq = ws->queue;
 210         Ctlr *ctlr = sq->ctlr;
 211
 212         coherence();
 213         ctlr->reg[DBell + ((sq-ctlr->sq)*2+0 << ctlr->dstrd)] = sq->tail & sq->mask;
 214         if(sq > ctlr->sq) {
 215                 assert(sq == &ctlr->sq[1+(m->machno % ctlr->nsq)]);
 216                 if(conf.nmach > ctlr->nsq)
 217                         unlock(sq);
 218                 spllo();
 219         } else
 220                 qunlock(sq->ctlr);
 221         while(waserror())
 222                 ;
 223         tsleep(ws->sleep, wdone, ws, 5);
 224         while(!wdone(ws)){
 225                 nvmeintr(nil, ctlr);
 226                 tsleep(ws->sleep, wdone, ws, 10);
 227         }
 228         poperror();
 229         return ws->status;
 230 }
 231
 232 void
 233 checkstatus(u32int status, char *info)
 234 {
 235         if(status == 0)
 236                 return;
 237         snprint(up->genbuf, sizeof(up->genbuf), "%s: status %ux", info, status);
 238         error(up->genbuf);
 239 }
 240
 241 static long
 242 nvmebio(SDunit *u, int lun, int write, void *a, long count, uvlong lba)
 243 {
 244         u32int nsid, s, n, m, *e;
 245         Ctlr *ctlr;
 246         uchar *p;
 247         WS ws;
 248
 249         USED(lun);
 250
 251         ctlr = u->dev->ctlr;
 252         nsid = ctlr->nsid[u->subno];
 253         s = u->secsize;
 254         p = a;
 255         while(count > 0){
 256                 m = (2*ctlr->mps - ((uintptr)p & ctlr->mps-1)) / s;
 257                 if((n = count) > m)
 258                         n = m;
 259                 e = qcmd(&ws, ctlr, 0, write ? 0x01 : 0x02, nsid, nil, p, n*s);
 260                 e[10] = lba;
 261                 e[11] = lba>>32;
 262                 e[12] = n-1;
 263                 e[13] = (count>n)<<6;   /* sequential request */
 264                 e[14] = 0;
 265                 e[15] = 0;
 266                 checkstatus(wcmd(&ws), write ? "write" : "read");
 267                 p += n*s;
 268                 count -= n;
 269                 lba += n;
 270         }
 271         return p - (uchar*)a;
 272 }
 273
 274 static int
 275 nvmerio(SDreq *r)
 276 {
 277         int i, count, rw;
 278         uvlong lba;
 279         SDunit *u;
 280
 281         u = r->unit;
 282         if(r->cmd[0] == 0x35 || r->cmd[0] == 0x91)
 283                 return sdsetsense(r, SDok, 0, 0, 0);
 284         if((i = sdfakescsi(r)) != SDnostatus)
 285                 return r->status = i;
 286         if((i = sdfakescsirw(r, &lba, &count, &rw)) != SDnostatus)
 287                 return i;
 288         r->rlen = nvmebio(u, r->lun, rw == SDwrite, r->data, count, lba);
 289         return r->status = SDok;
 290 }
 291
 292 static int
 293 nvmeverify(SDunit *u)
 294 {
 295         Ctlr *ctlr = u->dev->ctlr;
 296         return u->subno < ctlr->nnsid;
 297 }
 298
 299 static int
 300 nvmeonline(SDunit *u)
 301 {
 302         u32int *e, lbaf;
 303         uchar *info, *p;
 304         Ctlr *ctlr;
 305         WS ws;
 306
 307         if(u->sectors != 0)
 308                 return 1;
 309
 310         ctlr = u->dev->ctlr;
 311         if((info = mallocalign(0x1000, ctlr->mps, 0, 0)) == nil)
 312                 return 0;
 313
 314         e = qcmd(&ws, ctlr, 1, 0x06, ctlr->nsid[u->subno], nil, info, 0x1000);
 315         e[10] = 0; // identify namespace
 316         if(wcmd(&ws) != 0){
 317                 free(info);
 318                 return 0;
 319         }
 320         p = info;
 321         u->sectors = p[0] | p[1]<<8 | p[2]<<16 | p[3]<<24
 322                 | (u64int)p[4]<<32
 323                 | (u64int)p[5]<<40
 324                 | (u64int)p[6]<<48
 325                 | (u64int)p[7]<<56;
 326         p = &info[128 + 4*(info[26]&15)];
 327         lbaf = p[0] | p[1]<<8 | p[2]<<16 | p[3]<<24;
 328         u->secsize = 1<<((lbaf>>16)&0xFF);
 329         free(info);
 330
 331         memset(u->inquiry, 0, sizeof u->inquiry);
 332         u->inquiry[2] = 2;
 333         u->inquiry[3] = 2;
 334         u->inquiry[4] = sizeof u->inquiry - 4;
 335         memmove(u->inquiry+8, ctlr->ident+24, 20);
 336
 337         return 2;
 338 }
 339
 340 static int
 341 nvmerctl(SDunit *u, char *p, int l)
 342 {
 343         Ctlr *ctlr;
 344         char *e, *s;
 345
 346         if((ctlr = u->dev->ctlr) == nil || ctlr->ident == nil)
 347                 return 0;
 348
 349         e = p+l;
 350         s = p;
 351
 352         p = seprint(p, e, "model\t%.20s\n", (char*)ctlr->ident+24);
 353         p = seprint(p, e, "serial\t%.10s\n", (char*)ctlr->ident+4);
 354         p = seprint(p, e, "firm\t%.6s\n", (char*)ctlr->ident+64);
 355         p = seprint(p, e, "geometry %llud %lud\n", u->sectors, u->secsize);
 356
 357         return p-s;
 358 }
 359
 360 static void*
 361 cqalloc(Ctlr *ctlr, CQ *cq, u32int lgsize)
 362 {
 363         cq->ctlr = ctlr;
 364         cq->head = 0;
 365         cq->shift = lgsize-4;
 366         cq->mask = (1<<cq->shift)-1;
 367         if((cq->base = mallocalign(1<<lgsize, ctlr->mps, 0, 0)) == nil)
 368                 error(Enomem);
 369         memset(cq->base, 0, 1<<lgsize);
 370         return cq->base;
 371 }
 372
 373 static void*
 374 sqalloc(Ctlr *ctlr, SQ *sq, u32int lgsize)
 375 {
 376         sq->ctlr = ctlr;
 377         sq->tail = 0;
 378         sq->shift = lgsize-6;
 379         sq->mask = (1<<sq->shift)-1;
 380         if((sq->base = mallocalign(1<<lgsize, ctlr->mps, 0, 0)) == nil)
 381                 error(Enomem);
 382         if((sq->wait = mallocz(sizeof(WS*)*(sq->mask+1), 1)) == nil)
 383                 error(Enomem);
 384         memset(sq->base, 0, 1<<lgsize);
 385         return sq->base;
 386 }
 387
 388 static void
 389 setupqueues(Ctlr *ctlr)
 390 {
 391         u32int lgsize, st, *e;
 392         CQ *cq;
 393         SQ *sq;
 394         WS ws;
 395         int i;
 396
 397         /* Overkill */
 398         lgsize = 12-6+4;
 399         while(lgsize < 16+4 && lgsize < ctlr->mpsshift && 1<<lgsize < conf.nmach<<12-6+4)
 400                 lgsize++;
 401
 402         /* CQID1: shared completion queue */
 403         cq = &ctlr->cq[1];
 404         cqalloc(ctlr, cq, lgsize);
 405         e = qcmd(&ws, ctlr, 1, 0x05, 0, nil, cq->base, 1<<lgsize);
 406         e[10] = (cq - ctlr->cq) | cq->mask<<16;
 407         e[11] = 3; /* IEN | PC */
 408         checkstatus(wcmd(&ws), "create completion queue");
 409
 410         st = 0;
 411
 412         /* SQID[1..nmach]: submission queue per cpu */
 413         for(i=1; i<=conf.nmach; i++){
 414                 sq = &ctlr->sq[i];
 415                 sqalloc(ctlr, sq, 12);
 416                 e = qcmd(&ws, ctlr, 1, 0x01, 0, nil, sq->base, 0x1000);
 417                 e[10] = i | sq->mask<<16;
 418                 e[11] = (cq - ctlr->cq)<<16 | 1;        /* CQID<<16 | PC */
 419
 420                 st = wcmd(&ws);
 421                 if(st != 0){
 422                         free(sq->base);
 423                         free(sq->wait);
 424                         memset(sq, 0, sizeof(*sq));
 425                         break;
 426                 }
 427         }
 428
 429         ctlr->nsq = i - 1;
 430         if(ctlr->nsq < 1)
 431                 checkstatus(st, "create submission queues");
 432
 433         ilock(&ctlr->intr);
 434         ctlr->ints |= 1<<(cq - ctlr->cq);
 435         ctlr->reg[IntMc] = ctlr->ints;
 436         iunlock(&ctlr->intr);
 437 }
 438
 439 static void
 440 identify(Ctlr *ctlr)
 441 {
 442         u32int *e;
 443         WS ws;
 444
 445         if(ctlr->ident == nil)
 446                 if((ctlr->ident = mallocalign(0x1000, ctlr->mps, 0, 0)) == nil)
 447                         error(Enomem);
 448         if(ctlr->nsid == nil)
 449                 if((ctlr->nsid = mallocalign(0x1000, ctlr->mps, 0, 0)) == nil)
 450                         error(Enomem);
 451
 452         e = qcmd(&ws, ctlr, 1, 0x06, 0, nil, ctlr->ident, 0x1000);
 453         e[10] = 1; // identify controller
 454         checkstatus(wcmd(&ws), "identify controller");
 455
 456         e = qcmd(&ws, ctlr, 1, 0x06, 0, nil, ctlr->nsid, 0x1000);
 457         e[10] = 2; // namespace list
 458         if(wcmd(&ws) != 0)
 459                 ctlr->nsid[0] = 1;      /* assume namespace #1 */
 460
 461         ctlr->nnsid = 0;
 462         while(ctlr->nnsid < 1024 && ctlr->nsid[ctlr->nnsid] != 0)
 463                 ctlr->nnsid++;
 464 }
 465
 466 static int
 467 nvmedisable(SDev *sd)
 468 {
 469         char name[32];
 470         Ctlr *ctlr;
 471         int i;
 472
 473         ctlr = sd->ctlr;
 474
 475         /* mask interrupts */
 476         ilock(&ctlr->intr);
 477         ctlr->ints = 0;
 478         ctlr->reg[IntMs] = ~ctlr->ints;
 479         iunlock(&ctlr->intr);
 480
 481         /* disable controller */
 482         ctlr->reg[CCfg] = 0;
 483
 484         for(i = 0; i < 10; i++){
 485                 if((ctlr->reg[CSts] & 1) == 0)
 486                         break;
 487                 tsleep(&up->sleep, return0, nil, 100);
 488         }
 489
 490         snprint(name, sizeof(name), "%s (%s)", sd->name, sd->ifc->name);
 491         intrdisable(ctlr->pci->intl, nvmeintr, ctlr, ctlr->pci->tbdf, name);
 492
 493         pciclrbme(ctlr->pci);   /* dma disable */
 494
 495         for(i=0; i<nelem(ctlr->sq); i++){
 496                 free(ctlr->sq[i].base);
 497                 free(ctlr->sq[i].wait);
 498         }
 499         for(i=0; i<nelem(ctlr->cq); i++)
 500                 free(ctlr->cq[i].base);
 501
 502         memset(ctlr->sq, 0, sizeof(ctlr->sq));
 503         memset(ctlr->cq, 0, sizeof(ctlr->cq));
 504
 505         free(ctlr->ident);
 506         ctlr->ident = nil;
 507         free(ctlr->nsid);
 508         ctlr->nsid = nil;
 509         ctlr->nnsid = 0;
 510
 511         return 1;
 512 }
 513
 514 static int
 515 nvmeenable(SDev *sd)
 516 {
 517         char name[32];
 518         Ctlr *ctlr;
 519         u64int pa;
 520         int to;
 521
 522         ctlr = sd->ctlr;
 523
 524         snprint(name, sizeof(name), "%s (%s)", sd->name, sd->ifc->name);
 525         intrenable(ctlr->pci->intl, nvmeintr, ctlr, ctlr->pci->tbdf, name);
 526
 527         if(waserror()){
 528                 print("%s: %s\n", name, up->errstr);
 529                 nvmedisable(sd);
 530                 sd->nunit = 0;  /* hack: prevent further probing */
 531                 return 0;
 532         }
 533
 534         pa = PCIWADDR(cqalloc(ctlr, &ctlr->cq[0], ctlr->mpsshift));
 535         ctlr->reg[ACQBase0] = pa;
 536         ctlr->reg[ACQBase1] = pa>>32;
 537
 538         pa = PCIWADDR(sqalloc(ctlr, &ctlr->sq[0], ctlr->mpsshift));
 539         ctlr->reg[ASQBase0] = pa;
 540         ctlr->reg[ASQBase1] = pa>>32;
 541
 542         ctlr->reg[AQAttr] = ctlr->sq[0].mask | ctlr->cq[0].mask<<16;
 543
 544         /* dma enable */
 545         pcisetbme(ctlr->pci);
 546
 547         /* enable interrupt */
 548         ilock(&ctlr->intr);
 549         ctlr->ints = 1;
 550         ctlr->reg[IntMc] = ctlr->ints;
 551         iunlock(&ctlr->intr);
 552
 553         /* enable controller */
 554         ctlr->reg[CCfg] = 1 | (ctlr->mpsshift-12)<<7 | 6<<16 | 4<<20;
 555
 556         for(to = (ctlr->cap>>24) & 255; to >= 0; to--){
 557                 tsleep(&up->sleep, return0, nil, 500);
 558                 if((ctlr->reg[CSts] & 3) == 1)
 559                         goto Ready;
 560         }
 561         if(ctlr->reg[CSts] & 2)
 562                 error("fatal controller status during initialization");
 563         error("controller initialization timeout");
 564 Ready:
 565         identify(ctlr);
 566         setupqueues(ctlr);
 567         print("%s: using %d submit queues\n", name, ctlr->nsq);
 568         poperror();
 569
 570         return 1;
 571 }
 572
 573 static Ctlr*
 574 nvmepnpctlrs(void)
 575 {
 576         Ctlr *ctlr, *h, *t;
 577         Pcidev *p;
 578         int i;
 579
 580         h = t = nil;
 581         for(p = nil; p = pcimatch(p, 0, 0);){
 582                 if(p->ccrb != 1 || p->ccru != 8 || p->ccrp != 2)
 583                         continue;
 584                 if(p->mem[0].size == 0 || (p->mem[0].bar & 1) != 0)
 585                         continue;
 586                 if((ctlr = malloc(sizeof(*ctlr))) == nil){
 587                         print("nvme: no memory for Ctlr\n");
 588                         break;
 589                 }
 590                 pcienable(p);
 591                 ctlr->pci = p;
 592                 ctlr->reg = vmap(p->mem[0].bar & ~0xF, p->mem[0].size);
 593                 if(ctlr->reg == nil){
 594                         print("nvme: can't vmap bar0\n");
 595                 Bad:
 596                         if(ctlr->reg != nil)
 597                                 vunmap(ctlr->reg, p->mem[0].size);
 598                         pcidisable(p);
 599                         free(ctlr);
 600                         continue;
 601                 }
 602                 ctlr->cap = ctlr->reg[Cap0];
 603                 ctlr->cap |= (u64int)ctlr->reg[Cap1]<<32;
 604
 605                 /* mask interrupts */
 606                 ctlr->ints = 0;
 607                 ctlr->reg[IntMs] = ~ctlr->ints;
 608
 609                 /* disable controller */
 610                 ctlr->reg[CCfg] = 0;
 611
 612                 if((ctlr->cap&(1ULL<<37)) == 0){
 613                         print("nvme: doesnt support NVM commactlr set: %ux\n",
 614                                 (u32int)(ctlr->cap>>37) & 0xFF);
 615                         goto Bad;
 616                 }
 617
 618                 /* use 64K page size when possible */
 619                 ctlr->dstrd = (ctlr->cap >> 32) & 15;
 620                 for(i = (ctlr->cap >> 48) & 15; i < ((ctlr->cap >> 52) & 15); i++){
 621                         if(i >= 16-12)  /* 64K */
 622                                 break;
 623                 }
 624                 ctlr->mpsshift = i+12;
 625                 ctlr->mps = 1 << ctlr->mpsshift;
 626
 627                 if(h == nil)
 628                         h = ctlr;
 629                 else
 630                         t->next = ctlr;
 631                 t = ctlr;
 632         }
 633
 634         return h;
 635 }
 636
 637 SDifc sdnvmeifc;
 638
 639 static SDev*
 640 nvmepnp(void)
 641 {
 642         SDev *s, *h, *t;
 643         Ctlr *ctlr;
 644         int id;
 645
 646         h = t = nil;
 647
 648         id = 'N';
 649         for(ctlr = nvmepnpctlrs(); ctlr != nil; ctlr = ctlr->next){
 650                 if((s = malloc(sizeof(*s))) == nil)
 651                         break;
 652                 s->ctlr = ctlr;
 653                 s->idno = id++;
 654                 s->ifc = &sdnvmeifc;
 655                 s->nunit = 1024;
 656                 if(h)
 657                         t->next = s;
 658                 else
 659                         h = s;
 660                 t = s;
 661         }
 662
 663         return h;
 664 }
 665
 666 SDifc sdnvmeifc = {
 667         "nvme",                         /* name */
 668
 669         nvmepnp,                        /* pnp */
 670         nil,                            /* legacy */
 671         nvmeenable,                     /* enable */
 672         nvmedisable,                    /* disable */
 673
 674         nvmeverify,                     /* verify */
 675         nvmeonline,                     /* online */
 676         nvmerio,                        /* rio */
 677         nvmerctl,                       /* rctl */
 678         nil,                            /* wctl */
 679
 680         nvmebio,                        /* bio */
 681         nil,                            /* probe */
 682         nil,                            /* clear */
 683         nil,                            /* rtopctl */
 684         nil,                            /* wtopctl */
 685 };