sys/src/9/pc/sdvirtio.c

   1 #include "u.h"
   2 #include "../port/lib.h"
   3 #include "mem.h"
   4 #include "dat.h"
   5 #include "fns.h"
   6 #include "io.h"
   7 #include "ureg.h"
   8 #include "../port/error.h"
   9
  10 #include "../port/sd.h"
  11
  12 typedef struct Vring Vring;
  13 typedef struct Vdesc Vdesc;
  14 typedef struct Vused Vused;
  15 typedef struct Vqueue Vqueue;
  16 typedef struct Vdev Vdev;
  17
  18 typedef struct ScsiCfg ScsiCfg;
  19
  20 /* device types */
  21 enum {
  22         TypBlk  = 2,
  23         TypSCSI = 8,
  24 };
  25
  26 /* status flags */
  27 enum {
  28         Acknowledge = 1,
  29         Driver = 2,
  30         DriverOk = 4,
  31         Failed = 0x80,
  32 };
  33
  34 /* virtio ports */
  35 enum {
  36         Devfeat = 0,
  37         Drvfeat = 4,
  38         Qaddr = 8,
  39         Qsize = 12,
  40         Qselect = 14,
  41         Qnotify = 16,
  42         Status = 18,
  43         Isr = 19,
  44
  45         Devspec = 20,
  46 };
  47
  48 /* descriptor flags */
  49 enum {
  50         Next = 1,
  51         Write = 2,
  52         Indirect = 4,
  53 };
  54
  55 /* struct sizes */
  56 enum {
  57         VringSize = 4,
  58 };
  59
  60 struct Vring
  61 {
  62         u16int  flags;
  63         u16int  idx;
  64 };
  65
  66 struct Vdesc
  67 {
  68         u64int  addr;
  69         u32int  len;
  70         u16int  flags;
  71         u16int  next;
  72 };
  73
  74 struct Vused
  75 {
  76         u32int  id;
  77         u32int  len;
  78 };
  79
  80 struct Vqueue
  81 {
  82         Lock;
  83
  84         Vdev    *dev;
  85         int     idx;
  86
  87         int     size;
  88
  89         int     free;
  90         int     nfree;
  91
  92         Vdesc   *desc;
  93
  94         Vring   *avail;
  95         u16int  *availent;
  96         u16int  *availevent;
  97
  98         Vring   *used;
  99         Vused   *usedent;
 100         u16int  *usedevent;
 101         u16int  lastused;
 102
 103         void    *rock[];
 104 };
 105
 106 struct Vdev
 107 {
 108         int     typ;
 109
 110         Pcidev  *pci;
 111
 112         ulong   port;
 113         ulong   feat;
 114
 115         int     nqueue;
 116         Vqueue  *queue[16];
 117
 118         void    *cfg;   /* device specific config (for scsi) */
 119
 120         Vdev    *next;
 121 };
 122
 123 enum {
 124         CDBSIZE         = 32,
 125         SENSESIZE       = 96,
 126 };
 127
 128 struct ScsiCfg
 129 {
 130         u32int  num_queues;
 131         u32int  seg_max;
 132         u32int  max_sectors;
 133         u32int  cmd_per_lun;
 134         u32int  event_info_size;
 135         u32int  sense_size;
 136         u32int  cdb_size;
 137         u16int  max_channel;
 138         u16int  max_target;
 139         u32int  max_lun;
 140 };
 141
 142 static Vqueue*
 143 mkvqueue(int size)
 144 {
 145         Vqueue *q;
 146         uchar *p;
 147         int i;
 148
 149         q = malloc(sizeof(*q) + sizeof(void*)*size);
 150         p = mallocalign(
 151                 PGROUND(sizeof(Vdesc)*size +
 152                         VringSize +
 153                         sizeof(u16int)*size +
 154                         sizeof(u16int)) +
 155                 PGROUND(VringSize +
 156                         sizeof(Vused)*size +
 157                         sizeof(u16int)),
 158                 BY2PG, 0, 0);
 159         if(p == nil || q == nil){
 160                 print("virtio: no memory for Vqueue\n");
 161                 free(p);
 162                 free(q);
 163                 return nil;
 164         }
 165
 166         q->desc = (void*)p;
 167         p += sizeof(Vdesc)*size;
 168         q->avail = (void*)p;
 169         p += VringSize;
 170         q->availent = (void*)p;
 171         p += sizeof(u16int)*size;
 172         q->availevent = (void*)p;
 173         p += sizeof(u16int);
 174
 175         p = (uchar*)PGROUND((uintptr)p);
 176         q->used = (void*)p;
 177         p += VringSize;
 178         q->usedent = (void*)p;
 179         p += sizeof(Vused)*size;
 180         q->usedevent = (void*)p;
 181
 182         q->free = -1;
 183         q->nfree = q->size = size;
 184         for(i=0; i<size; i++){
 185                 q->desc[i].next = q->free;
 186                 q->free = i;
 187         }
 188
 189         return q;
 190 }
 191
 192 static Vdev*
 193 viopnpdevs(int typ)
 194 {
 195         Vdev *vd, *h, *t;
 196         Vqueue *q;
 197         Pcidev *p;
 198         int n, i;
 199
 200         h = t = nil;
 201         for(p = nil; p = pcimatch(p, 0x1AF4, 0);){
 202                 if((p->did < 0x1000) || (p->did > 0x103F))
 203                         continue;
 204                 if(p->rid != 0)
 205                         continue;
 206                 if(pcicfgr16(p, 0x2E) != typ)
 207                         continue;
 208                 if((vd = malloc(sizeof(*vd))) == nil){
 209                         print("virtio: no memory for Vdev\n");
 210                         break;
 211                 }
 212                 vd->port = p->mem[0].bar & ~0x1;
 213                 if(ioalloc(vd->port, p->mem[0].size, 0, "virtio") < 0){
 214                         print("virtio: port %lux in use\n", vd->port);
 215                         free(vd);
 216                         continue;
 217                 }
 218                 vd->typ = typ;
 219                 vd->pci = p;
 220
 221                 /* reset */
 222                 outb(vd->port+Status, 0);
 223
 224                 vd->feat = inl(vd->port+Devfeat);
 225                 outb(vd->port+Status, Acknowledge|Driver);
 226                 for(i=0; i<nelem(vd->queue); i++){
 227                         outs(vd->port+Qselect, i);
 228                         n = ins(vd->port+Qsize);
 229                         if(n == 0 || (n & (n-1)) != 0)
 230                                 break;
 231                         if((q = mkvqueue(n)) == nil)
 232                                 break;
 233                         q->dev = vd;
 234                         q->idx = i;
 235                         vd->queue[i] = q;
 236                         coherence();
 237                         outl(vd->port+Qaddr, PADDR(vd->queue[i]->desc)/BY2PG);
 238                 }
 239                 vd->nqueue = i;
 240
 241                 if(h == nil)
 242                         h = vd;
 243                 else
 244                         t->next = vd;
 245                 t = vd;
 246         }
 247
 248         return h;
 249 }
 250
 251 struct Rock {
 252         int done;
 253         Rendez *sleep;
 254 };
 255
 256 static void
 257 vqinterrupt(Vqueue *q)
 258 {
 259         int id, free, m;
 260         struct Rock *r;
 261         Rendez *z;
 262
 263         m = q->size-1;
 264
 265         ilock(q);
 266         while((q->lastused ^ q->used->idx) & m){
 267                 id = q->usedent[q->lastused++ & m].id;
 268                 if(r = q->rock[id]){
 269                         q->rock[id] = nil;
 270                         z = r->sleep;
 271                         r->done = 1;    /* hands off */
 272                         if(z != nil)
 273                                 wakeup(z);
 274                 }
 275                 do {
 276                         free = id;
 277                         id = q->desc[free].next;
 278                         q->desc[free].next = q->free;
 279                         q->free = free;
 280                         q->nfree++;
 281                 } while(q->desc[free].flags & Next);
 282         }
 283         iunlock(q);
 284 }
 285
 286 static void
 287 viointerrupt(Ureg *, void *arg)
 288 {
 289         Vdev *vd = arg;
 290
 291         if(inb(vd->port+Isr) & 1)
 292                 vqinterrupt(vd->queue[vd->typ == TypSCSI ? 2 : 0]);
 293 }
 294
 295 static int
 296 viodone(void *arg)
 297 {
 298         return ((struct Rock*)arg)->done;
 299 }
 300
 301 static void
 302 vqio(Vqueue *q, int head)
 303 {
 304         struct Rock rock;
 305
 306         rock.done = 0;
 307         rock.sleep = &up->sleep;
 308         q->rock[head] = &rock;
 309         q->availent[q->avail->idx & (q->size-1)] = head;
 310         coherence();
 311         q->avail->idx++;
 312         iunlock(q);
 313         if((q->used->flags & 1) == 0)
 314                 outs(q->dev->port+Qnotify, q->idx);
 315         while(!rock.done){
 316                 while(waserror())
 317                         ;
 318                 tsleep(rock.sleep, viodone, &rock, 1000);
 319                 poperror();
 320
 321                 if(!rock.done)
 322                         vqinterrupt(q);
 323         }
 324 }
 325
 326 static int
 327 vioblkreq(Vdev *vd, int typ, void *a, long count, long secsize, uvlong lba)
 328 {
 329         int need, free, head;
 330         Vqueue *q;
 331         Vdesc *d;
 332
 333         u8int status;
 334         struct Vioblkreqhdr {
 335                 u32int  typ;
 336                 u32int  prio;
 337                 u64int  lba;
 338         } req;
 339
 340         need = 2;
 341         if(a != nil)
 342                 need = 3;
 343
 344         status = -1;
 345         req.typ = typ;
 346         req.prio = 0;
 347         req.lba = lba;
 348
 349         q = vd->queue[0];
 350         ilock(q);
 351         while(q->nfree < need){
 352                 iunlock(q);
 353
 354                 if(!waserror())
 355                         tsleep(&up->sleep, return0, 0, 500);
 356                 poperror();
 357
 358                 ilock(q);
 359         }
 360
 361         head = free = q->free;
 362
 363         d = &q->desc[free]; free = d->next;
 364         d->addr = PADDR(&req);
 365         d->len = sizeof(req);
 366         d->flags = Next;
 367
 368         if(a != nil){
 369                 d = &q->desc[free]; free = d->next;
 370                 d->addr = PADDR(a);
 371                 d->len = secsize*count;
 372                 d->flags = typ ? Next : (Write|Next);
 373         }
 374
 375         d = &q->desc[free]; free = d->next;
 376         d->addr = PADDR(&status);
 377         d->len = sizeof(status);
 378         d->flags = Write;
 379
 380         q->free = free;
 381         q->nfree -= need;
 382
 383         /* queue io, unlock and wait for completion */
 384         vqio(q, head);
 385
 386         return status;
 387 }
 388
 389 static int
 390 vioscsireq(SDreq *r)
 391 {
 392         u8int resp[4+4+2+2+SENSESIZE];
 393         u8int req[8+8+3+CDBSIZE];
 394         int free, head;
 395         u32int len;
 396         Vqueue *q;
 397         Vdesc *d;
 398         Vdev *vd;
 399         SDunit *u;
 400         ScsiCfg *cfg;
 401
 402         u = r->unit;
 403         vd = u->dev->ctlr;
 404         cfg = vd->cfg;
 405
 406         memset(resp, 0, sizeof(resp));
 407         memset(req, 0, sizeof(req));
 408         req[0] = 1;
 409         req[1] = u->subno;
 410         req[2] = r->lun>>8;
 411         req[3] = r->lun&0xFF;
 412         *(u64int*)(&req[8]) = (uintptr)r;
 413
 414         memmove(&req[8+8+3], r->cmd, r->clen);
 415
 416         q = vd->queue[2];
 417         ilock(q);
 418         while(q->nfree < 3){
 419                 iunlock(q);
 420
 421                 if(!waserror())
 422                         tsleep(&up->sleep, return0, 0, 500);
 423                 poperror();
 424
 425                 ilock(q);
 426         }
 427
 428         head = free = q->free;
 429
 430         d = &q->desc[free]; free = d->next;
 431         d->addr = PADDR(req);
 432         d->len = 8+8+3+cfg->cdb_size;
 433         d->flags = Next;
 434
 435         if(r->write && r->dlen > 0){
 436                 d = &q->desc[free]; free = d->next;
 437                 d->addr = PADDR(r->data);
 438                 d->len = r->dlen;
 439                 d->flags = Next;
 440         }
 441
 442         d = &q->desc[free]; free = d->next;
 443         d->addr = PADDR(resp);
 444         d->len = 4+4+2+2+cfg->sense_size;
 445         d->flags = Write;
 446
 447         if(!r->write && r->dlen > 0){
 448                 d->flags |= Next;
 449
 450                 d = &q->desc[free]; free = d->next;
 451                 d->addr = PADDR(r->data);
 452                 d->len = r->dlen;
 453                 d->flags = Write;
 454         }
 455
 456         q->free = free;
 457         q->nfree -= 2 + (r->dlen > 0);
 458
 459         /* queue io, unlock and wait for completion */
 460         vqio(q, head);
 461
 462         /* response+status */
 463         r->status = resp[10];
 464         if(resp[11] != 0)
 465                 r->status = SDcheck;
 466
 467         /* sense_len */
 468         len = *((u32int*)&resp[0]);
 469         if(len > 0){
 470                 if(len > sizeof(r->sense))
 471                         len = sizeof(r->sense);
 472                 memmove(r->sense, &resp[4+4+2+2], len);
 473                 r->flags |= SDvalidsense;
 474         }
 475
 476         /* data residue */
 477         len = *((u32int*)&resp[4]);
 478         if(len > r->dlen)
 479                 r->rlen = 0;
 480         else
 481                 r->rlen = r->dlen - len;
 482
 483         return r->status;
 484
 485 }
 486
 487 static long
 488 viobio(SDunit *u, int lun, int write, void *a, long count, uvlong lba)
 489 {
 490         long ss, cc, max, ret;
 491         Vdev *vd;
 492
 493         vd = u->dev->ctlr;
 494         if(vd->typ == TypSCSI)
 495                 return scsibio(u, lun, write, a, count, lba);
 496
 497         max = 32;
 498         ss = u->secsize;
 499         ret = 0;
 500         while(count > 0){
 501                 if((cc = count) > max)
 502                         cc = max;
 503                 if(vioblkreq(vd, write != 0, (uchar*)a + ret, cc, ss, lba) != 0)
 504                         error(Eio);
 505                 ret += cc*ss;
 506                 count -= cc;
 507                 lba += cc;
 508         }
 509         return ret;
 510 }
 511
 512 static int
 513 viorio(SDreq *r)
 514 {
 515         int i, count, rw;
 516         uvlong lba;
 517         SDunit *u;
 518         Vdev *vd;
 519
 520         u = r->unit;
 521         vd = u->dev->ctlr;
 522         if(vd->typ == TypSCSI)
 523                 return vioscsireq(r);
 524         if(r->cmd[0] == 0x35 || r->cmd[0] == 0x91){
 525                 if(vioblkreq(vd, 4, nil, 0, 0, 0) != 0)
 526                         return sdsetsense(r, SDcheck, 3, 0xc, 2);
 527                 return sdsetsense(r, SDok, 0, 0, 0);
 528         }
 529         if((i = sdfakescsi(r)) != SDnostatus)
 530                 return r->status = i;
 531         if((i = sdfakescsirw(r, &lba, &count, &rw)) != SDnostatus)
 532                 return i;
 533         r->rlen = viobio(u, r->lun, rw == SDwrite, r->data, count, lba);
 534         return r->status = SDok;
 535 }
 536
 537 static int
 538 vioonline(SDunit *u)
 539 {
 540         uvlong cap;
 541         Vdev *vd;
 542
 543         vd = u->dev->ctlr;
 544         if(vd->typ == TypSCSI)
 545                 return scsionline(u);
 546
 547         cap = inl(vd->port+Devspec+4);
 548         cap <<= 32;
 549         cap |= inl(vd->port+Devspec);
 550         if(u->sectors != cap){
 551                 u->sectors = cap;
 552                 u->secsize = 512;
 553                 return 2;
 554         }
 555         return 1;
 556 }
 557
 558 static int
 559 vioverify(SDunit *u)
 560 {
 561         Vdev *vd;
 562
 563         vd = u->dev->ctlr;
 564         if(vd->typ == TypSCSI)
 565                 return scsiverify(u);
 566
 567         return 1;
 568 }
 569
 570 SDifc sdvirtioifc;
 571
 572 static int
 573 vioenable(SDev *sd)
 574 {
 575         char name[32];
 576         Vdev *vd;
 577
 578         vd = sd->ctlr;
 579         snprint(name, sizeof(name), "%s (%s)", sd->name, sd->ifc->name);
 580         intrenable(vd->pci->intl, viointerrupt, vd, vd->pci->tbdf, name);
 581         outb(vd->port+Status, inb(vd->port+Status) | DriverOk);
 582         return 1;
 583 }
 584
 585 static int
 586 viodisable(SDev *sd)
 587 {
 588         char name[32];
 589         Vdev *vd;
 590
 591         vd = sd->ctlr;
 592         snprint(name, sizeof(name), "%s (%s)", sd->name, sd->ifc->name);
 593         intrdisable(vd->pci->intl, viointerrupt, vd, vd->pci->tbdf, name);
 594         return 1;
 595 }
 596
 597 static SDev*
 598 viopnp(void)
 599 {
 600         SDev *s, *h, *t;
 601         Vdev *vd;
 602         int id;
 603
 604         h = t = nil;
 605
 606         id = 'F';
 607         for(vd =  viopnpdevs(TypBlk); vd; vd = vd->next){
 608                 if(vd->nqueue != 1)
 609                         continue;
 610
 611                 if((s = malloc(sizeof(*s))) == nil)
 612                         break;
 613                 s->ctlr = vd;
 614                 s->idno = id++;
 615                 s->ifc = &sdvirtioifc;
 616                 s->nunit = 1;
 617                 if(h)
 618                         t->next = s;
 619                 else
 620                         h = s;
 621                 t = s;
 622         }
 623
 624         id = '0';
 625         for(vd = viopnpdevs(TypSCSI); vd; vd = vd->next){
 626                 ScsiCfg *cfg;
 627
 628                 if(vd->nqueue < 3)
 629                         continue;
 630
 631                 if((cfg = malloc(sizeof(*cfg))) == nil)
 632                         break;
 633                 cfg->num_queues = inl(vd->port+Devspec+4*0);
 634                 cfg->seg_max = inl(vd->port+Devspec+4*1);
 635                 cfg->max_sectors = inl(vd->port+Devspec+4*2);
 636                 cfg->cmd_per_lun = inl(vd->port+Devspec+4*3);
 637                 cfg->event_info_size = inl(vd->port+Devspec+4*4);
 638                 cfg->sense_size = inl(vd->port+Devspec+4*5);
 639                 cfg->cdb_size = inl(vd->port+Devspec+4*6);
 640                 cfg->max_channel = ins(vd->port+Devspec+4*7);
 641                 cfg->max_target = ins(vd->port+Devspec+4*7+2);
 642                 cfg->max_lun = inl(vd->port+Devspec+4*8);
 643
 644                 if(cfg->max_target == 0){
 645                         free(cfg);
 646                         continue;
 647                 }
 648                 if((cfg->cdb_size > CDBSIZE) || (cfg->sense_size > SENSESIZE)){
 649                         print("sdvirtio: cdb %ud or sense size %ud too big\n",
 650                                 cfg->cdb_size, cfg->sense_size);
 651                         free(cfg);
 652                         continue;
 653                 }
 654                 vd->cfg = cfg;
 655
 656                 if((s = malloc(sizeof(*s))) == nil)
 657                         break;
 658                 s->ctlr = vd;
 659                 s->idno = id++;
 660                 s->ifc = &sdvirtioifc;
 661                 s->nunit = cfg->max_target;
 662                 if(h)
 663                         t->next = s;
 664                 else
 665                         h = s;
 666                 t = s;
 667         }
 668
 669         return h;
 670 }
 671
 672 SDifc sdvirtioifc = {
 673         "virtio",                       /* name */
 674
 675         viopnp,                         /* pnp */
 676         nil,                            /* legacy */
 677         vioenable,                      /* enable */
 678         viodisable,                     /* disable */
 679
 680         vioverify,                      /* verify */
 681         vioonline,                      /* online */
 682         viorio,                         /* rio */
 683         nil,                            /* rctl */
 684         nil,                            /* wctl */
 685
 686         viobio,                         /* bio */
 687         nil,                            /* probe */
 688         nil,                            /* clear */
 689         nil,                            /* rtopctl */
 690         nil,                            /* wtopctl */
 691 };