sys/src/9/pc/sdvirtio.c

   1 /*
   2  * virtio ethernet driver implementing the legacy interface:
   3  * http://docs.oasis-open.org/virtio/virtio/v1.0/virtio-v1.0.html
   4  */
   5 #include "u.h"
   6 #include "../port/lib.h"
   7 #include "mem.h"
   8 #include "dat.h"
   9 #include "fns.h"
  10 #include "io.h"
  11 #include "../port/pci.h"
  12 #include "ureg.h"
  13 #include "../port/error.h"
  14
  15 #include "../port/sd.h"
  16
  17 typedef struct Vring Vring;
  18 typedef struct Vdesc Vdesc;
  19 typedef struct Vused Vused;
  20 typedef struct Vqueue Vqueue;
  21 typedef struct Vdev Vdev;
  22
  23 typedef struct ScsiCfg ScsiCfg;
  24
  25 /* device types */
  26 enum {
  27         TypBlk  = 2,
  28         TypSCSI = 8,
  29 };
  30
  31 /* status flags */
  32 enum {
  33         Acknowledge = 1,
  34         Driver = 2,
  35         DriverOk = 4,
  36         Failed = 0x80,
  37 };
  38
  39 /* virtio ports */
  40 enum {
  41         Devfeat = 0,
  42         Drvfeat = 4,
  43         Qaddr = 8,
  44         Qsize = 12,
  45         Qselect = 14,
  46         Qnotify = 16,
  47         Status = 18,
  48         Isr = 19,
  49
  50         Devspec = 20,
  51 };
  52
  53 /* descriptor flags */
  54 enum {
  55         Next = 1,
  56         Write = 2,
  57         Indirect = 4,
  58 };
  59
  60 /* struct sizes */
  61 enum {
  62         VringSize = 4,
  63 };
  64
  65 struct Vring
  66 {
  67         u16int  flags;
  68         u16int  idx;
  69 };
  70
  71 struct Vdesc
  72 {
  73         u64int  addr;
  74         u32int  len;
  75         u16int  flags;
  76         u16int  next;
  77 };
  78
  79 struct Vused
  80 {
  81         u32int  id;
  82         u32int  len;
  83 };
  84
  85 struct Vqueue
  86 {
  87         Lock;
  88
  89         Vdev    *dev;
  90         int     idx;
  91
  92         int     size;
  93
  94         int     free;
  95         int     nfree;
  96
  97         Vdesc   *desc;
  98
  99         Vring   *avail;
 100         u16int  *availent;
 101         u16int  *availevent;
 102
 103         Vring   *used;
 104         Vused   *usedent;
 105         u16int  *usedevent;
 106         u16int  lastused;
 107
 108         void    *rock[];
 109 };
 110
 111 struct Vdev
 112 {
 113         int     typ;
 114
 115         Pcidev  *pci;
 116
 117         ulong   port;
 118         ulong   feat;
 119
 120         int     nqueue;
 121         Vqueue  *queue[16];
 122
 123         void    *cfg;   /* device specific config (for scsi) */
 124
 125         Vdev    *next;
 126 };
 127
 128 enum {
 129         CDBSIZE         = 32,
 130         SENSESIZE       = 96,
 131 };
 132
 133 struct ScsiCfg
 134 {
 135         u32int  num_queues;
 136         u32int  seg_max;
 137         u32int  max_sectors;
 138         u32int  cmd_per_lun;
 139         u32int  event_info_size;
 140         u32int  sense_size;
 141         u32int  cdb_size;
 142         u16int  max_channel;
 143         u16int  max_target;
 144         u32int  max_lun;
 145 };
 146
 147 static Vqueue*
 148 mkvqueue(int size)
 149 {
 150         Vqueue *q;
 151         uchar *p;
 152         int i;
 153
 154         q = malloc(sizeof(*q) + sizeof(void*)*size);
 155         p = mallocalign(
 156                 PGROUND(sizeof(Vdesc)*size +
 157                         VringSize +
 158                         sizeof(u16int)*size +
 159                         sizeof(u16int)) +
 160                 PGROUND(VringSize +
 161                         sizeof(Vused)*size +
 162                         sizeof(u16int)),
 163                 BY2PG, 0, 0);
 164         if(p == nil || q == nil){
 165                 print("virtio: no memory for Vqueue\n");
 166                 free(p);
 167                 free(q);
 168                 return nil;
 169         }
 170
 171         q->desc = (void*)p;
 172         p += sizeof(Vdesc)*size;
 173         q->avail = (void*)p;
 174         p += VringSize;
 175         q->availent = (void*)p;
 176         p += sizeof(u16int)*size;
 177         q->availevent = (void*)p;
 178         p += sizeof(u16int);
 179
 180         p = (uchar*)PGROUND((uintptr)p);
 181         q->used = (void*)p;
 182         p += VringSize;
 183         q->usedent = (void*)p;
 184         p += sizeof(Vused)*size;
 185         q->usedevent = (void*)p;
 186
 187         q->free = -1;
 188         q->nfree = q->size = size;
 189         for(i=0; i<size; i++){
 190                 q->desc[i].next = q->free;
 191                 q->free = i;
 192         }
 193
 194         return q;
 195 }
 196
 197 static Vdev*
 198 viopnpdevs(int typ)
 199 {
 200         Vdev *vd, *h, *t;
 201         Vqueue *q;
 202         Pcidev *p;
 203         int n, i;
 204
 205         h = t = nil;
 206         for(p = nil; p = pcimatch(p, 0x1AF4, 0);){
 207                 if((p->did < 0x1000) || (p->did > 0x103F))
 208                         continue;
 209                 if(p->rid != 0)
 210                         continue;
 211                 if((p->mem[0].bar & 1) == 0)
 212                         continue;
 213                 if(pcicfgr16(p, 0x2E) != typ)
 214                         continue;
 215                 if((vd = malloc(sizeof(*vd))) == nil){
 216                         print("virtio: no memory for Vdev\n");
 217                         break;
 218                 }
 219                 vd->port = p->mem[0].bar & ~3;
 220                 if(ioalloc(vd->port, p->mem[0].size, 0, "virtio") < 0){
 221                         print("virtio: port %lux in use\n", vd->port);
 222                         free(vd);
 223                         continue;
 224                 }
 225                 vd->typ = typ;
 226                 vd->pci = p;
 227                 pcienable(p);
 228
 229                 /* reset */
 230                 outb(vd->port+Status, 0);
 231
 232                 vd->feat = inl(vd->port+Devfeat);
 233                 outb(vd->port+Status, Acknowledge|Driver);
 234                 for(i=0; i<nelem(vd->queue); i++){
 235                         outs(vd->port+Qselect, i);
 236                         n = ins(vd->port+Qsize);
 237                         if(n == 0 || (n & (n-1)) != 0)
 238                                 break;
 239                         if((q = mkvqueue(n)) == nil)
 240                                 break;
 241                         q->dev = vd;
 242                         q->idx = i;
 243                         vd->queue[i] = q;
 244                         coherence();
 245                         outl(vd->port+Qaddr, PADDR(vd->queue[i]->desc)/BY2PG);
 246                 }
 247                 vd->nqueue = i;
 248
 249                 if(h == nil)
 250                         h = vd;
 251                 else
 252                         t->next = vd;
 253                 t = vd;
 254         }
 255
 256         return h;
 257 }
 258
 259 struct Rock {
 260         int done;
 261         Rendez *sleep;
 262 };
 263
 264 static void
 265 vqinterrupt(Vqueue *q)
 266 {
 267         int id, free, m;
 268         struct Rock *r;
 269         Rendez *z;
 270
 271         m = q->size-1;
 272
 273         ilock(q);
 274         while((q->lastused ^ q->used->idx) & m){
 275                 id = q->usedent[q->lastused++ & m].id;
 276                 if(r = q->rock[id]){
 277                         q->rock[id] = nil;
 278                         z = r->sleep;
 279                         r->done = 1;    /* hands off */
 280                         if(z != nil)
 281                                 wakeup(z);
 282                 }
 283                 do {
 284                         free = id;
 285                         id = q->desc[free].next;
 286                         q->desc[free].next = q->free;
 287                         q->free = free;
 288                         q->nfree++;
 289                 } while(q->desc[free].flags & Next);
 290         }
 291         iunlock(q);
 292 }
 293
 294 static void
 295 viointerrupt(Ureg *, void *arg)
 296 {
 297         Vdev *vd = arg;
 298
 299         if(inb(vd->port+Isr) & 1)
 300                 vqinterrupt(vd->queue[vd->typ == TypSCSI ? 2 : 0]);
 301 }
 302
 303 static int
 304 viodone(void *arg)
 305 {
 306         return ((struct Rock*)arg)->done;
 307 }
 308
 309 static void
 310 vqio(Vqueue *q, int head)
 311 {
 312         struct Rock rock;
 313
 314         rock.done = 0;
 315         rock.sleep = &up->sleep;
 316         q->rock[head] = &rock;
 317         q->availent[q->avail->idx & (q->size-1)] = head;
 318         coherence();
 319         q->avail->idx++;
 320         iunlock(q);
 321         if((q->used->flags & 1) == 0)
 322                 outs(q->dev->port+Qnotify, q->idx);
 323         while(!rock.done){
 324                 while(waserror())
 325                         ;
 326                 tsleep(rock.sleep, viodone, &rock, 1000);
 327                 poperror();
 328
 329                 if(!rock.done)
 330                         vqinterrupt(q);
 331         }
 332 }
 333
 334 static int
 335 vioblkreq(Vdev *vd, int typ, void *a, long count, long secsize, uvlong lba)
 336 {
 337         int need, free, head;
 338         Vqueue *q;
 339         Vdesc *d;
 340
 341         u8int status;
 342         struct Vioblkreqhdr {
 343                 u32int  typ;
 344                 u32int  prio;
 345                 u64int  lba;
 346         } req;
 347
 348         need = 2;
 349         if(a != nil)
 350                 need = 3;
 351
 352         status = -1;
 353         req.typ = typ;
 354         req.prio = 0;
 355         req.lba = lba;
 356
 357         q = vd->queue[0];
 358         ilock(q);
 359         while(q->nfree < need){
 360                 iunlock(q);
 361
 362                 if(!waserror())
 363                         tsleep(&up->sleep, return0, 0, 500);
 364                 poperror();
 365
 366                 ilock(q);
 367         }
 368
 369         head = free = q->free;
 370
 371         d = &q->desc[free]; free = d->next;
 372         d->addr = PADDR(&req);
 373         d->len = sizeof(req);
 374         d->flags = Next;
 375
 376         if(a != nil){
 377                 d = &q->desc[free]; free = d->next;
 378                 d->addr = PADDR(a);
 379                 d->len = secsize*count;
 380                 d->flags = typ ? Next : (Write|Next);
 381         }
 382
 383         d = &q->desc[free]; free = d->next;
 384         d->addr = PADDR(&status);
 385         d->len = sizeof(status);
 386         d->flags = Write;
 387
 388         q->free = free;
 389         q->nfree -= need;
 390
 391         /* queue io, unlock and wait for completion */
 392         vqio(q, head);
 393
 394         return status;
 395 }
 396
 397 static int
 398 vioscsireq(SDreq *r)
 399 {
 400         u8int resp[4+4+2+2+SENSESIZE];
 401         u8int req[8+8+3+CDBSIZE];
 402         int free, head;
 403         u32int len;
 404         Vqueue *q;
 405         Vdesc *d;
 406         Vdev *vd;
 407         SDunit *u;
 408         ScsiCfg *cfg;
 409
 410         u = r->unit;
 411         vd = u->dev->ctlr;
 412         cfg = vd->cfg;
 413
 414         memset(resp, 0, sizeof(resp));
 415         memset(req, 0, sizeof(req));
 416         req[0] = 1;
 417         req[1] = u->subno;
 418         req[2] = r->lun>>8;
 419         req[3] = r->lun&0xFF;
 420         *(u64int*)(&req[8]) = (uintptr)r;
 421
 422         memmove(&req[8+8+3], r->cmd, r->clen);
 423
 424         q = vd->queue[2];
 425         ilock(q);
 426         while(q->nfree < 3){
 427                 iunlock(q);
 428
 429                 if(!waserror())
 430                         tsleep(&up->sleep, return0, 0, 500);
 431                 poperror();
 432
 433                 ilock(q);
 434         }
 435
 436         head = free = q->free;
 437
 438         d = &q->desc[free]; free = d->next;
 439         d->addr = PADDR(req);
 440         d->len = 8+8+3+cfg->cdb_size;
 441         d->flags = Next;
 442
 443         if(r->write && r->dlen > 0){
 444                 d = &q->desc[free]; free = d->next;
 445                 d->addr = PADDR(r->data);
 446                 d->len = r->dlen;
 447                 d->flags = Next;
 448         }
 449
 450         d = &q->desc[free]; free = d->next;
 451         d->addr = PADDR(resp);
 452         d->len = 4+4+2+2+cfg->sense_size;
 453         d->flags = Write;
 454
 455         if(!r->write && r->dlen > 0){
 456                 d->flags |= Next;
 457
 458                 d = &q->desc[free]; free = d->next;
 459                 d->addr = PADDR(r->data);
 460                 d->len = r->dlen;
 461                 d->flags = Write;
 462         }
 463
 464         q->free = free;
 465         q->nfree -= 2 + (r->dlen > 0);
 466
 467         /* queue io, unlock and wait for completion */
 468         vqio(q, head);
 469
 470         /* response+status */
 471         r->status = resp[10];
 472         if(resp[11] != 0)
 473                 r->status = SDcheck;
 474
 475         /* sense_len */
 476         len = *((u32int*)&resp[0]);
 477         if(len > 0){
 478                 if(len > sizeof(r->sense))
 479                         len = sizeof(r->sense);
 480                 memmove(r->sense, &resp[4+4+2+2], len);
 481                 r->flags |= SDvalidsense;
 482         }
 483
 484         /* data residue */
 485         len = *((u32int*)&resp[4]);
 486         if(len > r->dlen)
 487                 r->rlen = 0;
 488         else
 489                 r->rlen = r->dlen - len;
 490
 491         return r->status;
 492
 493 }
 494
 495 static long
 496 viobio(SDunit *u, int lun, int write, void *a, long count, uvlong lba)
 497 {
 498         long ss, cc, max, ret;
 499         Vdev *vd;
 500
 501         vd = u->dev->ctlr;
 502         if(vd->typ == TypSCSI)
 503                 return scsibio(u, lun, write, a, count, lba);
 504
 505         max = 32;
 506         ss = u->secsize;
 507         ret = 0;
 508         while(count > 0){
 509                 if((cc = count) > max)
 510                         cc = max;
 511                 if(vioblkreq(vd, write != 0, (uchar*)a + ret, cc, ss, lba) != 0)
 512                         error(Eio);
 513                 ret += cc*ss;
 514                 count -= cc;
 515                 lba += cc;
 516         }
 517         return ret;
 518 }
 519
 520 static int
 521 viorio(SDreq *r)
 522 {
 523         int i, count, rw;
 524         uvlong lba;
 525         SDunit *u;
 526         Vdev *vd;
 527
 528         u = r->unit;
 529         vd = u->dev->ctlr;
 530         if(vd->typ == TypSCSI)
 531                 return vioscsireq(r);
 532         if(r->cmd[0] == 0x35 || r->cmd[0] == 0x91){
 533                 if(vioblkreq(vd, 4, nil, 0, 0, 0) != 0)
 534                         return sdsetsense(r, SDcheck, 3, 0xc, 2);
 535                 return sdsetsense(r, SDok, 0, 0, 0);
 536         }
 537         if((i = sdfakescsi(r)) != SDnostatus)
 538                 return r->status = i;
 539         if((i = sdfakescsirw(r, &lba, &count, &rw)) != SDnostatus)
 540                 return i;
 541         r->rlen = viobio(u, r->lun, rw == SDwrite, r->data, count, lba);
 542         return r->status = SDok;
 543 }
 544
 545 static int
 546 vioonline(SDunit *u)
 547 {
 548         uvlong cap;
 549         Vdev *vd;
 550
 551         vd = u->dev->ctlr;
 552         if(vd->typ == TypSCSI)
 553                 return scsionline(u);
 554
 555         cap = inl(vd->port+Devspec+4);
 556         cap <<= 32;
 557         cap |= inl(vd->port+Devspec);
 558         if(u->sectors != cap){
 559                 u->sectors = cap;
 560                 u->secsize = 512;
 561                 return 2;
 562         }
 563         return 1;
 564 }
 565
 566 static int
 567 vioverify(SDunit *u)
 568 {
 569         Vdev *vd;
 570
 571         vd = u->dev->ctlr;
 572         if(vd->typ == TypSCSI)
 573                 return scsiverify(u);
 574
 575         return 1;
 576 }
 577
 578 SDifc sdvirtioifc;
 579
 580 static int
 581 vioenable(SDev *sd)
 582 {
 583         char name[32];
 584         Vdev *vd;
 585
 586         vd = sd->ctlr;
 587         pcisetbme(vd->pci);
 588         snprint(name, sizeof(name), "%s (%s)", sd->name, sd->ifc->name);
 589         intrenable(vd->pci->intl, viointerrupt, vd, vd->pci->tbdf, name);
 590         outb(vd->port+Status, inb(vd->port+Status) | DriverOk);
 591         return 1;
 592 }
 593
 594 static int
 595 viodisable(SDev *sd)
 596 {
 597         char name[32];
 598         Vdev *vd;
 599
 600         vd = sd->ctlr;
 601         snprint(name, sizeof(name), "%s (%s)", sd->name, sd->ifc->name);
 602         intrdisable(vd->pci->intl, viointerrupt, vd, vd->pci->tbdf, name);
 603         pciclrbme(vd->pci);
 604         return 1;
 605 }
 606
 607 static SDev*
 608 viopnp(void)
 609 {
 610         SDev *s, *h, *t;
 611         Vdev *vd;
 612         int id;
 613
 614         h = t = nil;
 615
 616         id = 'F';
 617         for(vd =  viopnpdevs(TypBlk); vd; vd = vd->next){
 618                 if(vd->nqueue == 0)
 619                         continue;
 620
 621                 if((s = malloc(sizeof(*s))) == nil)
 622                         break;
 623                 s->ctlr = vd;
 624                 s->idno = id++;
 625                 s->ifc = &sdvirtioifc;
 626                 s->nunit = 1;
 627                 if(h)
 628                         t->next = s;
 629                 else
 630                         h = s;
 631                 t = s;
 632         }
 633
 634         id = '0';
 635         for(vd = viopnpdevs(TypSCSI); vd; vd = vd->next){
 636                 ScsiCfg *cfg;
 637
 638                 if(vd->nqueue < 3)
 639                         continue;
 640
 641                 if((cfg = malloc(sizeof(*cfg))) == nil)
 642                         break;
 643                 cfg->num_queues = inl(vd->port+Devspec+4*0);
 644                 cfg->seg_max = inl(vd->port+Devspec+4*1);
 645                 cfg->max_sectors = inl(vd->port+Devspec+4*2);
 646                 cfg->cmd_per_lun = inl(vd->port+Devspec+4*3);
 647                 cfg->event_info_size = inl(vd->port+Devspec+4*4);
 648                 cfg->sense_size = inl(vd->port+Devspec+4*5);
 649                 cfg->cdb_size = inl(vd->port+Devspec+4*6);
 650                 cfg->max_channel = ins(vd->port+Devspec+4*7);
 651                 cfg->max_target = ins(vd->port+Devspec+4*7+2);
 652                 cfg->max_lun = inl(vd->port+Devspec+4*8);
 653
 654                 if(cfg->max_target == 0){
 655                         free(cfg);
 656                         continue;
 657                 }
 658                 if((cfg->cdb_size > CDBSIZE) || (cfg->sense_size > SENSESIZE)){
 659                         print("sdvirtio: cdb %ud or sense size %ud too big\n",
 660                                 cfg->cdb_size, cfg->sense_size);
 661                         free(cfg);
 662                         continue;
 663                 }
 664                 vd->cfg = cfg;
 665
 666                 if((s = malloc(sizeof(*s))) == nil)
 667                         break;
 668                 s->ctlr = vd;
 669                 s->idno = id++;
 670                 s->ifc = &sdvirtioifc;
 671                 s->nunit = cfg->max_target;
 672                 if(h)
 673                         t->next = s;
 674                 else
 675                         h = s;
 676                 t = s;
 677         }
 678
 679         return h;
 680 }
 681
 682 SDifc sdvirtioifc = {
 683         "virtio",                       /* name */
 684
 685         viopnp,                         /* pnp */
 686         nil,                            /* legacy */
 687         vioenable,                      /* enable */
 688         viodisable,                     /* disable */
 689
 690         vioverify,                      /* verify */
 691         vioonline,                      /* online */
 692         viorio,                         /* rio */
 693         nil,                            /* rctl */
 694         nil,                            /* wctl */
 695
 696         viobio,                         /* bio */
 697         nil,                            /* probe */
 698         nil,                            /* clear */
 699         nil,                            /* rtopctl */
 700         nil,                            /* wtopctl */
 701 };