sys/src/9/pc/sdvirtio.c

   1 #include "u.h"
   2 #include "../port/lib.h"
   3 #include "mem.h"
   4 #include "dat.h"
   5 #include "fns.h"
   6 #include "io.h"
   7 #include "ureg.h"
   8 #include "../port/error.h"
   9
  10 #include "../port/sd.h"
  11
  12 typedef struct Vring Vring;
  13 typedef struct Vdesc Vdesc;
  14 typedef struct Vused Vused;
  15 typedef struct Vqueue Vqueue;
  16 typedef struct Vdev Vdev;
  17
  18 typedef struct ScsiCfg ScsiCfg;
  19
  20 /* device types */
  21 enum {
  22         TypBlk  = 2,
  23         TypSCSI = 8,
  24 };
  25
  26 /* status flags */
  27 enum {
  28         Acknowledge = 1,
  29         Driver = 2,
  30         DriverOk = 4,
  31         Failed = 0x80,
  32 };
  33
  34 /* virtio ports */
  35 enum {
  36         Devfeat = 0,
  37         Drvfeat = 4,
  38         Qaddr = 8,
  39         Qsize = 12,
  40         Qselect = 14,
  41         Qnotify = 16,
  42         Status = 18,
  43         Isr = 19,
  44
  45         Devspec = 20,
  46 };
  47
  48 /* descriptor flags */
  49 enum {
  50         Next = 1,
  51         Write = 2,
  52         Indirect = 4,
  53 };
  54
  55 /* struct sizes */
  56 enum {
  57         VringSize = 4,
  58 };
  59
  60 struct Vring
  61 {
  62         u16int  flags;
  63         u16int  idx;
  64 };
  65
  66 struct Vdesc
  67 {
  68         u64int  addr;
  69         u32int  len;
  70         u16int  flags;
  71         u16int  next;
  72 };
  73
  74 struct Vused
  75 {
  76         u32int  id;
  77         u32int  len;
  78 };
  79
  80 struct Vqueue
  81 {
  82         Lock;
  83
  84         Vdev    *dev;
  85         int     idx;
  86
  87         int     size;
  88
  89         int     free;
  90         int     nfree;
  91
  92         Vdesc   *desc;
  93
  94         Vring   *avail;
  95         u16int  *availent;
  96         u16int  *availevent;
  97
  98         Vring   *used;
  99         Vused   *usedent;
 100         u16int  *usedevent;
 101         u16int  lastused;
 102
 103         void    *rock[];
 104 };
 105
 106 struct Vdev
 107 {
 108         int     typ;
 109
 110         Pcidev  *pci;
 111
 112         ulong   port;
 113         ulong   feat;
 114
 115         int     nqueue;
 116         Vqueue  *queue[16];
 117
 118         void    *cfg;   /* device specific config (for scsi) */
 119
 120         Vdev    *next;
 121 };
 122
 123 enum {
 124         CDBSIZE         = 32,
 125         SENSESIZE       = 96,
 126 };
 127
 128 struct ScsiCfg
 129 {
 130         u32int  num_queues;
 131         u32int  seg_max;
 132         u32int  max_sectors;
 133         u32int  cmd_per_lun;
 134         u32int  event_info_size;
 135         u32int  sense_size;
 136         u32int  cdb_size;
 137         u16int  max_channel;
 138         u16int  max_target;
 139         u32int  max_lun;
 140 };
 141
 142 static Vqueue*
 143 mkvqueue(int size)
 144 {
 145         Vqueue *q;
 146         uchar *p;
 147         int i;
 148
 149         q = malloc(sizeof(*q) + sizeof(void*)*size);
 150         p = mallocalign(
 151                 PGROUND(sizeof(Vdesc)*size +
 152                         VringSize +
 153                         sizeof(u16int)*size +
 154                         sizeof(u16int)) +
 155                 PGROUND(VringSize +
 156                         sizeof(Vused)*size +
 157                         sizeof(u16int)),
 158                 BY2PG, 0, 0);
 159         if(p == nil || q == nil){
 160                 print("virtio: no memory for Vqueue\n");
 161                 free(p);
 162                 free(q);
 163                 return nil;
 164         }
 165
 166         q->desc = (void*)p;
 167         p += sizeof(Vdesc)*size;
 168         q->avail = (void*)p;
 169         p += VringSize;
 170         q->availent = (void*)p;
 171         p += sizeof(u16int)*size;
 172         q->availevent = (void*)p;
 173         p += sizeof(u16int);
 174
 175         p = (uchar*)PGROUND((uintptr)p);
 176         q->used = (void*)p;
 177         p += VringSize;
 178         q->usedent = (void*)p;
 179         p += sizeof(Vused)*size;
 180         q->usedevent = (void*)p;
 181
 182         q->free = -1;
 183         q->nfree = q->size = size;
 184         for(i=0; i<size; i++){
 185                 q->desc[i].next = q->free;
 186                 q->free = i;
 187         }
 188
 189         return q;
 190 }
 191
 192 static Vdev*
 193 viopnpdevs(int typ)
 194 {
 195         Vdev *vd, *h, *t;
 196         Vqueue *q;
 197         Pcidev *p;
 198         int n, i;
 199
 200         h = t = nil;
 201         for(p = nil; p = pcimatch(p, 0x1AF4, 0);){
 202                 if((p->did < 0x1000) || (p->did > 0x103F))
 203                         continue;
 204                 if(p->rid != 0)
 205                         continue;
 206                 if(pcicfgr16(p, 0x2E) != typ)
 207                         continue;
 208                 if((vd = malloc(sizeof(*vd))) == nil){
 209                         print("virtio: no memory for Vdev\n");
 210                         break;
 211                 }
 212                 vd->port = p->mem[0].bar & ~0x1;
 213                 if(ioalloc(vd->port, p->mem[0].size, 0, "virtio") < 0){
 214                         print("virtio: port %lux in use\n", vd->port);
 215                         free(vd);
 216                         continue;
 217                 }
 218                 vd->typ = typ;
 219                 vd->pci = p;
 220                 pcienable(p);
 221
 222                 /* reset */
 223                 outb(vd->port+Status, 0);
 224
 225                 vd->feat = inl(vd->port+Devfeat);
 226                 outb(vd->port+Status, Acknowledge|Driver);
 227                 for(i=0; i<nelem(vd->queue); i++){
 228                         outs(vd->port+Qselect, i);
 229                         n = ins(vd->port+Qsize);
 230                         if(n == 0 || (n & (n-1)) != 0)
 231                                 break;
 232                         if((q = mkvqueue(n)) == nil)
 233                                 break;
 234                         q->dev = vd;
 235                         q->idx = i;
 236                         vd->queue[i] = q;
 237                         coherence();
 238                         outl(vd->port+Qaddr, PADDR(vd->queue[i]->desc)/BY2PG);
 239                 }
 240                 vd->nqueue = i;
 241
 242                 if(h == nil)
 243                         h = vd;
 244                 else
 245                         t->next = vd;
 246                 t = vd;
 247         }
 248
 249         return h;
 250 }
 251
 252 struct Rock {
 253         int done;
 254         Rendez *sleep;
 255 };
 256
 257 static void
 258 vqinterrupt(Vqueue *q)
 259 {
 260         int id, free, m;
 261         struct Rock *r;
 262         Rendez *z;
 263
 264         m = q->size-1;
 265
 266         ilock(q);
 267         while((q->lastused ^ q->used->idx) & m){
 268                 id = q->usedent[q->lastused++ & m].id;
 269                 if(r = q->rock[id]){
 270                         q->rock[id] = nil;
 271                         z = r->sleep;
 272                         r->done = 1;    /* hands off */
 273                         if(z != nil)
 274                                 wakeup(z);
 275                 }
 276                 do {
 277                         free = id;
 278                         id = q->desc[free].next;
 279                         q->desc[free].next = q->free;
 280                         q->free = free;
 281                         q->nfree++;
 282                 } while(q->desc[free].flags & Next);
 283         }
 284         iunlock(q);
 285 }
 286
 287 static void
 288 viointerrupt(Ureg *, void *arg)
 289 {
 290         Vdev *vd = arg;
 291
 292         if(inb(vd->port+Isr) & 1)
 293                 vqinterrupt(vd->queue[vd->typ == TypSCSI ? 2 : 0]);
 294 }
 295
 296 static int
 297 viodone(void *arg)
 298 {
 299         return ((struct Rock*)arg)->done;
 300 }
 301
 302 static void
 303 vqio(Vqueue *q, int head)
 304 {
 305         struct Rock rock;
 306
 307         rock.done = 0;
 308         rock.sleep = &up->sleep;
 309         q->rock[head] = &rock;
 310         q->availent[q->avail->idx & (q->size-1)] = head;
 311         coherence();
 312         q->avail->idx++;
 313         iunlock(q);
 314         if((q->used->flags & 1) == 0)
 315                 outs(q->dev->port+Qnotify, q->idx);
 316         while(!rock.done){
 317                 while(waserror())
 318                         ;
 319                 tsleep(rock.sleep, viodone, &rock, 1000);
 320                 poperror();
 321
 322                 if(!rock.done)
 323                         vqinterrupt(q);
 324         }
 325 }
 326
 327 static int
 328 vioblkreq(Vdev *vd, int typ, void *a, long count, long secsize, uvlong lba)
 329 {
 330         int need, free, head;
 331         Vqueue *q;
 332         Vdesc *d;
 333
 334         u8int status;
 335         struct Vioblkreqhdr {
 336                 u32int  typ;
 337                 u32int  prio;
 338                 u64int  lba;
 339         } req;
 340
 341         need = 2;
 342         if(a != nil)
 343                 need = 3;
 344
 345         status = -1;
 346         req.typ = typ;
 347         req.prio = 0;
 348         req.lba = lba;
 349
 350         q = vd->queue[0];
 351         ilock(q);
 352         while(q->nfree < need){
 353                 iunlock(q);
 354
 355                 if(!waserror())
 356                         tsleep(&up->sleep, return0, 0, 500);
 357                 poperror();
 358
 359                 ilock(q);
 360         }
 361
 362         head = free = q->free;
 363
 364         d = &q->desc[free]; free = d->next;
 365         d->addr = PADDR(&req);
 366         d->len = sizeof(req);
 367         d->flags = Next;
 368
 369         if(a != nil){
 370                 d = &q->desc[free]; free = d->next;
 371                 d->addr = PADDR(a);
 372                 d->len = secsize*count;
 373                 d->flags = typ ? Next : (Write|Next);
 374         }
 375
 376         d = &q->desc[free]; free = d->next;
 377         d->addr = PADDR(&status);
 378         d->len = sizeof(status);
 379         d->flags = Write;
 380
 381         q->free = free;
 382         q->nfree -= need;
 383
 384         /* queue io, unlock and wait for completion */
 385         vqio(q, head);
 386
 387         return status;
 388 }
 389
 390 static int
 391 vioscsireq(SDreq *r)
 392 {
 393         u8int resp[4+4+2+2+SENSESIZE];
 394         u8int req[8+8+3+CDBSIZE];
 395         int free, head;
 396         u32int len;
 397         Vqueue *q;
 398         Vdesc *d;
 399         Vdev *vd;
 400         SDunit *u;
 401         ScsiCfg *cfg;
 402
 403         u = r->unit;
 404         vd = u->dev->ctlr;
 405         cfg = vd->cfg;
 406
 407         memset(resp, 0, sizeof(resp));
 408         memset(req, 0, sizeof(req));
 409         req[0] = 1;
 410         req[1] = u->subno;
 411         req[2] = r->lun>>8;
 412         req[3] = r->lun&0xFF;
 413         *(u64int*)(&req[8]) = (uintptr)r;
 414
 415         memmove(&req[8+8+3], r->cmd, r->clen);
 416
 417         q = vd->queue[2];
 418         ilock(q);
 419         while(q->nfree < 3){
 420                 iunlock(q);
 421
 422                 if(!waserror())
 423                         tsleep(&up->sleep, return0, 0, 500);
 424                 poperror();
 425
 426                 ilock(q);
 427         }
 428
 429         head = free = q->free;
 430
 431         d = &q->desc[free]; free = d->next;
 432         d->addr = PADDR(req);
 433         d->len = 8+8+3+cfg->cdb_size;
 434         d->flags = Next;
 435
 436         if(r->write && r->dlen > 0){
 437                 d = &q->desc[free]; free = d->next;
 438                 d->addr = PADDR(r->data);
 439                 d->len = r->dlen;
 440                 d->flags = Next;
 441         }
 442
 443         d = &q->desc[free]; free = d->next;
 444         d->addr = PADDR(resp);
 445         d->len = 4+4+2+2+cfg->sense_size;
 446         d->flags = Write;
 447
 448         if(!r->write && r->dlen > 0){
 449                 d->flags |= Next;
 450
 451                 d = &q->desc[free]; free = d->next;
 452                 d->addr = PADDR(r->data);
 453                 d->len = r->dlen;
 454                 d->flags = Write;
 455         }
 456
 457         q->free = free;
 458         q->nfree -= 2 + (r->dlen > 0);
 459
 460         /* queue io, unlock and wait for completion */
 461         vqio(q, head);
 462
 463         /* response+status */
 464         r->status = resp[10];
 465         if(resp[11] != 0)
 466                 r->status = SDcheck;
 467
 468         /* sense_len */
 469         len = *((u32int*)&resp[0]);
 470         if(len > 0){
 471                 if(len > sizeof(r->sense))
 472                         len = sizeof(r->sense);
 473                 memmove(r->sense, &resp[4+4+2+2], len);
 474                 r->flags |= SDvalidsense;
 475         }
 476
 477         /* data residue */
 478         len = *((u32int*)&resp[4]);
 479         if(len > r->dlen)
 480                 r->rlen = 0;
 481         else
 482                 r->rlen = r->dlen - len;
 483
 484         return r->status;
 485
 486 }
 487
 488 static long
 489 viobio(SDunit *u, int lun, int write, void *a, long count, uvlong lba)
 490 {
 491         long ss, cc, max, ret;
 492         Vdev *vd;
 493
 494         vd = u->dev->ctlr;
 495         if(vd->typ == TypSCSI)
 496                 return scsibio(u, lun, write, a, count, lba);
 497
 498         max = 32;
 499         ss = u->secsize;
 500         ret = 0;
 501         while(count > 0){
 502                 if((cc = count) > max)
 503                         cc = max;
 504                 if(vioblkreq(vd, write != 0, (uchar*)a + ret, cc, ss, lba) != 0)
 505                         error(Eio);
 506                 ret += cc*ss;
 507                 count -= cc;
 508                 lba += cc;
 509         }
 510         return ret;
 511 }
 512
 513 static int
 514 viorio(SDreq *r)
 515 {
 516         int i, count, rw;
 517         uvlong lba;
 518         SDunit *u;
 519         Vdev *vd;
 520
 521         u = r->unit;
 522         vd = u->dev->ctlr;
 523         if(vd->typ == TypSCSI)
 524                 return vioscsireq(r);
 525         if(r->cmd[0] == 0x35 || r->cmd[0] == 0x91){
 526                 if(vioblkreq(vd, 4, nil, 0, 0, 0) != 0)
 527                         return sdsetsense(r, SDcheck, 3, 0xc, 2);
 528                 return sdsetsense(r, SDok, 0, 0, 0);
 529         }
 530         if((i = sdfakescsi(r)) != SDnostatus)
 531                 return r->status = i;
 532         if((i = sdfakescsirw(r, &lba, &count, &rw)) != SDnostatus)
 533                 return i;
 534         r->rlen = viobio(u, r->lun, rw == SDwrite, r->data, count, lba);
 535         return r->status = SDok;
 536 }
 537
 538 static int
 539 vioonline(SDunit *u)
 540 {
 541         uvlong cap;
 542         Vdev *vd;
 543
 544         vd = u->dev->ctlr;
 545         if(vd->typ == TypSCSI)
 546                 return scsionline(u);
 547
 548         cap = inl(vd->port+Devspec+4);
 549         cap <<= 32;
 550         cap |= inl(vd->port+Devspec);
 551         if(u->sectors != cap){
 552                 u->sectors = cap;
 553                 u->secsize = 512;
 554                 return 2;
 555         }
 556         return 1;
 557 }
 558
 559 static int
 560 vioverify(SDunit *u)
 561 {
 562         Vdev *vd;
 563
 564         vd = u->dev->ctlr;
 565         if(vd->typ == TypSCSI)
 566                 return scsiverify(u);
 567
 568         return 1;
 569 }
 570
 571 SDifc sdvirtioifc;
 572
 573 static int
 574 vioenable(SDev *sd)
 575 {
 576         char name[32];
 577         Vdev *vd;
 578
 579         vd = sd->ctlr;
 580         pcisetbme(vd->pci);
 581         snprint(name, sizeof(name), "%s (%s)", sd->name, sd->ifc->name);
 582         intrenable(vd->pci->intl, viointerrupt, vd, vd->pci->tbdf, name);
 583         outb(vd->port+Status, inb(vd->port+Status) | DriverOk);
 584         return 1;
 585 }
 586
 587 static int
 588 viodisable(SDev *sd)
 589 {
 590         char name[32];
 591         Vdev *vd;
 592
 593         vd = sd->ctlr;
 594         snprint(name, sizeof(name), "%s (%s)", sd->name, sd->ifc->name);
 595         intrdisable(vd->pci->intl, viointerrupt, vd, vd->pci->tbdf, name);
 596         pciclrbme(vd->pci);
 597         return 1;
 598 }
 599
 600 static SDev*
 601 viopnp(void)
 602 {
 603         SDev *s, *h, *t;
 604         Vdev *vd;
 605         int id;
 606
 607         h = t = nil;
 608
 609         id = 'F';
 610         for(vd =  viopnpdevs(TypBlk); vd; vd = vd->next){
 611                 if(vd->nqueue != 1)
 612                         continue;
 613
 614                 if((s = malloc(sizeof(*s))) == nil)
 615                         break;
 616                 s->ctlr = vd;
 617                 s->idno = id++;
 618                 s->ifc = &sdvirtioifc;
 619                 s->nunit = 1;
 620                 if(h)
 621                         t->next = s;
 622                 else
 623                         h = s;
 624                 t = s;
 625         }
 626
 627         id = '0';
 628         for(vd = viopnpdevs(TypSCSI); vd; vd = vd->next){
 629                 ScsiCfg *cfg;
 630
 631                 if(vd->nqueue < 3)
 632                         continue;
 633
 634                 if((cfg = malloc(sizeof(*cfg))) == nil)
 635                         break;
 636                 cfg->num_queues = inl(vd->port+Devspec+4*0);
 637                 cfg->seg_max = inl(vd->port+Devspec+4*1);
 638                 cfg->max_sectors = inl(vd->port+Devspec+4*2);
 639                 cfg->cmd_per_lun = inl(vd->port+Devspec+4*3);
 640                 cfg->event_info_size = inl(vd->port+Devspec+4*4);
 641                 cfg->sense_size = inl(vd->port+Devspec+4*5);
 642                 cfg->cdb_size = inl(vd->port+Devspec+4*6);
 643                 cfg->max_channel = ins(vd->port+Devspec+4*7);
 644                 cfg->max_target = ins(vd->port+Devspec+4*7+2);
 645                 cfg->max_lun = inl(vd->port+Devspec+4*8);
 646
 647                 if(cfg->max_target == 0){
 648                         free(cfg);
 649                         continue;
 650                 }
 651                 if((cfg->cdb_size > CDBSIZE) || (cfg->sense_size > SENSESIZE)){
 652                         print("sdvirtio: cdb %ud or sense size %ud too big\n",
 653                                 cfg->cdb_size, cfg->sense_size);
 654                         free(cfg);
 655                         continue;
 656                 }
 657                 vd->cfg = cfg;
 658
 659                 if((s = malloc(sizeof(*s))) == nil)
 660                         break;
 661                 s->ctlr = vd;
 662                 s->idno = id++;
 663                 s->ifc = &sdvirtioifc;
 664                 s->nunit = cfg->max_target;
 665                 if(h)
 666                         t->next = s;
 667                 else
 668                         h = s;
 669                 t = s;
 670         }
 671
 672         return h;
 673 }
 674
 675 SDifc sdvirtioifc = {
 676         "virtio",                       /* name */
 677
 678         viopnp,                         /* pnp */
 679         nil,                            /* legacy */
 680         vioenable,                      /* enable */
 681         viodisable,                     /* disable */
 682
 683         vioverify,                      /* verify */
 684         vioonline,                      /* online */
 685         viorio,                         /* rio */
 686         nil,                            /* rctl */
 687         nil,                            /* wctl */
 688
 689         viobio,                         /* bio */
 690         nil,                            /* probe */
 691         nil,                            /* clear */
 692         nil,                            /* rtopctl */
 693         nil,                            /* wtopctl */
 694 };