sys/src/9/pc/sdvirtio.c

   1 #include "u.h"
   2 #include "../port/lib.h"
   3 #include "mem.h"
   4 #include "dat.h"
   5 #include "fns.h"
   6 #include "io.h"
   7 #include "../port/pci.h"
   8 #include "ureg.h"
   9 #include "../port/error.h"
  10
  11 #include "../port/sd.h"
  12
  13 typedef struct Vring Vring;
  14 typedef struct Vdesc Vdesc;
  15 typedef struct Vused Vused;
  16 typedef struct Vqueue Vqueue;
  17 typedef struct Vdev Vdev;
  18
  19 typedef struct ScsiCfg ScsiCfg;
  20
  21 /* device types */
  22 enum {
  23         TypBlk  = 2,
  24         TypSCSI = 8,
  25 };
  26
  27 /* status flags */
  28 enum {
  29         Acknowledge = 1,
  30         Driver = 2,
  31         DriverOk = 4,
  32         Failed = 0x80,
  33 };
  34
  35 /* virtio ports */
  36 enum {
  37         Devfeat = 0,
  38         Drvfeat = 4,
  39         Qaddr = 8,
  40         Qsize = 12,
  41         Qselect = 14,
  42         Qnotify = 16,
  43         Status = 18,
  44         Isr = 19,
  45
  46         Devspec = 20,
  47 };
  48
  49 /* descriptor flags */
  50 enum {
  51         Next = 1,
  52         Write = 2,
  53         Indirect = 4,
  54 };
  55
  56 /* struct sizes */
  57 enum {
  58         VringSize = 4,
  59 };
  60
  61 struct Vring
  62 {
  63         u16int  flags;
  64         u16int  idx;
  65 };
  66
  67 struct Vdesc
  68 {
  69         u64int  addr;
  70         u32int  len;
  71         u16int  flags;
  72         u16int  next;
  73 };
  74
  75 struct Vused
  76 {
  77         u32int  id;
  78         u32int  len;
  79 };
  80
  81 struct Vqueue
  82 {
  83         Lock;
  84
  85         Vdev    *dev;
  86         int     idx;
  87
  88         int     size;
  89
  90         int     free;
  91         int     nfree;
  92
  93         Vdesc   *desc;
  94
  95         Vring   *avail;
  96         u16int  *availent;
  97         u16int  *availevent;
  98
  99         Vring   *used;
 100         Vused   *usedent;
 101         u16int  *usedevent;
 102         u16int  lastused;
 103
 104         void    *rock[];
 105 };
 106
 107 struct Vdev
 108 {
 109         int     typ;
 110
 111         Pcidev  *pci;
 112
 113         ulong   port;
 114         ulong   feat;
 115
 116         int     nqueue;
 117         Vqueue  *queue[16];
 118
 119         void    *cfg;   /* device specific config (for scsi) */
 120
 121         Vdev    *next;
 122 };
 123
 124 enum {
 125         CDBSIZE         = 32,
 126         SENSESIZE       = 96,
 127 };
 128
 129 struct ScsiCfg
 130 {
 131         u32int  num_queues;
 132         u32int  seg_max;
 133         u32int  max_sectors;
 134         u32int  cmd_per_lun;
 135         u32int  event_info_size;
 136         u32int  sense_size;
 137         u32int  cdb_size;
 138         u16int  max_channel;
 139         u16int  max_target;
 140         u32int  max_lun;
 141 };
 142
 143 static Vqueue*
 144 mkvqueue(int size)
 145 {
 146         Vqueue *q;
 147         uchar *p;
 148         int i;
 149
 150         q = malloc(sizeof(*q) + sizeof(void*)*size);
 151         p = mallocalign(
 152                 PGROUND(sizeof(Vdesc)*size +
 153                         VringSize +
 154                         sizeof(u16int)*size +
 155                         sizeof(u16int)) +
 156                 PGROUND(VringSize +
 157                         sizeof(Vused)*size +
 158                         sizeof(u16int)),
 159                 BY2PG, 0, 0);
 160         if(p == nil || q == nil){
 161                 print("virtio: no memory for Vqueue\n");
 162                 free(p);
 163                 free(q);
 164                 return nil;
 165         }
 166
 167         q->desc = (void*)p;
 168         p += sizeof(Vdesc)*size;
 169         q->avail = (void*)p;
 170         p += VringSize;
 171         q->availent = (void*)p;
 172         p += sizeof(u16int)*size;
 173         q->availevent = (void*)p;
 174         p += sizeof(u16int);
 175
 176         p = (uchar*)PGROUND((uintptr)p);
 177         q->used = (void*)p;
 178         p += VringSize;
 179         q->usedent = (void*)p;
 180         p += sizeof(Vused)*size;
 181         q->usedevent = (void*)p;
 182
 183         q->free = -1;
 184         q->nfree = q->size = size;
 185         for(i=0; i<size; i++){
 186                 q->desc[i].next = q->free;
 187                 q->free = i;
 188         }
 189
 190         return q;
 191 }
 192
 193 static Vdev*
 194 viopnpdevs(int typ)
 195 {
 196         Vdev *vd, *h, *t;
 197         Vqueue *q;
 198         Pcidev *p;
 199         int n, i;
 200
 201         h = t = nil;
 202         for(p = nil; p = pcimatch(p, 0x1AF4, 0);){
 203                 if((p->did < 0x1000) || (p->did > 0x103F))
 204                         continue;
 205                 if(p->rid != 0)
 206                         continue;
 207                 if((p->mem[0].bar & 1) == 0)
 208                         continue;
 209                 if(pcicfgr16(p, 0x2E) != typ)
 210                         continue;
 211                 if((vd = malloc(sizeof(*vd))) == nil){
 212                         print("virtio: no memory for Vdev\n");
 213                         break;
 214                 }
 215                 vd->port = p->mem[0].bar & ~3;
 216                 if(ioalloc(vd->port, p->mem[0].size, 0, "virtio") < 0){
 217                         print("virtio: port %lux in use\n", vd->port);
 218                         free(vd);
 219                         continue;
 220                 }
 221                 vd->typ = typ;
 222                 vd->pci = p;
 223                 pcienable(p);
 224
 225                 /* reset */
 226                 outb(vd->port+Status, 0);
 227
 228                 vd->feat = inl(vd->port+Devfeat);
 229                 outb(vd->port+Status, Acknowledge|Driver);
 230                 for(i=0; i<nelem(vd->queue); i++){
 231                         outs(vd->port+Qselect, i);
 232                         n = ins(vd->port+Qsize);
 233                         if(n == 0 || (n & (n-1)) != 0)
 234                                 break;
 235                         if((q = mkvqueue(n)) == nil)
 236                                 break;
 237                         q->dev = vd;
 238                         q->idx = i;
 239                         vd->queue[i] = q;
 240                         coherence();
 241                         outl(vd->port+Qaddr, PADDR(vd->queue[i]->desc)/BY2PG);
 242                 }
 243                 vd->nqueue = i;
 244
 245                 if(h == nil)
 246                         h = vd;
 247                 else
 248                         t->next = vd;
 249                 t = vd;
 250         }
 251
 252         return h;
 253 }
 254
 255 struct Rock {
 256         int done;
 257         Rendez *sleep;
 258 };
 259
 260 static void
 261 vqinterrupt(Vqueue *q)
 262 {
 263         int id, free, m;
 264         struct Rock *r;
 265         Rendez *z;
 266
 267         m = q->size-1;
 268
 269         ilock(q);
 270         while((q->lastused ^ q->used->idx) & m){
 271                 id = q->usedent[q->lastused++ & m].id;
 272                 if(r = q->rock[id]){
 273                         q->rock[id] = nil;
 274                         z = r->sleep;
 275                         r->done = 1;    /* hands off */
 276                         if(z != nil)
 277                                 wakeup(z);
 278                 }
 279                 do {
 280                         free = id;
 281                         id = q->desc[free].next;
 282                         q->desc[free].next = q->free;
 283                         q->free = free;
 284                         q->nfree++;
 285                 } while(q->desc[free].flags & Next);
 286         }
 287         iunlock(q);
 288 }
 289
 290 static void
 291 viointerrupt(Ureg *, void *arg)
 292 {
 293         Vdev *vd = arg;
 294
 295         if(inb(vd->port+Isr) & 1)
 296                 vqinterrupt(vd->queue[vd->typ == TypSCSI ? 2 : 0]);
 297 }
 298
 299 static int
 300 viodone(void *arg)
 301 {
 302         return ((struct Rock*)arg)->done;
 303 }
 304
 305 static void
 306 vqio(Vqueue *q, int head)
 307 {
 308         struct Rock rock;
 309
 310         rock.done = 0;
 311         rock.sleep = &up->sleep;
 312         q->rock[head] = &rock;
 313         q->availent[q->avail->idx & (q->size-1)] = head;
 314         coherence();
 315         q->avail->idx++;
 316         iunlock(q);
 317         if((q->used->flags & 1) == 0)
 318                 outs(q->dev->port+Qnotify, q->idx);
 319         while(!rock.done){
 320                 while(waserror())
 321                         ;
 322                 tsleep(rock.sleep, viodone, &rock, 1000);
 323                 poperror();
 324
 325                 if(!rock.done)
 326                         vqinterrupt(q);
 327         }
 328 }
 329
 330 static int
 331 vioblkreq(Vdev *vd, int typ, void *a, long count, long secsize, uvlong lba)
 332 {
 333         int need, free, head;
 334         Vqueue *q;
 335         Vdesc *d;
 336
 337         u8int status;
 338         struct Vioblkreqhdr {
 339                 u32int  typ;
 340                 u32int  prio;
 341                 u64int  lba;
 342         } req;
 343
 344         need = 2;
 345         if(a != nil)
 346                 need = 3;
 347
 348         status = -1;
 349         req.typ = typ;
 350         req.prio = 0;
 351         req.lba = lba;
 352
 353         q = vd->queue[0];
 354         ilock(q);
 355         while(q->nfree < need){
 356                 iunlock(q);
 357
 358                 if(!waserror())
 359                         tsleep(&up->sleep, return0, 0, 500);
 360                 poperror();
 361
 362                 ilock(q);
 363         }
 364
 365         head = free = q->free;
 366
 367         d = &q->desc[free]; free = d->next;
 368         d->addr = PADDR(&req);
 369         d->len = sizeof(req);
 370         d->flags = Next;
 371
 372         if(a != nil){
 373                 d = &q->desc[free]; free = d->next;
 374                 d->addr = PADDR(a);
 375                 d->len = secsize*count;
 376                 d->flags = typ ? Next : (Write|Next);
 377         }
 378
 379         d = &q->desc[free]; free = d->next;
 380         d->addr = PADDR(&status);
 381         d->len = sizeof(status);
 382         d->flags = Write;
 383
 384         q->free = free;
 385         q->nfree -= need;
 386
 387         /* queue io, unlock and wait for completion */
 388         vqio(q, head);
 389
 390         return status;
 391 }
 392
 393 static int
 394 vioscsireq(SDreq *r)
 395 {
 396         u8int resp[4+4+2+2+SENSESIZE];
 397         u8int req[8+8+3+CDBSIZE];
 398         int free, head;
 399         u32int len;
 400         Vqueue *q;
 401         Vdesc *d;
 402         Vdev *vd;
 403         SDunit *u;
 404         ScsiCfg *cfg;
 405
 406         u = r->unit;
 407         vd = u->dev->ctlr;
 408         cfg = vd->cfg;
 409
 410         memset(resp, 0, sizeof(resp));
 411         memset(req, 0, sizeof(req));
 412         req[0] = 1;
 413         req[1] = u->subno;
 414         req[2] = r->lun>>8;
 415         req[3] = r->lun&0xFF;
 416         *(u64int*)(&req[8]) = (uintptr)r;
 417
 418         memmove(&req[8+8+3], r->cmd, r->clen);
 419
 420         q = vd->queue[2];
 421         ilock(q);
 422         while(q->nfree < 3){
 423                 iunlock(q);
 424
 425                 if(!waserror())
 426                         tsleep(&up->sleep, return0, 0, 500);
 427                 poperror();
 428
 429                 ilock(q);
 430         }
 431
 432         head = free = q->free;
 433
 434         d = &q->desc[free]; free = d->next;
 435         d->addr = PADDR(req);
 436         d->len = 8+8+3+cfg->cdb_size;
 437         d->flags = Next;
 438
 439         if(r->write && r->dlen > 0){
 440                 d = &q->desc[free]; free = d->next;
 441                 d->addr = PADDR(r->data);
 442                 d->len = r->dlen;
 443                 d->flags = Next;
 444         }
 445
 446         d = &q->desc[free]; free = d->next;
 447         d->addr = PADDR(resp);
 448         d->len = 4+4+2+2+cfg->sense_size;
 449         d->flags = Write;
 450
 451         if(!r->write && r->dlen > 0){
 452                 d->flags |= Next;
 453
 454                 d = &q->desc[free]; free = d->next;
 455                 d->addr = PADDR(r->data);
 456                 d->len = r->dlen;
 457                 d->flags = Write;
 458         }
 459
 460         q->free = free;
 461         q->nfree -= 2 + (r->dlen > 0);
 462
 463         /* queue io, unlock and wait for completion */
 464         vqio(q, head);
 465
 466         /* response+status */
 467         r->status = resp[10];
 468         if(resp[11] != 0)
 469                 r->status = SDcheck;
 470
 471         /* sense_len */
 472         len = *((u32int*)&resp[0]);
 473         if(len > 0){
 474                 if(len > sizeof(r->sense))
 475                         len = sizeof(r->sense);
 476                 memmove(r->sense, &resp[4+4+2+2], len);
 477                 r->flags |= SDvalidsense;
 478         }
 479
 480         /* data residue */
 481         len = *((u32int*)&resp[4]);
 482         if(len > r->dlen)
 483                 r->rlen = 0;
 484         else
 485                 r->rlen = r->dlen - len;
 486
 487         return r->status;
 488
 489 }
 490
 491 static long
 492 viobio(SDunit *u, int lun, int write, void *a, long count, uvlong lba)
 493 {
 494         long ss, cc, max, ret;
 495         Vdev *vd;
 496
 497         vd = u->dev->ctlr;
 498         if(vd->typ == TypSCSI)
 499                 return scsibio(u, lun, write, a, count, lba);
 500
 501         max = 32;
 502         ss = u->secsize;
 503         ret = 0;
 504         while(count > 0){
 505                 if((cc = count) > max)
 506                         cc = max;
 507                 if(vioblkreq(vd, write != 0, (uchar*)a + ret, cc, ss, lba) != 0)
 508                         error(Eio);
 509                 ret += cc*ss;
 510                 count -= cc;
 511                 lba += cc;
 512         }
 513         return ret;
 514 }
 515
 516 static int
 517 viorio(SDreq *r)
 518 {
 519         int i, count, rw;
 520         uvlong lba;
 521         SDunit *u;
 522         Vdev *vd;
 523
 524         u = r->unit;
 525         vd = u->dev->ctlr;
 526         if(vd->typ == TypSCSI)
 527                 return vioscsireq(r);
 528         if(r->cmd[0] == 0x35 || r->cmd[0] == 0x91){
 529                 if(vioblkreq(vd, 4, nil, 0, 0, 0) != 0)
 530                         return sdsetsense(r, SDcheck, 3, 0xc, 2);
 531                 return sdsetsense(r, SDok, 0, 0, 0);
 532         }
 533         if((i = sdfakescsi(r)) != SDnostatus)
 534                 return r->status = i;
 535         if((i = sdfakescsirw(r, &lba, &count, &rw)) != SDnostatus)
 536                 return i;
 537         r->rlen = viobio(u, r->lun, rw == SDwrite, r->data, count, lba);
 538         return r->status = SDok;
 539 }
 540
 541 static int
 542 vioonline(SDunit *u)
 543 {
 544         uvlong cap;
 545         Vdev *vd;
 546
 547         vd = u->dev->ctlr;
 548         if(vd->typ == TypSCSI)
 549                 return scsionline(u);
 550
 551         cap = inl(vd->port+Devspec+4);
 552         cap <<= 32;
 553         cap |= inl(vd->port+Devspec);
 554         if(u->sectors != cap){
 555                 u->sectors = cap;
 556                 u->secsize = 512;
 557                 return 2;
 558         }
 559         return 1;
 560 }
 561
 562 static int
 563 vioverify(SDunit *u)
 564 {
 565         Vdev *vd;
 566
 567         vd = u->dev->ctlr;
 568         if(vd->typ == TypSCSI)
 569                 return scsiverify(u);
 570
 571         return 1;
 572 }
 573
 574 SDifc sdvirtioifc;
 575
 576 static int
 577 vioenable(SDev *sd)
 578 {
 579         char name[32];
 580         Vdev *vd;
 581
 582         vd = sd->ctlr;
 583         pcisetbme(vd->pci);
 584         snprint(name, sizeof(name), "%s (%s)", sd->name, sd->ifc->name);
 585         intrenable(vd->pci->intl, viointerrupt, vd, vd->pci->tbdf, name);
 586         outb(vd->port+Status, inb(vd->port+Status) | DriverOk);
 587         return 1;
 588 }
 589
 590 static int
 591 viodisable(SDev *sd)
 592 {
 593         char name[32];
 594         Vdev *vd;
 595
 596         vd = sd->ctlr;
 597         snprint(name, sizeof(name), "%s (%s)", sd->name, sd->ifc->name);
 598         intrdisable(vd->pci->intl, viointerrupt, vd, vd->pci->tbdf, name);
 599         pciclrbme(vd->pci);
 600         return 1;
 601 }
 602
 603 static SDev*
 604 viopnp(void)
 605 {
 606         SDev *s, *h, *t;
 607         Vdev *vd;
 608         int id;
 609
 610         h = t = nil;
 611
 612         id = 'F';
 613         for(vd =  viopnpdevs(TypBlk); vd; vd = vd->next){
 614                 if(vd->nqueue == 0)
 615                         continue;
 616
 617                 if((s = malloc(sizeof(*s))) == nil)
 618                         break;
 619                 s->ctlr = vd;
 620                 s->idno = id++;
 621                 s->ifc = &sdvirtioifc;
 622                 s->nunit = 1;
 623                 if(h)
 624                         t->next = s;
 625                 else
 626                         h = s;
 627                 t = s;
 628         }
 629
 630         id = '0';
 631         for(vd = viopnpdevs(TypSCSI); vd; vd = vd->next){
 632                 ScsiCfg *cfg;
 633
 634                 if(vd->nqueue < 3)
 635                         continue;
 636
 637                 if((cfg = malloc(sizeof(*cfg))) == nil)
 638                         break;
 639                 cfg->num_queues = inl(vd->port+Devspec+4*0);
 640                 cfg->seg_max = inl(vd->port+Devspec+4*1);
 641                 cfg->max_sectors = inl(vd->port+Devspec+4*2);
 642                 cfg->cmd_per_lun = inl(vd->port+Devspec+4*3);
 643                 cfg->event_info_size = inl(vd->port+Devspec+4*4);
 644                 cfg->sense_size = inl(vd->port+Devspec+4*5);
 645                 cfg->cdb_size = inl(vd->port+Devspec+4*6);
 646                 cfg->max_channel = ins(vd->port+Devspec+4*7);
 647                 cfg->max_target = ins(vd->port+Devspec+4*7+2);
 648                 cfg->max_lun = inl(vd->port+Devspec+4*8);
 649
 650                 if(cfg->max_target == 0){
 651                         free(cfg);
 652                         continue;
 653                 }
 654                 if((cfg->cdb_size > CDBSIZE) || (cfg->sense_size > SENSESIZE)){
 655                         print("sdvirtio: cdb %ud or sense size %ud too big\n",
 656                                 cfg->cdb_size, cfg->sense_size);
 657                         free(cfg);
 658                         continue;
 659                 }
 660                 vd->cfg = cfg;
 661
 662                 if((s = malloc(sizeof(*s))) == nil)
 663                         break;
 664                 s->ctlr = vd;
 665                 s->idno = id++;
 666                 s->ifc = &sdvirtioifc;
 667                 s->nunit = cfg->max_target;
 668                 if(h)
 669                         t->next = s;
 670                 else
 671                         h = s;
 672                 t = s;
 673         }
 674
 675         return h;
 676 }
 677
 678 SDifc sdvirtioifc = {
 679         "virtio",                       /* name */
 680
 681         viopnp,                         /* pnp */
 682         nil,                            /* legacy */
 683         vioenable,                      /* enable */
 684         viodisable,                     /* disable */
 685
 686         vioverify,                      /* verify */
 687         vioonline,                      /* online */
 688         viorio,                         /* rio */
 689         nil,                            /* rctl */
 690         nil,                            /* wctl */
 691
 692         viobio,                         /* bio */
 693         nil,                            /* probe */
 694         nil,                            /* clear */
 695         nil,                            /* rtopctl */
 696         nil,                            /* wtopctl */
 697 };