sys/src/9/pc/sdvirtio.c

   1 #include "u.h"
   2 #include "../port/lib.h"
   3 #include "mem.h"
   4 #include "dat.h"
   5 #include "fns.h"
   6 #include "io.h"
   7 #include "ureg.h"
   8 #include "../port/error.h"
   9
  10 #include "../port/sd.h"
  11
  12 typedef struct Vring Vring;
  13 typedef struct Vdesc Vdesc;
  14 typedef struct Vused Vused;
  15 typedef struct Vqueue Vqueue;
  16 typedef struct Vdev Vdev;
  17
  18 typedef struct ScsiCfg ScsiCfg;
  19
  20 /* device types */
  21 enum {
  22         TypBlk  = 2,
  23         TypSCSI = 8,
  24 };
  25
  26 /* status flags */
  27 enum {
  28         Acknowledge = 1,
  29         Driver = 2,
  30         DriverOk = 4,
  31         Failed = 0x80,
  32 };
  33
  34 /* virtio ports */
  35 enum {
  36         Devfeat = 0,
  37         Drvfeat = 4,
  38         Qaddr = 8,
  39         Qsize = 12,
  40         Qselect = 14,
  41         Qnotify = 16,
  42         Status = 18,
  43         Isr = 19,
  44
  45         Devspec = 20,
  46 };
  47
  48 /* descriptor flags */
  49 enum {
  50         Next = 1,
  51         Write = 2,
  52         Indirect = 4,
  53 };
  54
  55 /* struct sizes */
  56 enum {
  57         VringSize = 4,
  58 };
  59
  60 struct Vring
  61 {
  62         u16int  flags;
  63         u16int  idx;
  64 };
  65
  66 struct Vdesc
  67 {
  68         u64int  addr;
  69         u32int  len;
  70         u16int  flags;
  71         u16int  next;
  72 };
  73
  74 struct Vused
  75 {
  76         u32int  id;
  77         u32int  len;
  78 };
  79
  80 struct Vqueue
  81 {
  82         Lock;
  83
  84         Vdev    *dev;
  85         int     idx;
  86
  87         int     size;
  88
  89         int     free;
  90         int     nfree;
  91
  92         Vdesc   *desc;
  93
  94         Vring   *avail;
  95         u16int  *availent;
  96         u16int  *availevent;
  97
  98         Vring   *used;
  99         Vused   *usedent;
 100         u16int  *usedevent;
 101         u16int  lastused;
 102
 103         void    *rock[];
 104 };
 105
 106 struct Vdev
 107 {
 108         int     typ;
 109
 110         Pcidev  *pci;
 111
 112         ulong   port;
 113         ulong   feat;
 114
 115         int     nqueue;
 116         Vqueue  *queue[16];
 117
 118         void    *cfg;   /* device specific config (for scsi) */
 119
 120         Vdev    *next;
 121 };
 122
 123 enum {
 124         CDBSIZE         = 32,
 125         SENSESIZE       = 96,
 126 };
 127
 128 struct ScsiCfg
 129 {
 130         u32int  num_queues;
 131         u32int  seg_max;
 132         u32int  max_sectors;
 133         u32int  cmd_per_lun;
 134         u32int  event_info_size;
 135         u32int  sense_size;
 136         u32int  cdb_size;
 137         u16int  max_channel;
 138         u16int  max_target;
 139         u32int  max_lun;
 140 };
 141
 142 static Vqueue*
 143 mkvqueue(int size)
 144 {
 145         Vqueue *q;
 146         uchar *p;
 147         int i;
 148
 149         q = malloc(sizeof(*q) + sizeof(void*)*size);
 150         p = mallocalign(
 151                 PGROUND(sizeof(Vdesc)*size +
 152                         VringSize +
 153                         sizeof(u16int)*size +
 154                         sizeof(u16int)) +
 155                 PGROUND(VringSize +
 156                         sizeof(Vused)*size +
 157                         sizeof(u16int)),
 158                 BY2PG, 0, 0);
 159         if(p == nil || q == nil){
 160                 print("virtio: no memory for Vqueue\n");
 161                 free(p);
 162                 free(q);
 163                 return nil;
 164         }
 165
 166         q->desc = (void*)p;
 167         p += sizeof(Vdesc)*size;
 168         q->avail = (void*)p;
 169         p += VringSize;
 170         q->availent = (void*)p;
 171         p += sizeof(u16int)*size;
 172         q->availevent = (void*)p;
 173         p += sizeof(u16int);
 174
 175         p = (uchar*)PGROUND((uintptr)p);
 176         q->used = (void*)p;
 177         p += VringSize;
 178         q->usedent = (void*)p;
 179         p += sizeof(Vused)*size;
 180         q->usedevent = (void*)p;
 181
 182         q->free = -1;
 183         q->nfree = q->size = size;
 184         for(i=0; i<size; i++){
 185                 q->desc[i].next = q->free;
 186                 q->free = i;
 187         }
 188
 189         return q;
 190 }
 191
 192 static Vdev*
 193 viopnpdevs(int typ)
 194 {
 195         Vdev *vd, *h, *t;
 196         Vqueue *q;
 197         Pcidev *p;
 198         int n, i;
 199
 200         h = t = nil;
 201         for(p = nil; p = pcimatch(p, 0x1AF4, 0);){
 202                 if((p->did < 0x1000) || (p->did > 0x103F))
 203                         continue;
 204                 if(p->rid != 0)
 205                         continue;
 206                 if(pcicfgr16(p, 0x2E) != typ)
 207                         continue;
 208                 if((vd = malloc(sizeof(*vd))) == nil){
 209                         print("virtio: no memory for Vdev\n");
 210                         break;
 211                 }
 212                 vd->port = p->mem[0].bar & ~0x1;
 213                 if(ioalloc(vd->port, p->mem[0].size, 0, "virtio") < 0){
 214                         print("virtio: port %lux in use\n", vd->port);
 215                         free(vd);
 216                         continue;
 217                 }
 218                 vd->typ = typ;
 219                 vd->pci = p;
 220
 221                 /* reset */
 222                 outb(vd->port+Status, 0);
 223
 224                 vd->feat = inl(vd->port+Devfeat);
 225                 outb(vd->port+Status, Acknowledge|Driver);
 226                 for(i=0; i<nelem(vd->queue); i++){
 227                         outs(vd->port+Qselect, i);
 228                         n = ins(vd->port+Qsize);
 229                         if(n == 0 || (n & (n-1)) != 0)
 230                                 break;
 231                         if((q = mkvqueue(n)) == nil)
 232                                 break;
 233                         q->dev = vd;
 234                         q->idx = i;
 235                         vd->queue[i] = q;
 236                         coherence();
 237                         outl(vd->port+Qaddr, PADDR(vd->queue[i]->desc)/BY2PG);
 238                 }
 239                 vd->nqueue = i;
 240
 241                 if(h == nil)
 242                         h = vd;
 243                 else
 244                         t->next = vd;
 245                 t = vd;
 246         }
 247
 248         return h;
 249 }
 250
 251 struct Rock {
 252         int done;
 253         Rendez *sleep;
 254 };
 255
 256 static void
 257 vqinterrupt(Vqueue *q)
 258 {
 259         int id, free, m;
 260         struct Rock *r;
 261         Rendez *z;
 262
 263         m = q->size-1;
 264
 265         ilock(q);
 266         while((q->lastused ^ q->used->idx) & m){
 267                 id = q->usedent[q->lastused++ & m].id;
 268                 if(r = q->rock[id]){
 269                         q->rock[id] = nil;
 270                         z = r->sleep;
 271                         r->done = 1;    /* hands off */
 272                         if(z != nil)
 273                                 wakeup(z);
 274                 }
 275                 do {
 276                         free = id;
 277                         id = q->desc[free].next;
 278                         q->desc[free].next = q->free;
 279                         q->free = free;
 280                         q->nfree++;
 281                 } while(q->desc[free].flags & Next);
 282         }
 283         iunlock(q);
 284 }
 285
 286 static void
 287 viointerrupt(Ureg *, void *arg)
 288 {
 289         Vdev *vd = arg;
 290
 291         if(inb(vd->port+Isr) & 1)
 292                 vqinterrupt(vd->queue[vd->typ == TypSCSI ? 2 : 0]);
 293 }
 294
 295 static int
 296 viodone(void *arg)
 297 {
 298         return ((struct Rock*)arg)->done;
 299 }
 300
 301 static int
 302 vioblkreq(Vdev *vd, int typ, void *a, long count, long secsize, uvlong lba)
 303 {
 304         struct Rock rock;
 305         int free, head;
 306         Vqueue *q;
 307         Vdesc *d;
 308
 309         u8int status;
 310         struct Vioblkreqhdr {
 311                 u32int  typ;
 312                 u32int  prio;
 313                 u64int  lba;
 314         } req;
 315
 316         status = 0;
 317         req.typ = typ;
 318         req.prio = 0;
 319         req.lba = lba;
 320
 321         rock.done = 0;
 322         rock.sleep = &up->sleep;
 323
 324         q = vd->queue[0];
 325         ilock(q);
 326         while(q->nfree < 3){
 327                 iunlock(q);
 328
 329                 if(!waserror())
 330                         tsleep(&up->sleep, return0, 0, 500);
 331                 poperror();
 332
 333                 ilock(q);
 334         }
 335
 336         head = free = q->free;
 337
 338         d = &q->desc[free]; free = d->next;
 339         d->addr = PADDR(&req);
 340         d->len = sizeof(req);
 341         d->flags = Next;
 342
 343         d = &q->desc[free]; free = d->next;
 344         d->addr = PADDR(a);
 345         d->len = secsize*count;
 346         d->flags = typ ? Next : (Write|Next);
 347
 348         d = &q->desc[free]; free = d->next;
 349         d->addr = PADDR(&status);
 350         d->len = sizeof(status);
 351         d->flags = Write;
 352
 353         q->free = free;
 354         q->nfree -= 3;
 355
 356         q->rock[head] = &rock;
 357
 358         coherence();
 359         q->availent[q->avail->idx++ & (q->size-1)] = head;
 360         coherence();
 361         outs(vd->port+Qnotify, q->idx);
 362         iunlock(q);
 363
 364         while(!rock.done){
 365                 while(waserror())
 366                         ;
 367                 tsleep(rock.sleep, viodone, &rock, 1000);
 368                 poperror();
 369
 370                 if(!rock.done)
 371                         vqinterrupt(q);
 372         }
 373
 374         return status;
 375 }
 376
 377 static int
 378 vioscsireq(SDreq *r)
 379 {
 380         u8int resp[4+4+2+2+SENSESIZE];
 381         u8int req[8+8+3+CDBSIZE];
 382         struct Rock rock;
 383         int free, head;
 384         u32int len;
 385         Vqueue *q;
 386         Vdesc *d;
 387         Vdev *vd;
 388         SDunit *u;
 389         ScsiCfg *cfg;
 390
 391         u = r->unit;
 392         vd = u->dev->ctlr;
 393         cfg = vd->cfg;
 394
 395         memset(resp, 0, sizeof(resp));
 396         memset(req, 0, sizeof(req));
 397         req[0] = 1;
 398         req[1] = u->subno;
 399         req[2] = r->lun>>8;
 400         req[3] = r->lun&0xFF;
 401         *(u64int*)(&req[8]) = (uintptr)r;
 402
 403         memmove(&req[8+8+3], r->cmd, r->clen);
 404
 405         rock.done = 0;
 406         rock.sleep = &up->sleep;
 407
 408         q = vd->queue[2];
 409         ilock(q);
 410         while(q->nfree < 3){
 411                 iunlock(q);
 412
 413                 if(!waserror())
 414                         tsleep(&up->sleep, return0, 0, 500);
 415                 poperror();
 416
 417                 ilock(q);
 418         }
 419
 420         head = free = q->free;
 421
 422         d = &q->desc[free]; free = d->next;
 423         d->addr = PADDR(req);
 424         d->len = 8+8+3+cfg->cdb_size;
 425         d->flags = Next;
 426
 427         if(r->write && r->dlen > 0){
 428                 d = &q->desc[free]; free = d->next;
 429                 d->addr = PADDR(r->data);
 430                 d->len = r->dlen;
 431                 d->flags = Next;
 432         }
 433
 434         d = &q->desc[free]; free = d->next;
 435         d->addr = PADDR(resp);
 436         d->len = 4+4+2+2+cfg->sense_size;
 437         d->flags = Write;
 438
 439         if(!r->write && r->dlen > 0){
 440                 d->flags |= Next;
 441
 442                 d = &q->desc[free]; free = d->next;
 443                 d->addr = PADDR(r->data);
 444                 d->len = r->dlen;
 445                 d->flags = Write;
 446         }
 447
 448         q->free = free;
 449         q->nfree -= 2 + (r->dlen > 0);
 450
 451         q->rock[head] = &rock;
 452
 453         coherence();
 454         q->availent[q->avail->idx++ & (q->size-1)] = head;
 455         coherence();
 456         outs(vd->port+Qnotify, q->idx);
 457         iunlock(q);
 458
 459         while(!rock.done){
 460                 while(waserror())
 461                         ;
 462                 tsleep(rock.sleep, viodone, &rock, 1000);
 463                 poperror();
 464
 465                 if(!rock.done)
 466                         vqinterrupt(q);
 467         }
 468
 469         /* response+status */
 470         r->status = resp[10];
 471         if(resp[11] != 0)
 472                 r->status = SDcheck;
 473
 474         /* sense_len */
 475         len = *((u32int*)&resp[0]);
 476         if(len > 0){
 477                 if(len > sizeof(r->sense))
 478                         len = sizeof(r->sense);
 479                 memmove(r->sense, &resp[4+4+2+2], len);
 480                 r->flags |= SDvalidsense;
 481         }
 482
 483         /* data residue */
 484         len = *((u32int*)&resp[4]);
 485         if(len > r->dlen)
 486                 r->rlen = 0;
 487         else
 488                 r->rlen = r->dlen - len;
 489
 490         return r->status;
 491
 492 }
 493
 494 static long
 495 viobio(SDunit *u, int lun, int write, void *a, long count, uvlong lba)
 496 {
 497         long ss, cc, max, ret;
 498         Vdev *vd;
 499
 500         vd = u->dev->ctlr;
 501         if(vd->typ == TypSCSI)
 502                 return scsibio(u, lun, write, a, count, lba);
 503
 504         max = 32;
 505         ss = u->secsize;
 506         ret = 0;
 507         while(count > 0){
 508                 if((cc = count) > max)
 509                         cc = max;
 510                 if(vioblkreq(vd, write != 0, (uchar*)a + ret, cc, ss, lba) != 0)
 511                         error(Eio);
 512                 ret += cc*ss;
 513                 count -= cc;
 514                 lba += cc;
 515         }
 516         return ret;
 517 }
 518
 519 static int
 520 viorio(SDreq *r)
 521 {
 522         int i, count, rw;
 523         uvlong lba;
 524         SDunit *u;
 525         Vdev *vd;
 526
 527         u = r->unit;
 528         vd = u->dev->ctlr;
 529         if(vd->typ == TypSCSI)
 530                 return vioscsireq(r);
 531         if(r->cmd[0] == 0x35 || r->cmd[0] == 0x91){
 532                 if(vioblkreq(vd, 4, nil, 0, 0, 0) != 0)
 533                         return sdsetsense(r, SDcheck, 3, 0xc, 2);
 534                 return sdsetsense(r, SDok, 0, 0, 0);
 535         }
 536         if((i = sdfakescsi(r)) != SDnostatus)
 537                 return r->status = i;
 538         if((i = sdfakescsirw(r, &lba, &count, &rw)) != SDnostatus)
 539                 return i;
 540         r->rlen = viobio(u, r->lun, rw == SDwrite, r->data, count, lba);
 541         return r->status = SDok;
 542 }
 543
 544 static int
 545 vioonline(SDunit *u)
 546 {
 547         uvlong cap;
 548         Vdev *vd;
 549
 550         vd = u->dev->ctlr;
 551         if(vd->typ == TypSCSI)
 552                 return scsionline(u);
 553
 554         cap = inl(vd->port+Devspec+4);
 555         cap <<= 32;
 556         cap |= inl(vd->port+Devspec);
 557         if(u->sectors != cap){
 558                 u->sectors = cap;
 559                 u->secsize = 512;
 560                 return 2;
 561         }
 562         return 1;
 563 }
 564
 565 static int
 566 vioverify(SDunit *u)
 567 {
 568         Vdev *vd;
 569
 570         vd = u->dev->ctlr;
 571         if(vd->typ == TypSCSI)
 572                 return scsiverify(u);
 573
 574         return 1;
 575 }
 576
 577 SDifc sdvirtioifc;
 578
 579 static void
 580 vdevenable(Vdev *vd)
 581 {
 582         intrenable(vd->pci->intl, viointerrupt, vd, vd->pci->tbdf, "virtio");
 583         outb(vd->port+Status, inb(vd->port+Status) | DriverOk);
 584 }
 585
 586 static SDev*
 587 viopnp(void)
 588 {
 589         SDev *s, *h, *t;
 590         Vdev *vd;
 591         int id;
 592
 593         h = t = nil;
 594
 595         id = 'F';
 596         for(vd =  viopnpdevs(TypBlk); vd; vd = vd->next){
 597                 if(vd->nqueue != 1)
 598                         continue;
 599
 600                 vdevenable(vd);
 601
 602                 if((s = malloc(sizeof(*s))) == nil)
 603                         break;
 604                 s->ctlr = vd;
 605                 s->idno = id++;
 606                 s->ifc = &sdvirtioifc;
 607                 s->nunit = 1;
 608                 if(h)
 609                         t->next = s;
 610                 else
 611                         h = s;
 612                 t = s;
 613         }
 614
 615         id = '0';
 616         for(vd = viopnpdevs(TypSCSI); vd; vd = vd->next){
 617                 ScsiCfg *cfg;
 618
 619                 if(vd->nqueue < 3)
 620                         continue;
 621
 622                 if((cfg = malloc(sizeof(*cfg))) == nil)
 623                         break;
 624                 cfg->num_queues = inl(vd->port+Devspec+4*0);
 625                 cfg->seg_max = inl(vd->port+Devspec+4*1);
 626                 cfg->max_sectors = inl(vd->port+Devspec+4*2);
 627                 cfg->cmd_per_lun = inl(vd->port+Devspec+4*3);
 628                 cfg->event_info_size = inl(vd->port+Devspec+4*4);
 629                 cfg->sense_size = inl(vd->port+Devspec+4*5);
 630                 cfg->cdb_size = inl(vd->port+Devspec+4*6);
 631                 cfg->max_channel = ins(vd->port+Devspec+4*7);
 632                 cfg->max_target = ins(vd->port+Devspec+4*7+2);
 633                 cfg->max_lun = inl(vd->port+Devspec+4*8);
 634
 635                 if(cfg->max_target == 0){
 636                         free(cfg);
 637                         continue;
 638                 }
 639                 if((cfg->cdb_size > CDBSIZE) || (cfg->sense_size > SENSESIZE)){
 640                         print("sdvirtio: cdb %ud or sense size %ud too big\n",
 641                                 cfg->cdb_size, cfg->sense_size);
 642                         free(cfg);
 643                         continue;
 644                 }
 645                 vd->cfg = cfg;
 646
 647                 vdevenable(vd);
 648
 649                 if((s = malloc(sizeof(*s))) == nil)
 650                         break;
 651                 s->ctlr = vd;
 652                 s->idno = id++;
 653                 s->ifc = &sdvirtioifc;
 654                 s->nunit = cfg->max_target;
 655                 if(h)
 656                         t->next = s;
 657                 else
 658                         h = s;
 659                 t = s;
 660         }
 661
 662         return h;
 663 }
 664
 665 SDifc sdvirtioifc = {
 666         "virtio",                       /* name */
 667
 668         viopnp,                         /* pnp */
 669         nil,                            /* legacy */
 670         nil,                            /* enable */
 671         nil,                            /* disable */
 672
 673         vioverify,                      /* verify */
 674         vioonline,                      /* online */
 675         viorio,                         /* rio */
 676         nil,                            /* rctl */
 677         nil,                            /* wctl */
 678
 679         viobio,                         /* bio */
 680         nil,                            /* probe */
 681         nil,                            /* clear */
 682         nil,                            /* rtopctl */
 683         nil,                            /* wtopctl */
 684 };