sys/src/9/pc/usbxhci.c

   1 #include        "u.h"
   2 #include        "../port/lib.h"
   3 #include        "mem.h"
   4 #include        "dat.h"
   5 #include        "fns.h"
   6 #include        "io.h"
   7 #include        "../port/error.h"
   8
   9 #include        "../port/usb.h"
  10
  11 enum {
  12         /* Capability Registers */
  13         CAPLENGTH       = 0x00/4,       // 1
  14         HCIVERSION      = 0x02/4,       // 2
  15         HCSPARAMS1      = 0x04/4,
  16         HCSPARAMS2      = 0x08/4,
  17         HCSPARAMS3      = 0x0C/4,
  18
  19         HCCPARAMS       = 0x10/4,
  20                 AC64    = 1<<0,
  21                 BNC     = 1<<1,
  22                 CSZ     = 1<<2,
  23                 PPC     = 1<<3,
  24                 PIND    = 1<<4,
  25                 LHRC    = 1<<5,
  26                 LTC     = 1<<6,
  27                 NSS     = 1<<7,
  28
  29         DBOFF           = 0x14/4,
  30         RTSOFF          = 0x18/4,
  31
  32         /* Operational Registers */
  33         USBCMD          = 0x00/4,       /* USB Command Register */
  34                 RUNSTOP = 1<<0,         /* Run/Stop - RW */
  35                 HCRST   = 1<<1,         /* Host Controller Reset - RW */
  36                 INTE    = 1<<2,         /* Interrupter Enable - RW */
  37                 HSEE    = 1<<3,         /* Host System Error Enable - RW */
  38                 LHCRST  = 1<<7,         /* Light Host Controller Reset - RO/RW */
  39                 CSS     = 1<<8,         /* Controller Save State - RW */
  40                 CRS     = 1<<9,         /* Controller Restore State - RW */
  41                 EWE     = 1<<10,        /* Enable Wrap Event - RW */
  42                 EU3S    = 1<<11,        /* Enable U3 MFINDEX Stop - RW */
  43
  44         USBSTS          = 0x04/4,       /* USB Status Register */
  45                 HCH     = 1<<0,         /* HCHalted - RO */
  46                 HSE     = 1<<2,         /* Host System Error - RW1C */
  47                 EINT    = 1<<3,         /* Event Interrupt - RW1C */
  48                 PCD     = 1<<4,         /* Port Change Detect - RW1C */
  49                 SSS     = 1<<8,         /* Save State Status - RO */
  50                 RSS     = 1<<9,         /* Restore State Status - RO */
  51                 SRE     = 1<<10,        /* Save/Restore Error - RW1C */
  52                 CNR     = 1<<11,        /* Controller Not Ready - RO */
  53                 HCE     = 1<<12,        /* Host Controller Error - RO */
  54
  55         PAGESIZE        = 0x08/4,       /* Page Size - RO */
  56
  57         DNCTRL          = 0x14/4,       /* Device Notification Control Register - RW */
  58
  59         CRCR            = 0x18/4,       /* Command Ring Control Register - RW */
  60                 RCS     = 1<<0,         /* Ring Cycle State - RW */
  61                 CS      = 1<<1,         /* Command Stop - RW1S */
  62                 CA      = 1<<2,         /* Command Abort - RW1S */
  63                 CRR     = 1<<3,         /* Command Ring Running - RO */
  64
  65         DCBAAP          = 0x30/4,       // 8
  66
  67         CONFIG          = 0x38/4,       /* Configure Register (MaxSlotEn[7:0]) */
  68
  69         /* Port Register Set */
  70         PORTSC          = 0x00/4,       /* Port tatus and Control Register */
  71                 CCS     = 1<<0,         /* Current Connect Status - ROS */
  72                 PED     = 1<<1,         /* Port Enable/Disabled - RW1CS */
  73                 OCA     = 1<<3,         /* Over-current Active - RO */
  74                 PR      = 1<<4,         /* Port Reset - RW1S */
  75                 PLS     = 15<<5,        /* Port Link State - RWS */
  76                 PP      = 1<<9,         /* Port Power - RWS */
  77                 PS      = 15<<10,       /* Port Speed - ROS */
  78                 PIC     = 3<<14,        /* Port Indicator Control - RWS */
  79                 LWS     = 1<<16,        /* Port Link Write Strobe - RW */
  80                 CSC     = 1<<17,        /* Connect Status Change - RW1CS */
  81                 PEC     = 1<<18,        /* Port Enabled/Disabled Change - RW1CS */
  82                 WRC     = 1<<19,        /* Warm Port Reset Change - RW1CS */
  83                 OCC     = 1<<20,        /* Over-current Change - RW1CS */
  84                 PRC     = 1<<21,        /* Port Reset Change - RW1CS */
  85                 PLC     = 1<<22,        /* Port Link State Change - RW1CS */
  86                 CEC     = 1<<23,        /* Port Config Error Change - RW1CS */
  87                 CAS     = 1<<24,        /* Cold Attach Status - RO */
  88                 WCE     = 1<<25,        /* Wake on Connect Enable - RWS */
  89                 WDE     = 1<<26,        /* Wake on Disconnect Enable - RWS */
  90                 WOE     = 1<<27,        /* Wake on Over-current Enable - RWS */
  91                 DR      = 1<<30,        /* Device Removable - RO */
  92                 WPR     = 1<<31,        /* Warm Port Reset - RW1S */
  93
  94         PORTPMSC        = 0x04/4,
  95         PORTLI          = 0x08/4,
  96
  97         /* Host Controller Runtime Register */
  98         MFINDEX         = 0x0000/4,     /* Microframe Index */
  99         IR0             = 0x0020/4,     /* Interrupt Register Set 0 */
 100
 101         /* Interrupter Registers */
 102         IMAN            = 0x00/4,       /* Interrupter Management */
 103         IMOD            = 0x04/4,       /* Interrupter Moderation */
 104         ERSTSZ          = 0x08/4,       /* Event Ring Segment Table Size */
 105         ERSTBA          = 0x10/4,       /* Event Ring Segment Table Base Address */
 106         ERDP            = 0x18/4,       /* Event Ring Dequeue Pointer */
 107
 108         /* TRB flags */
 109         TR_ENT          = 1<<1,
 110         TR_ISP          = 1<<2,
 111         TR_NS           = 1<<3,
 112         TR_CH           = 1<<4,
 113         TR_IOC          = 1<<5,
 114         TR_IDT          = 1<<6,
 115         TR_BEI          = 1<<9,
 116
 117         /* TRB types */
 118         TR_RESERVED     = 0<<10,
 119         TR_NORMAL       = 1<<10,
 120         TR_SETUPSTAGE   = 2<<10,
 121         TR_DATASTAGE    = 3<<10,
 122         TR_STATUSSTAGE  = 4<<10,
 123         TR_ISOCH        = 5<<10,
 124         TR_LINK         = 6<<10,
 125         TR_EVENTDATA    = 7<<10,
 126         TR_NOOP         = 8<<10,
 127
 128         CR_ENABLESLOT   = 9<<10,
 129         CR_DISABLESLOT  = 10<<10,
 130         CR_ADDRESSDEV   = 11<<10,
 131         CR_CONFIGEP     = 12<<10,
 132         CR_EVALCTX      = 13<<10,
 133         CR_RESETEP      = 14<<10,
 134         CR_STOPEP       = 15<<10,
 135         CR_SETTRDQP     = 16<<10,
 136         CR_RESETDEV     = 17<<10,
 137         CR_FORCECMD     = 18<<10,
 138         CR_NEGBW        = 19<<10,
 139         CR_SETLAT       = 20<<10,
 140         CR_GETPORTBW    = 21<<10,
 141         CR_FORCEHDR     = 22<<10,
 142         CR_NOOP         = 23<<10,
 143
 144         ER_TRANSFER     = 32<<10,
 145         ER_CMDCOMPL     = 33<<10,
 146         ER_PORTSC       = 34<<10,
 147         ER_BWREQ        = 35<<10,
 148         ER_DOORBELL     = 36<<10,
 149         ER_HCE          = 37<<10,
 150         ER_DEVNOTE      = 38<<10,
 151         ER_MFINDEXWRAP  = 39<<10,
 152 };
 153
 154 typedef struct Ctlr Ctlr;
 155 typedef struct Wait Wait;
 156 typedef struct Ring Ring;
 157 typedef struct Slot Slot;
 158 typedef struct Epio Epio;
 159 typedef struct Port Port;
 160
 161 struct Wait
 162 {
 163         Wait    *next;
 164         Ring    *ring;
 165         u32int  *td;
 166         u32int  er[4];
 167         Rendez  *z;
 168 };
 169
 170 struct Ring
 171 {
 172         int     id;
 173
 174         Slot    *slot;
 175
 176         u32int  *base;
 177
 178         u32int  mask;
 179         u32int  shift;
 180
 181         u32int  rp;
 182         u32int  wp;
 183
 184         u32int  *ctx;
 185         u32int  *doorbell;
 186
 187         int     stopped;
 188
 189         Wait    *pending;
 190         Lock;
 191 };
 192
 193 struct Slot
 194 {
 195         int     id;
 196
 197         Ctlr    *ctlr;
 198         Udev    *dev;
 199
 200         u32int  *ibase;
 201         u32int  *obase;
 202
 203         /* endpoint rings */
 204         int     nep;
 205         Ring    epr[32];
 206 };
 207
 208 struct Port
 209 {
 210         char    spec[4];
 211         int     proto;
 212
 213         u32int  *reg;
 214 };
 215
 216 struct Ctlr
 217 {
 218         Pcidev  *pcidev;
 219
 220         u32int  *mmio;
 221
 222         u32int  *opr;   /* operational registers */
 223         u32int  *rts;   /* runtime registers */
 224         u32int  *dba;   /* doorbell array */
 225
 226         u64int  *dcba;  /* device context base array */
 227
 228         u64int  *sba;   /* scratchpad buffer array */
 229         void    *sbp;   /* scratchpad buffer pages */
 230
 231         u32int  *erst[1];       /* event ring segment table */
 232         Ring    er[1];          /* event ring segment */
 233         Ring    cr[1];          /* command ring segment */
 234         QLock   cmdlock;
 235
 236         u32int  µframe;
 237
 238         QLock   slotlock;
 239         Slot    **slot;         /* slots by slot id */
 240         Port    *port;
 241
 242         u32int  hccparams;
 243
 244         int     csz;
 245         int     pagesize;
 246         int     nscratch;
 247         int     nintrs;
 248         int     nslots;
 249
 250         void    (*setrptr)(u32int*, u64int);
 251
 252         Rendez  recover;
 253         void    *active;
 254         uintptr base;
 255 };
 256
 257 struct Epio
 258 {
 259         QLock;
 260
 261         Ring    *ring;
 262         Block   *b;
 263
 264         /* iso */
 265         u32int  frame;
 266         u32int  period;
 267         u32int  incr;
 268         u32int  tdsz;
 269
 270         int     nleft;
 271 };
 272
 273 static char Ebadlen[] = "bad usb request length";
 274 static char Enotconfig[] = "usb endpoint not configured";
 275 static char Erecover[] = "xhci controller needs reset";
 276
 277 static void
 278 setrptr32(u32int *reg, u64int pa)
 279 {
 280         coherence();
 281         reg[0] = pa;
 282         reg[1] = pa>>32;
 283 }
 284
 285 static void
 286 setrptr64(u32int *reg, u64int pa)
 287 {
 288         coherence();
 289         *((u64int*)reg) = pa;
 290 }
 291
 292 static u32int
 293 µframe(Ctlr *ctlr)
 294 {
 295         u32int µ;
 296         do {
 297                 µ = (ctlr->rts[MFINDEX] & (1<<14)-1) |
 298                         (ctlr->µframe & ~((1<<14)-1));
 299         } while((int)(µ - ctlr->µframe) < 0);
 300         return µ;
 301 }
 302
 303 static void
 304 freering(Ring *r)
 305 {
 306         if(r == nil)
 307                 return;
 308         free(r->base);
 309         memset(r, 0, sizeof(*r));
 310 }
 311
 312 static Ring*
 313 initring(Ring *r, int shift)
 314 {
 315         r->id = 0;
 316         r->ctx = nil;
 317         r->slot = nil;
 318         r->doorbell = nil;
 319         r->pending = nil;
 320         r->stopped = 0;
 321         r->shift = shift;
 322         r->mask = (1<<shift)-1;
 323         r->rp = r->wp = 0;
 324         r->base = mallocalign(4*4<<shift, 64, 0, 64*1024);
 325         if(r->base == nil){
 326                 freering(r);
 327                 error(Enomem);
 328         }
 329         return r;
 330 }
 331
 332 static void
 333 flushring(Ring *r)
 334 {
 335         Rendez *z;
 336         Wait *w;
 337
 338         while((w = r->pending) != nil){
 339                 r->pending = w->next;
 340                 w->next = nil;
 341                 if((z = w->z) != nil){
 342                         w->z = nil;
 343                         wakeup(z);
 344                 }
 345         }
 346 }
 347
 348 static u64int
 349 resetring(Ring *r)
 350 {
 351         u64int pa;
 352
 353         ilock(r);
 354         flushring(r);
 355         r->rp = r->wp;
 356         pa = PADDR(&r->base[4*(r->wp & r->mask)]) | ((~r->wp>>r->shift) & 1);
 357         iunlock(r);
 358
 359         return pa;
 360 }
 361
 362 static u32int*
 363 xecp(Ctlr *ctlr, uchar id, u32int *p)
 364 {
 365         u32int x, *e;
 366
 367         e = &ctlr->mmio[ctlr->pcidev->mem[0].size/4];
 368         if(p == nil){
 369                 p = ctlr->mmio;
 370                 x = ctlr->hccparams>>16;
 371         } else {
 372                 assert(p < e);
 373                 x = (*p>>8) & 255;
 374         }
 375         while(x != 0){
 376                 p += x;
 377                 if(p >= e)
 378                         break;
 379                 x = *p;
 380                 if((x & 255) == id)
 381                         return p;
 382                 x >>= 8;
 383                 x &= 255;
 384         }
 385         return nil;
 386 }
 387
 388 static void
 389 handoff(Ctlr *ctlr)
 390 {
 391         u32int *r;
 392         int i;
 393
 394         if((r = xecp(ctlr, 1, nil)) == nil)
 395                 return;
 396         r[0] |= 1<<24;          /* request ownership */
 397         for(i = 0; (r[0] & (1<<16)) != 0 && i<100; i++)
 398                 tsleep(&up->sleep, return0, nil, 10);
 399         r[1] = 0;               /* disable SMI interrupts */
 400         r[0] &= ~(1<<16);       /* in case of timeout */
 401 }
 402
 403 static void
 404 shutdown(Hci *hp)
 405 {
 406         Ctlr *ctlr = hp->aux;
 407         int i;
 408
 409         ctlr->opr[USBCMD] = 0;
 410         for(i=0; (ctlr->opr[USBSTS] & HCH) == 0 && i < 10; i++)
 411                 delay(10);
 412         intrdisable(ctlr->pcidev->intl, hp->interrupt, hp, ctlr->pcidev->tbdf, hp->type);
 413         pciclrbme(ctlr->pcidev);
 414 }
 415
 416 static void
 417 release(Ctlr *ctlr)
 418 {
 419         int i;
 420
 421         freering(ctlr->cr);
 422         for(i=0; i<nelem(ctlr->er); i++){
 423                 freering(&ctlr->er[i]);
 424                 free(ctlr->erst[i]);
 425                 ctlr->erst[i] = nil;
 426         }
 427         free(ctlr->port), ctlr->port = nil;
 428         free(ctlr->slot), ctlr->slot = nil;
 429         free(ctlr->dcba), ctlr->dcba = nil;
 430         free(ctlr->sba), ctlr->sba = nil;
 431         free(ctlr->sbp), ctlr->sbp = nil;
 432 }
 433
 434 static void recover(void *arg);
 435
 436 static void
 437 init(Hci *hp)
 438 {
 439         Ctlr *ctlr;
 440         Port *pp;
 441         u32int *x;
 442         uchar *p;
 443         int i, j;
 444
 445         ctlr = hp->aux;
 446         if(ctlr->mmio[CAPLENGTH] == -1)
 447                 error("controller vanished");
 448
 449         ctlr->opr = &ctlr->mmio[(ctlr->mmio[CAPLENGTH]&0xFF)/4];
 450         ctlr->dba = &ctlr->mmio[ctlr->mmio[DBOFF]/4];
 451         ctlr->rts = &ctlr->mmio[ctlr->mmio[RTSOFF]/4];
 452
 453         ctlr->hccparams = ctlr->mmio[HCCPARAMS];
 454         handoff(ctlr);
 455
 456         for(i=0; (ctlr->opr[USBSTS] & CNR) != 0 && i<100; i++)
 457                 tsleep(&up->sleep, return0, nil, 10);
 458
 459         ctlr->opr[USBCMD] = HCRST;
 460         for(i=0; (ctlr->opr[USBSTS] & (CNR|HCH)) != HCH && i<100; i++)
 461                 tsleep(&up->sleep, return0, nil, 10);
 462
 463         pcisetbme(ctlr->pcidev);
 464         pcisetpms(ctlr->pcidev, 0);
 465         intrenable(ctlr->pcidev->intl, hp->interrupt, hp, ctlr->pcidev->tbdf, hp->type);
 466
 467         if(waserror()){
 468                 shutdown(hp);
 469                 release(ctlr);
 470                 nexterror();
 471         }
 472
 473         ctlr->csz = (ctlr->hccparams & CSZ) != 0;
 474         if(ctlr->hccparams & AC64)
 475                 ctlr->setrptr = setrptr64;
 476         else
 477                 ctlr->setrptr = setrptr32;
 478         ctlr->pagesize = ctlr->opr[PAGESIZE]<<12;
 479
 480         ctlr->nscratch = (ctlr->mmio[HCSPARAMS2] >> 27) & 0x1F;
 481         ctlr->nintrs = (ctlr->mmio[HCSPARAMS1] >> 8) & 0x7FF;
 482         ctlr->nslots = (ctlr->mmio[HCSPARAMS1] >> 0) & 0xFF;
 483
 484         hp->highspeed = 1;
 485         hp->superspeed = 0;
 486         hp->nports = (ctlr->mmio[HCSPARAMS1] >> 24) & 0xFF;
 487         ctlr->port = malloc(hp->nports * sizeof(Port));
 488         if(ctlr->port == nil)
 489                 error(Enomem);
 490         for(i=0; i<hp->nports; i++)
 491                 ctlr->port[i].reg = &ctlr->opr[0x400/4 + i*4];
 492
 493         x = nil;
 494         while((x = xecp(ctlr, 2, x)) != nil){
 495                 i = x[2]&255;
 496                 j = (x[2]>>8)&255;
 497                 while(j--){
 498                         if(i < 1 || i > hp->nports)
 499                                 break;
 500                         pp = &ctlr->port[i-1];
 501                         pp->proto = x[0]>>16;
 502                         memmove(pp->spec, &x[1], 4);
 503                         if(memcmp(pp->spec, "USB ", 4) == 0 && pp->proto >= 0x0300)
 504                                 hp->superspeed |= 1<<(i-1);
 505                         i++;
 506                 }
 507         }
 508
 509         ctlr->slot = malloc((1+ctlr->nslots)*sizeof(ctlr->slot[0]));
 510         ctlr->dcba = mallocalign((1+ctlr->nslots)*8, 64, 0, ctlr->pagesize);
 511         if(ctlr->slot == nil || ctlr->dcba == nil)
 512                 error(Enomem);
 513         if(ctlr->nscratch != 0){
 514                 ctlr->sba = mallocalign(ctlr->nscratch*8, 64, 0, ctlr->pagesize);
 515                 ctlr->sbp = mallocalign(ctlr->nscratch*ctlr->pagesize, ctlr->pagesize, 0, 0);
 516                 if(ctlr->sba == nil || ctlr->sbp == nil)
 517                         error(Enomem);
 518                 for(i=0, p = ctlr->sbp; i<ctlr->nscratch; i++, p += ctlr->pagesize){
 519                         memset(p, 0, ctlr->pagesize);
 520                         ctlr->sba[i] = PADDR(p);
 521                 }
 522                 ctlr->dcba[0] = PADDR(ctlr->sba);
 523         } else {
 524                 ctlr->dcba[0] = 0;
 525         }
 526         for(i=1; i<=ctlr->nslots; i++)
 527                 ctlr->dcba[i] = 0;
 528         ctlr->opr[CONFIG] = ctlr->nslots;       /* MaxSlotsEn */
 529         ctlr->setrptr(&ctlr->opr[DCBAAP], PADDR(ctlr->dcba));
 530
 531         initring(ctlr->cr, 8);          /* 256 entries */
 532         ctlr->cr->id = 0;
 533         ctlr->cr->doorbell = &ctlr->dba[0];
 534         ctlr->setrptr(&ctlr->opr[CRCR], resetring(ctlr->cr));
 535
 536         for(i=0; i<ctlr->nintrs; i++){
 537                 u32int *irs = &ctlr->rts[IR0 + i*8];
 538
 539                 if(i >= nelem(ctlr->er)){
 540                         irs[ERSTSZ] = 0;        /* disable ring */
 541                         continue;
 542                 }
 543
 544                 /* allocate and link into event ring segment table */
 545                 initring(&ctlr->er[i], 8);      /* 256 entries */
 546                 ctlr->erst[i] = mallocalign(16, 64, 0, 0);
 547                 if(ctlr->erst[i] == nil)
 548                         error(Enomem);
 549                 *((u64int*)ctlr->erst[i]) = PADDR(ctlr->er[i].base);
 550                 ctlr->erst[i][2] = ctlr->er[i].mask+1;
 551                 ctlr->erst[i][3] = 0;
 552
 553                 irs[ERSTSZ] = 1;        /* just one segment */
 554                 ctlr->setrptr(&irs[ERDP], PADDR(ctlr->er[i].base));
 555                 ctlr->setrptr(&irs[ERSTBA], PADDR(ctlr->erst[i]));
 556
 557                 irs[IMAN] = 3;
 558                 irs[IMOD] = 0;
 559         }
 560         poperror();
 561
 562         ctlr->µframe = 0;
 563         ctlr->opr[USBCMD] = RUNSTOP|INTE|HSEE|EWE;
 564         for(i=0; (ctlr->opr[USBSTS] & (CNR|HCH)) != 0 && i<100; i++)
 565                 tsleep(&up->sleep, return0, nil, 10);
 566
 567         kproc("xhcirecover", recover, hp);
 568 }
 569
 570 static int
 571 needrecover(void *arg)
 572 {
 573         Ctlr *ctlr = arg;
 574         return  ctlr->er->stopped ||
 575                 (ctlr->opr[USBSTS] & (HCH|HCE|HSE)) != 0;
 576 }
 577
 578 static void
 579 recover(void *arg)
 580 {
 581         Hci *hp = arg;
 582         Ctlr *ctlr = hp->aux;
 583
 584         while(waserror())
 585                 ;
 586         while(!needrecover(ctlr))
 587                 tsleep(&ctlr->recover, needrecover, ctlr, 1000);
 588
 589         shutdown(hp);
 590
 591         /*
 592          * flush all transactions and wait until all devices have
 593          * been detached by usbd.
 594          */
 595         for(;;){
 596                 int i, j, active;
 597
 598                 ilock(ctlr->cr);
 599                 ctlr->cr->stopped = 1;
 600                 flushring(ctlr->cr);
 601                 iunlock(ctlr->cr);
 602
 603                 active = 0;
 604                 qlock(&ctlr->slotlock);
 605                 for(i=1; i<=ctlr->nslots; i++){
 606                         Slot *slot = ctlr->slot[i];
 607                         if(slot == nil)
 608                                 continue;
 609                         active++;
 610                         for(j=0; j < slot->nep; j++){
 611                                 Ring *ring = &slot->epr[j];
 612                                 if(ring->base == nil)
 613                                         continue;
 614                                 ilock(ring);
 615                                 ring->stopped = 1;
 616                                 flushring(ring);
 617                                 iunlock(ring);
 618                         }
 619                 }
 620                 qunlock(&ctlr->slotlock);
 621                 if(active == 0)
 622                         break;
 623
 624                 tsleep(&up->sleep, return0, nil, 100);
 625         }
 626
 627         qlock(&ctlr->slotlock);
 628         qlock(&ctlr->cmdlock);
 629
 630         release(ctlr);
 631         if(waserror()) {
 632                 print("xhci recovery failed: %s\n", up->errstr);
 633         } else {
 634                 init(hp);
 635                 poperror();
 636         }
 637
 638         qunlock(&ctlr->cmdlock);
 639         qunlock(&ctlr->slotlock);
 640
 641         pexit("", 1);
 642 }
 643
 644 static void
 645 dump(Hci *)
 646 {
 647 }
 648
 649 static void
 650 queuetd(Ring *r, u32int c, u32int s, u64int p, Wait *w)
 651 {
 652         u32int *td, x;
 653
 654         x = r->wp++;
 655         if((x & r->mask) == r->mask){
 656                 td = r->base + 4*(x & r->mask);
 657                 *(u64int*)td = PADDR(r->base);
 658                 td[2] = 0;
 659                 td[3] = ((~x>>r->shift)&1) | (1<<1) | TR_LINK;
 660                 x = r->wp++;
 661         }
 662         td = r->base + 4*(x & r->mask);
 663         if(w != nil){
 664                 w->er[0] = w->er[1] = w->er[2] = w->er[3] = 0;
 665                 w->ring = r;
 666                 w->td = td;
 667                 w->z = &up->sleep;
 668
 669                 ilock(r);
 670                 w->next = r->pending;
 671                 r->pending = w;
 672                 iunlock(r);
 673         }
 674         coherence();
 675         *(u64int*)td = p;
 676         td[2] = s;
 677         td[3] = ((~x>>r->shift)&1) | c;
 678 }
 679
 680 static char *ccerrtab[] = {
 681 [2]     "Data Buffer Error",
 682 [3]     "Babble Detected Error",
 683 [4]     "USB Transaction Error",
 684 [5]     "TRB Error",
 685 [6]     "Stall Error",
 686 [7]     "Resume Error",
 687 [8]     "Bandwidth Error",
 688 [9]     "No Slots Available",
 689 [10]    "Invalid Stream Type",
 690 [11]    "Slot Not Enabled",
 691 [12]    "Endpoint Not Enabled",
 692 [13]    "Short Packet",
 693 [14]    "Ring Underrun",
 694 [15]    "Ring Overrun",
 695 [16]    "VF Event Ring Full",
 696 [17]    "Parameter Error",
 697 [18]    "Bandwidth Overrun Error",
 698 [19]    "Context State Error",
 699 [20]    "No Ping Response",
 700 [21]    "Event Ring Full",
 701 [22]    "Incompatible Device",
 702 [23]    "Missed Service Error",
 703 [24]    "Command Ring Stopped",
 704 [25]    "Command Aborted",
 705 [26]    "Stopped",
 706 [27]    "Stoppe - Length Invalid",
 707 [29]    "Max Exit Latency Too Large",
 708 [31]    "Isoch Buffer Overrun",
 709 [32]    "Event Lost Error",
 710 [33]    "Undefined Error",
 711 [34]    "Invalid Stream ID",
 712 [35]    "Secondary Bandwidth Error",
 713 [36]    "Split Transaction Error",
 714 };
 715
 716 static char*
 717 ccerrstr(u32int cc)
 718 {
 719         char *s;
 720
 721         if(cc == 1 || cc == 13)
 722                 return nil;
 723         if(cc < nelem(ccerrtab) && ccerrtab[cc] != nil)
 724                 s = ccerrtab[cc];
 725         else
 726                 s = "???";
 727         return s;
 728 }
 729
 730 static int
 731 waitdone(void *a)
 732 {
 733         return ((Wait*)a)->z == nil;
 734 }
 735
 736 static char*
 737 ctlrcmd(Ctlr *ctlr, u32int c, u32int s, u64int p, u32int *er);
 738
 739 static char*
 740 waittd(Ctlr *ctlr, Wait *w, int tmout)
 741 {
 742         Ring *r = w->ring;
 743
 744         coherence();
 745         *r->doorbell = r->id;
 746
 747         while(waserror()){
 748                 if(!r->stopped) {
 749                         if(r == ctlr->cr)
 750                                 ctlr->opr[CRCR] |= CA;
 751                         else
 752                                 ctlrcmd(ctlr, CR_STOPEP | (r->id<<16) | (r->slot->id<<24), 0, 0, nil);
 753                         r->stopped = 1;
 754                 }
 755                 tmout = 0;
 756         }
 757         if(tmout > 0){
 758                 tsleep(&up->sleep, waitdone, w, tmout);
 759                 if(!waitdone(w))
 760                         error("timed out");
 761         } else {
 762                 while(!waitdone(w))
 763                         sleep(&up->sleep, waitdone, w);
 764         }
 765         poperror();
 766
 767         return ccerrstr(w->er[2]>>24);
 768 }
 769
 770 static char*
 771 ctlrcmd(Ctlr *ctlr, u32int c, u32int s, u64int p, u32int *er)
 772 {
 773         Wait w[1];
 774         char *err;
 775
 776         qlock(&ctlr->cmdlock);
 777         if(needrecover(ctlr)){
 778                 qunlock(&ctlr->cmdlock);
 779                 return Erecover;
 780         }
 781         ctlr->cr->stopped = 0;
 782         queuetd(ctlr->cr, c, s, p, w);
 783         err = waittd(ctlr, w, 5000);
 784
 785         qunlock(&ctlr->cmdlock);
 786
 787         if(er != nil)
 788                 memmove(er, w->er, 4*4);
 789
 790         return err;
 791 }
 792
 793 static void
 794 completering(Ring *r, u32int *er)
 795 {
 796         Wait *w, **wp;
 797         u32int *td, x;
 798         u64int pa;
 799
 800         pa = (*(u64int*)er) & ~15ULL;
 801         ilock(r);
 802
 803         for(x = r->rp; (int)(r->wp - x) > 0;){
 804                 td = &r->base[4*(x++ & r->mask)];
 805                 if((u64int)PADDR(td) == pa){
 806                         r->rp = x;
 807                         break;
 808                 }
 809         }
 810
 811         wp = &r->pending;
 812         while(w = *wp){
 813                 if((u64int)PADDR(w->td) == pa){
 814                         Rendez *z = w->z;
 815
 816                         memmove(w->er, er, 4*4);
 817                         *wp = w->next;
 818                         w->next = nil;
 819
 820                         if(z != nil){
 821                                 w->z = nil;
 822                                 wakeup(z);
 823                         }
 824                         break;
 825                 } else {
 826                         wp = &w->next;
 827                 }
 828         }
 829
 830         iunlock(r);
 831 }
 832
 833 static void
 834 interrupt(Ureg*, void *arg)
 835 {
 836         Hci *hp = arg;
 837         Ctlr *ctlr = hp->aux;
 838         Ring *ring = ctlr->er;
 839         Slot *slot;
 840         u32int *irs, *td, x;
 841
 842         if(ring->base == nil)
 843                 return;
 844
 845         irs = &ctlr->rts[IR0];
 846         x = irs[IMAN];
 847         if(x & 1) irs[IMAN] = x & 3;
 848
 849         for(x = ring->rp;; x=++ring->rp){
 850                 td = ring->base + 4*(x & ring->mask);
 851                 if((((x>>ring->shift)^td[3])&1) == 0)
 852                         break;
 853
 854                 switch(td[3] & 0xFC00){
 855                 case ER_CMDCOMPL:
 856                         completering(ctlr->cr, td);
 857                         break;
 858                 case ER_TRANSFER:
 859                         x = td[3]>>24;
 860                         if(x == 0 || x > ctlr->nslots)
 861                                 break;
 862                         slot = ctlr->slot[x];
 863                         if(slot == nil)
 864                                 break;
 865                         completering(&slot->epr[(td[3]>>16)-1&31], td);
 866                         break;
 867                 case ER_MFINDEXWRAP:
 868                         ctlr->µframe = (ctlr->rts[MFINDEX] & (1<<14)-1) |
 869                                 (ctlr->µframe+(1<<14) & ~((1<<14)-1));
 870                         break;
 871                 case ER_HCE:
 872                         iprint("xhci: host controller error: %ux %ux %ux %ux\n",
 873                                 td[0], td[1], td[2], td[3]);
 874                         ctlr->er->stopped = 1;
 875                         wakeup(&ctlr->recover);
 876                         return;
 877                 case ER_PORTSC:
 878                         break;
 879                 case ER_BWREQ:
 880                 case ER_DOORBELL:
 881                 case ER_DEVNOTE:
 882                 default:
 883                         iprint("xhci: event %ud: %ux %ux %ux %ux\n",
 884                                 x, td[0], td[1], td[2], td[3]);
 885                 }
 886         }
 887
 888         ctlr->setrptr(&irs[ERDP], PADDR(td) | (1<<3));
 889 }
 890
 891 static void
 892 freeslot(void *arg)
 893 {
 894         Slot *slot;
 895
 896         if(arg == nil)
 897                 return;
 898         slot = arg;
 899         if(slot->id != 0){
 900                 Ctlr *ctlr = slot->ctlr;
 901                 qlock(&ctlr->slotlock);
 902                 if(ctlr->slot != nil && ctlr->slot[slot->id] == slot){
 903                         ctlrcmd(ctlr, CR_DISABLESLOT | (slot->id<<24), 0, 0, nil);
 904                         ctlr->dcba[slot->id] = 0;
 905                         ctlr->slot[slot->id] = nil;
 906                 }
 907                 qunlock(&ctlr->slotlock);
 908         }
 909         freering(&slot->epr[0]);
 910         free(slot->ibase);
 911         free(slot->obase);
 912         free(slot);
 913 }
 914
 915 static Slot*
 916 allocslot(Ctlr *ctlr, Udev *dev)
 917 {
 918         u32int r[4];
 919         Slot *slot;
 920         char *err;
 921
 922         slot = malloc(sizeof(Slot));
 923         if(slot == nil)
 924                 error(Enomem);
 925
 926         slot->ctlr = ctlr;
 927         slot->dev = dev;
 928         slot->nep = 0;
 929         slot->id = 0;
 930
 931         qlock(&ctlr->slotlock);
 932         if(waserror()){
 933                 qunlock(&ctlr->slotlock);
 934                 freeslot(slot);
 935                 nexterror();
 936         }
 937         if(ctlr->slot == nil)
 938                 error(Erecover);
 939         slot->ibase = mallocalign(32*33 << ctlr->csz, 64, 0, ctlr->pagesize);
 940         slot->obase = mallocalign(32*32 << ctlr->csz, 64, 0, ctlr->pagesize);
 941         if(slot->ibase == nil || slot->obase == nil)
 942                 error(Enomem);
 943         if((err = ctlrcmd(ctlr, CR_ENABLESLOT, 0, 0, r)) != nil)
 944                 error(err);
 945         slot->id = r[3]>>24;
 946         if(slot->id <= 0 || slot->id > ctlr->nslots || ctlr->slot[slot->id] != nil){
 947                 slot->id = 0;
 948                 error("bad slot id from controller");
 949         }
 950         poperror();
 951
 952         ctlr->dcba[slot->id] = PADDR(slot->obase);
 953         ctlr->slot[slot->id] = slot;
 954         qunlock(&ctlr->slotlock);
 955
 956         dev->aux = slot;
 957         dev->free = freeslot;
 958
 959         return slot;
 960 }
 961
 962 static void
 963 setdebug(Hci *, int)
 964 {
 965 }
 966
 967 static void
 968 epclose(Ep *ep)
 969 {
 970         Ctlr *ctlr;
 971         Slot *slot;
 972         Ring *ring;
 973         Epio *io;
 974
 975         if(ep->dev->isroot)
 976                 return;
 977
 978         io = ep->aux;
 979         if(io == nil)
 980                 return;
 981         ep->aux = nil;
 982
 983         ctlr = ep->hp->aux;
 984         slot = ep->dev->aux;
 985
 986         if(ep->nb > 0 && (io[OREAD].ring != nil || io[OWRITE].ring != nil)){
 987                 u32int *w;
 988
 989                 /* input control context */
 990                 w = slot->ibase;
 991                 memset(w, 0, 32<<ctlr->csz);
 992                 w[1] = 1;
 993                 if((ring = io[OREAD].ring) != nil){
 994                         w[0] |= 1 << ring->id;
 995                         if(ring->id == slot->nep)
 996                                 slot->nep--;
 997                         ctlrcmd(ctlr, CR_STOPEP | (ring->id<<16) | (slot->id<<24), 0, 0, nil);
 998                 }
 999                 if((ring = io[OWRITE].ring) != nil){
1000                         w[0] |= 1 << ring->id;
1001                         if(ring->id == slot->nep)
1002                                 slot->nep--;
1003                         ctlrcmd(ctlr, CR_STOPEP | (ring->id<<16) | (slot->id<<24), 0, 0, nil);
1004                 }
1005
1006                 /* (input) slot context */
1007                 w += 8<<ctlr->csz;
1008                 w[0] = (w[0] & ~(0x1F<<27)) | slot->nep<<27;
1009
1010                 /* (input) ep context */
1011                 w += ep->nb*2*8<<ctlr->csz;
1012                 memset(w, 0, 2*32<<ctlr->csz);
1013
1014                 ctlrcmd(ctlr, CR_CONFIGEP | (slot->id<<24), 0, PADDR(slot->ibase), nil);
1015
1016                 freering(io[OREAD].ring);
1017                 freering(io[OWRITE].ring);
1018         }
1019         freeb(io[OREAD].b);
1020         freeb(io[OWRITE].b);
1021         free(io);
1022 }
1023
1024 static void
1025 initepctx(u32int *w, Ring *r, Ep *ep)
1026 {
1027         int ival;
1028
1029         if(ep->dev->speed == Lowspeed || ep->dev->speed == Fullspeed){
1030                 for(ival=3; ival < 11 && (1<<ival) < ep->pollival; ival++)
1031                         ;
1032         } else {
1033                 for(ival=0; ival < 15 && (1<<ival) < ep->pollival; ival++)
1034                         ;
1035         }
1036         w[0] = ival<<16;
1037         w[1] = ((ep->ttype-Tctl)|(r->id&1)<<2)<<3 | (ep->ntds-1)<<8 | ep->maxpkt<<16;
1038         if(ep->ttype != Tiso)
1039                 w[1] |= 3<<1;
1040         *((u64int*)&w[2]) = PADDR(r->base) | 1;
1041
1042         w[4] = ep->maxpkt;
1043         if(ep->ttype == Tintr || ep->ttype == Tiso)
1044                 w[4] |= (ep->maxpkt*ep->ntds)<<16;
1045 }
1046
1047 static void
1048 initisoio(Epio *io, Ep *ep)
1049 {
1050         if(io->ring == nil)
1051                 return;
1052         io->frame = 0;
1053         io->period = ep->pollival<<3*(ep->dev->speed == Fullspeed);
1054         io->incr = (ep->hz*io->period<<8)/8000;
1055         io->tdsz = (io->incr+255>>8)*ep->samplesz;
1056         io->b = allocb((io->ring->mask+1)*io->tdsz);
1057 }
1058
1059 static void
1060 initep(Ep *ep)
1061 {
1062         Epio *io;
1063         Ctlr *ctlr;
1064         Slot *slot;
1065         Ring *ring;
1066         u32int *w;
1067         char *err;
1068
1069         io = ep->aux;
1070         ctlr = ep->hp->aux;
1071         slot = ep->dev->aux;
1072
1073         io[OREAD].ring = io[OWRITE].ring = nil;
1074         if(ep->nb == 0){
1075                 io[OWRITE].ring = &slot->epr[0];
1076                 return;
1077         }
1078
1079         /* (input) control context */
1080         w = slot->ibase;
1081         memset(w, 0, 32<<ctlr->csz);
1082         w[1] = 1;
1083
1084         if(waserror()){
1085                 freering(io[OWRITE].ring), io[OWRITE].ring = nil;
1086                 freering(io[OREAD].ring), io[OREAD].ring = nil;
1087                 nexterror();
1088         }
1089         if(ep->mode != OREAD){
1090                 ring = initring(io[OWRITE].ring = &slot->epr[ep->nb*2-1], 8);
1091                 ring->id = ep->nb*2;
1092                 if(ring->id > slot->nep)
1093                         slot->nep = ring->id;
1094                 ring->slot = slot;
1095                 ring->doorbell = &ctlr->dba[slot->id];
1096                 ring->ctx = &slot->obase[ring->id*8<<ctlr->csz];
1097                 w[1] |= 1 << ring->id;
1098         }
1099         if(ep->mode != OWRITE){
1100                 ring = initring(io[OREAD].ring = &slot->epr[ep->nb*2], 8);
1101                 ring->id = ep->nb*2+1;
1102                 if(ring->id > slot->nep)
1103                         slot->nep = ring->id;
1104                 ring->slot = slot;
1105                 ring->doorbell = &ctlr->dba[slot->id];
1106                 ring->ctx = &slot->obase[ring->id*8<<ctlr->csz];
1107                 w[1] |= 1 << ring->id;
1108         }
1109
1110         /* (input) slot context */
1111         w += 8<<ctlr->csz;
1112         w[0] = (w[0] & ~(0x1F<<27)) | slot->nep<<27;
1113
1114         /* (input) ep context */
1115         w += ep->nb*2*8<<ctlr->csz;
1116         memset(w, 0, 2*32<<ctlr->csz);
1117         if(io[OWRITE].ring != nil)
1118                 initepctx(w, io[OWRITE].ring, ep);
1119
1120         w += 8<<ctlr->csz;
1121         if(io[OREAD].ring != nil)
1122                 initepctx(w, io[OREAD].ring, ep);
1123
1124         if((err = ctlrcmd(ctlr, CR_CONFIGEP | (slot->id<<24), 0,
1125                 PADDR(slot->ibase), nil)) != nil){
1126                 error(err);
1127         }
1128         if(ep->ttype == Tiso){
1129                 initisoio(io+OWRITE, ep);
1130                 initisoio(io+OREAD, ep);
1131         }
1132         poperror();
1133 }
1134
1135 static int
1136 speedid(int speed)
1137 {
1138         switch(speed){
1139         case Fullspeed:         return 1;
1140         case Lowspeed:          return 2;
1141         case Highspeed:         return 3;
1142         case Superspeed:        return 4;
1143         }
1144         return 0;
1145 }
1146
1147 static void
1148 epopen(Ep *ep)
1149 {
1150         Ctlr *ctlr = ep->hp->aux;
1151         Slot *slot, *hub;
1152         Ring *ring;
1153         Epio *io;
1154         Udev *dev;
1155         char *err;
1156         u32int *w;
1157         int i;
1158
1159         if(ep->dev->isroot)
1160                 return;
1161         if(needrecover(ctlr))
1162                 error(Erecover);
1163         io = malloc(sizeof(Epio)*2);
1164         if(io == nil)
1165                 error(Enomem);
1166         ep->aux = io;
1167         if(waserror()){
1168                 epclose(ep);
1169                 nexterror();
1170         }
1171         dev = ep->dev;
1172         slot = dev->aux;
1173         if(slot != nil && slot->dev == dev){
1174                 initep(ep);
1175                 poperror();
1176                 return;
1177         }
1178
1179         /* first open has to be control endpoint */
1180         if(ep->nb != 0)
1181                 error(Egreg);
1182
1183         slot = allocslot(ctlr, dev);
1184
1185         /* allocate control ep 0 ring */
1186         ring = initring(io[OWRITE].ring = &slot->epr[0], 4);
1187         ring->id = 1;
1188         slot->nep = 1;
1189         ring->slot = slot;
1190         ring->doorbell = &ctlr->dba[slot->id];
1191         ring->ctx = &slot->obase[8];
1192
1193         /* (input) control context */
1194         w = slot->ibase;
1195         memset(w, 0, 3*32<<ctlr->csz);
1196         w[1] = 3;       /* A0, A1 */
1197
1198         /* (input) slot context */
1199         w += 8<<ctlr->csz;
1200         w[2] = w[3] = 0;
1201         w[0] = dev->routestr | speedid(dev->speed)<<20 |
1202                 (dev->speed == Highspeed && dev->ishub != 0
1203                 ||dev->speed < Highspeed && dev->ishub == 0)<<25 |
1204                 (dev->ishub != 0)<<26 | slot->nep<<27;
1205         w[1] = dev->rootport<<16;
1206
1207         /* find the parent hub that this device is conected to */
1208         qlock(&ctlr->slotlock);
1209         for(i=1; i<=ctlr->nslots; i++){
1210                 hub = ctlr->slot[i];
1211                 if(hub == nil || hub->dev == nil || hub->dev->aux != hub)
1212                         continue;
1213                 if(hub == slot || hub->dev == dev)
1214                         continue;
1215                 if(!hub->dev->ishub)
1216                         continue;
1217                 if(hub->dev->addr != dev->hub)
1218                         continue;
1219                 if(hub->dev->rootport != dev->rootport)
1220                         continue;
1221
1222                 if(dev->speed < Highspeed && hub->dev->speed == Highspeed)
1223                         w[2] = hub->id | dev->port<<8;
1224                 break;
1225         }
1226         qunlock(&ctlr->slotlock);
1227
1228         /* (input) ep context 0 */
1229         w += 8<<ctlr->csz;
1230         initepctx(w, io[OWRITE].ring, ep);
1231
1232         if((err = ctlrcmd(ctlr, CR_ADDRESSDEV | (slot->id<<24), 0,
1233                 PADDR(slot->ibase), nil)) != nil){
1234                 error(err);
1235         }
1236
1237         /* (output) slot context */
1238         w = slot->obase;
1239         ep->dev->addr = w[3] & 0xFF;
1240         poperror();
1241 }
1242
1243 static long
1244 isoread(Ep *, uchar *, long)
1245 {
1246         error(Egreg);
1247         return 0;
1248 }
1249
1250 static long
1251 isowrite(Ep *ep, uchar *p, long n)
1252 {
1253         uchar *s, *d;
1254         Ctlr *ctlr;
1255         Epio *io;
1256         u32int i, µ;
1257         long m;
1258
1259         s = p;
1260         io = (Epio*)ep->aux + OWRITE;
1261         qlock(io);
1262         if(waserror()){
1263                 qunlock(io);
1264                 nexterror();
1265         }
1266         µ = io->period;
1267         ctlr = ep->hp->aux;
1268         if(needrecover(ctlr))
1269                 error(Erecover);
1270         for(i = io->frame;; i++){
1271                 for(;;){
1272                         m = (int)(io->ring->wp - io->ring->rp);
1273                         if(m <= 0)
1274                                 i = (80 + µframe(ctlr))/µ;
1275                         if(m < io->ring->mask)
1276                                 break;
1277                         *io->ring->doorbell = io->ring->id;
1278                         tsleep(&up->sleep, return0, nil, 5);
1279                 }
1280                 m = ((io->incr + (i*io->incr&255))>>8)*ep->samplesz;
1281                 d = io->b->rp + (i&io->ring->mask)*io->tdsz;
1282                 m -= io->nleft, d += io->nleft;
1283                 if(n < m){
1284                         memmove(d, p, n);
1285                         p += n;
1286                         io->nleft += n;
1287                         break;
1288                 }
1289                 memmove(d, p, m);
1290                 p += m, n -= m;
1291                 m += io->nleft, d -= io->nleft;
1292                 io->nleft = 0;
1293                 queuetd(io->ring, TR_ISOCH | (i*µ/8 & 0x7ff)<<20 | TR_IOC, m, PADDR(d), nil);
1294         }
1295         io->frame = i;
1296         while(io->ring->rp != io->ring->wp){
1297                 int d = (int)(i*µ - µframe(ctlr))/8;
1298                 d -= ep->sampledelay*1000 / ep->hz;
1299                 if(d < 5)
1300                         break;
1301                 *io->ring->doorbell = io->ring->id;
1302                 tsleep(&up->sleep, return0, nil, d);
1303         }
1304         qunlock(io);
1305         poperror();
1306
1307         return p - s;
1308 }
1309
1310 static char*
1311 unstall(Ring *r)
1312 {
1313         char *err;
1314
1315         switch(r->ctx[0]&7){
1316         case 2: /* halted */
1317         case 4: /* error */
1318                 r->stopped = 1;
1319                 err = ctlrcmd(r->slot->ctlr, CR_RESETEP | (r->id<<16) | (r->slot->id<<24), 0, 0, nil);
1320                 if(err != nil)
1321                         return err;
1322         }
1323         if(r->stopped){
1324                 err = ctlrcmd(r->slot->ctlr, CR_SETTRDQP | (r->id<<16) | (r->slot->id<<24), 0, resetring(r), nil);
1325                 if(err != nil)
1326                         return err;
1327                 r->stopped = 0;
1328         }
1329         if(r->wp - r->rp >= r->mask)
1330                 return "Ring Full";
1331         return nil;
1332 }
1333
1334 static long
1335 epread(Ep *ep, void *va, long n)
1336 {
1337         Epio *io;
1338         uchar *p;
1339         char *err;
1340         Wait w[1];
1341
1342         if(ep->dev->isroot)
1343                 error(Egreg);
1344
1345         p = va;
1346         if(ep->ttype == Tctl){
1347                 io = (Epio*)ep->aux + OREAD;
1348                 qlock(io);
1349                 if(io->b == nil || BLEN(io->b) == 0){
1350                         qunlock(io);
1351                         return 0;
1352                 }
1353                 if(n > BLEN(io->b))
1354                         n = BLEN(io->b);
1355                 memmove(p, io->b->rp, n);
1356                 io->b->rp += n;
1357                 qunlock(io);
1358                 return n;
1359         } else if(ep->ttype == Tiso)
1360                 return isoread(ep, p, n);
1361
1362         if((uintptr)p <= KZERO){
1363                 Block *b;
1364
1365                 b = allocb(n);
1366                 if(waserror()){
1367                         freeb(b);
1368                         nexterror();
1369                 }
1370                 n = epread(ep, b->rp, n);
1371                 memmove(p, b->rp, n);
1372                 freeb(b);
1373                 poperror();
1374                 return n;
1375         }
1376
1377         io = (Epio*)ep->aux + OREAD;
1378         qlock(io);
1379         if(waserror()){
1380                 qunlock(io);
1381                 nexterror();
1382         }
1383
1384         if((err = unstall(io->ring)) != nil)
1385                 error(err);
1386
1387         queuetd(io->ring, TR_NORMAL | TR_IOC, n, PADDR(p), w);
1388         if((err = waittd((Ctlr*)ep->hp->aux, w, ep->tmout)) != nil)
1389                 error(err);
1390
1391         qunlock(io);
1392         poperror();
1393
1394         n -= (w->er[2] & 0xFFFFFF);
1395         if(n < 0)
1396                 n = 0;
1397
1398         return n;
1399 }
1400
1401 static long
1402 epwrite(Ep *ep, void *va, long n)
1403 {
1404         Wait w[3];
1405         Epio *io;
1406         uchar *p;
1407         char *err;
1408
1409         if(ep->dev->isroot)
1410                 error(Egreg);
1411
1412         p = va;
1413         if(ep->ttype == Tctl){
1414                 int dir, len;
1415                 Ring *ring;
1416
1417                 if(n < 8)
1418                         error(Eshort);
1419
1420                 if(p[0] == 0x00 && p[1] == 0x05)
1421                         return n;
1422
1423                 io = (Epio*)ep->aux + OREAD;
1424                 ring = io[OWRITE-OREAD].ring;
1425                 qlock(io);
1426                 if(waserror()){
1427                         ilock(ring);
1428                         ring->pending = nil;
1429                         iunlock(ring);
1430                         qunlock(io);
1431                         nexterror();
1432                 }
1433                 if(io->b != nil){
1434                         freeb(io->b);
1435                         io->b = nil;
1436                 }
1437                 len = GET2(&p[6]);
1438                 dir = (p[0] & Rd2h) != 0;
1439                 if(len > 0){
1440                         io->b = allocb(len);
1441                         if(dir == 0){   /* out */
1442                                 assert(len >= n-8);
1443                                 memmove(io->b->wp, p+8, n-8);
1444                         } else {
1445                                 memset(io->b->wp, 0, len);
1446                                 io->b->wp += len;
1447                         }
1448                 }
1449                 if((err = unstall(ring)) != nil)
1450                         error(err);
1451
1452                 if((ring->ctx[1]>>16) != ep->maxpkt){
1453                         Slot *slot = ring->slot;
1454                         Ctlr *ctlr = slot->ctlr;
1455                         u32int *w = slot->ibase;
1456                         w[0] = 0;
1457                         w[1] = 1<<ring->id;
1458                         w += (ring->id+1)*8<<ctlr->csz;
1459                         initepctx(w, ring, ep);
1460                         if((err = ctlrcmd(ctlr, CR_EVALCTX | (slot->id<<24), 0, PADDR(slot->ibase), nil)) != nil)
1461                                 error(err);
1462                 }
1463
1464                 queuetd(ring, TR_SETUPSTAGE | (len > 0 ? 2+dir : 0)<<16 | TR_IDT | TR_IOC, 8,
1465                         p[0] | p[1]<<8 | GET2(&p[2])<<16 |
1466                         (u64int)(GET2(&p[4]) | len<<16)<<32, &w[0]);
1467                 if(len > 0)
1468                         queuetd(ring, TR_DATASTAGE | dir<<16 | TR_IOC, len,
1469                                 PADDR(io->b->rp), &w[1]);
1470                 queuetd(ring, TR_STATUSSTAGE | (len == 0 || !dir)<<16 | TR_IOC, 0, 0, &w[2]);
1471
1472                 if((err = waittd((Ctlr*)ep->hp->aux, &w[0], ep->tmout)) != nil)
1473                         error(err);
1474                 if(len > 0){
1475                         if((err = waittd((Ctlr*)ep->hp->aux, &w[1], ep->tmout)) != nil)
1476                                 error(err);
1477                         if(dir != 0){
1478                                 io->b->wp -= (w[1].er[2] & 0xFFFFFF);
1479                                 if(io->b->wp < io->b->rp)
1480                                         io->b->wp = io->b->rp;
1481                         }
1482                 }
1483                 if((err = waittd((Ctlr*)ep->hp->aux, &w[2], ep->tmout)) != nil)
1484                         error(err);
1485
1486                 qunlock(io);
1487                 poperror();
1488
1489                 return n;
1490         } else if(ep->ttype == Tiso)
1491                 return isowrite(ep, p, n);
1492
1493         if((uintptr)p <= KZERO){
1494                 Block *b;
1495
1496                 b = allocb(n);
1497                 if(waserror()){
1498                         freeb(b);
1499                         nexterror();
1500                 }
1501                 memmove(b->wp, p, n);
1502                 n = epwrite(ep, b->wp, n);
1503                 freeb(b);
1504                 poperror();
1505                 return n;
1506         }
1507
1508         io = (Epio*)ep->aux + OWRITE;
1509         qlock(io);
1510         if(waserror()){
1511                 qunlock(io);
1512                 nexterror();
1513         }
1514
1515         if((err = unstall(io->ring)) != nil)
1516                 error(err);
1517
1518         queuetd(io->ring, TR_NORMAL | TR_IOC, n, PADDR(p), w);
1519         if((err = waittd((Ctlr*)ep->hp->aux, w, ep->tmout)) != nil)
1520                 error(err);
1521
1522         qunlock(io);
1523         poperror();
1524
1525         return n;
1526 }
1527
1528 static char*
1529 seprintep(char *s, char*, Ep*)
1530 {
1531         return s;
1532 }
1533
1534 static int
1535 portstatus(Hci *hp, int port)
1536 {
1537         Ctlr *ctlr = hp->aux;
1538         u32int psc, ps;
1539
1540         if(ctlr->port == nil || needrecover(ctlr))
1541                 return 0;
1542
1543         ps = 0;
1544         psc = ctlr->port[port-1].reg[PORTSC];
1545         if(psc & CCS)   ps |= HPpresent;
1546         if(psc & PED)   ps |= HPenable;
1547         if(psc & OCA)   ps |= HPovercurrent;
1548         if(psc & PR)    ps |= HPreset;
1549
1550         if((hp->superspeed & (1<<(port-1))) != 0){
1551                 ps |= psc & (PLS|PP);
1552                 if(psc & CSC)   ps |= 1<<0+16;
1553                 if(psc & OCC)   ps |= 1<<3+16;
1554                 if(psc & PRC)   ps |= 1<<4+16;
1555                 if(psc & WRC)   ps |= 1<<5+16;
1556                 if(psc & PLC)   ps |= 1<<6+16;
1557                 if(psc & CEC)   ps |= 1<<7+16;
1558         } else {
1559                 if((ps & HPreset) == 0){
1560                         switch((psc>>10)&15){
1561                         case 1:
1562                                 /* full speed */
1563                                 break;
1564                         case 2:
1565                                 ps |= HPslow;
1566                                 break;
1567                         case 3:
1568                                 ps |= HPhigh;
1569                                 break;
1570                         }
1571                 }
1572                 if(psc & PP)    ps |= HPpower;
1573                 if(psc & CSC)   ps |= HPstatuschg;
1574                 if(psc & PRC)   ps |= HPchange;
1575         }
1576
1577         return ps;
1578 }
1579
1580 static int
1581 portenable(Hci*, int, int)
1582 {
1583         return 0;
1584 }
1585
1586 static int
1587 portreset(Hci *hp, int port, int on)
1588 {
1589         Ctlr *ctlr = hp->aux;
1590
1591         if(ctlr->port == nil || needrecover(ctlr))
1592                 return 0;
1593
1594         if(on){
1595                 ctlr->port[port-1].reg[PORTSC] |= PR;
1596                 tsleep(&up->sleep, return0, nil, 200);
1597         }
1598         return 0;
1599 }
1600
1601
1602 static Ctlr *ctlrs[Nhcis];
1603
1604 static void
1605 scanpci(void)
1606 {
1607         static int already = 0;
1608         int i;
1609         uintptr io;
1610         Ctlr *ctlr;
1611         Pcidev *p;
1612         u32int *mmio;
1613
1614         if(already)
1615                 return;
1616         already = 1;
1617         p = nil;
1618         while ((p = pcimatch(p, 0, 0)) != nil) {
1619                 /*
1620                  * Find XHCI controllers (Programming Interface = 0x30).
1621                  */
1622                 if(p->ccrb != Pcibcserial || p->ccru != Pciscusb || p->ccrp != 0x30)
1623                         continue;
1624                 io = p->mem[0].bar & ~0x0f;
1625                 if(io == 0)
1626                         continue;
1627                 print("usbxhci: %#x %#x: port %#p size %#x irq %d\n",
1628                         p->vid, p->did, io, p->mem[0].size, p->intl);
1629                 mmio = vmap(io, p->mem[0].size);
1630                 if(mmio == nil){
1631                         print("usbxhci: cannot map registers\n");
1632                         continue;
1633                 }
1634                 ctlr = malloc(sizeof(Ctlr));
1635                 if(ctlr == nil){
1636                         print("usbxhci: no memory\n");
1637                         vunmap(mmio, p->mem[0].size);
1638                         continue;
1639                 }
1640                 ctlr->base = io;
1641                 ctlr->active = nil;
1642                 ctlr->pcidev = p;
1643                 ctlr->mmio = mmio;
1644                 for(i = 0; i < nelem(ctlrs); i++)
1645                         if(ctlrs[i] == nil){
1646                                 ctlrs[i] = ctlr;
1647                                 break;
1648                         }
1649                 if(i >= nelem(ctlrs))
1650                         print("xhci: bug: more than %d controllers\n", nelem(ctlrs));
1651         }
1652 }
1653
1654 static int
1655 reset(Hci *hp)
1656 {
1657         Ctlr *ctlr;
1658         int i;
1659
1660         if(getconf("*nousbxhci"))
1661                 return -1;
1662
1663         scanpci();
1664
1665         /*
1666          * Any adapter matches if no hp->port is supplied,
1667          * otherwise the ports must match.
1668          */
1669         for(i = 0; i < nelem(ctlrs) && ctlrs[i] != nil; i++){
1670                 ctlr = ctlrs[i];
1671                 if(ctlr->active == nil)
1672                 if(hp->port == 0 || hp->port == ctlr->base){
1673                         ctlr->active = hp;
1674                         goto Found;
1675                 }
1676         }
1677         return -1;
1678
1679 Found:
1680         hp->aux = ctlr;
1681         hp->port = ctlr->base;
1682         hp->irq = ctlr->pcidev->intl;
1683         hp->tbdf = ctlr->pcidev->tbdf;
1684
1685         hp->init = init;
1686         hp->dump = dump;
1687         hp->interrupt = interrupt;
1688         hp->epopen = epopen;
1689         hp->epclose = epclose;
1690         hp->epread = epread;
1691         hp->epwrite = epwrite;
1692         hp->seprintep = seprintep;
1693         hp->portenable = portenable;
1694         hp->portreset = portreset;
1695         hp->portstatus = portstatus;
1696         hp->shutdown = shutdown;
1697         hp->debug = setdebug;
1698         hp->type = "xhci";
1699
1700         return 0;
1701 }
1702
1703 void
1704 usbxhcilink(void)
1705 {
1706         addhcitype("xhci", reset);
1707 }