sys/src/9/pc/ether82563.c

   1 /*
   2  * Intel 8256[367], 8257[1-9], 8258[03], i350
   3  *      Gigabit Ethernet PCI-Express Controllers
   4  * Coraid EtherDrive® hba
   5  */
   6 #include "u.h"
   7 #include "../port/lib.h"
   8 #include "mem.h"
   9 #include "dat.h"
  10 #include "fns.h"
  11 #include "io.h"
  12 #include "../port/pci.h"
  13 #include "../port/error.h"
  14 #include "../port/netif.h"
  15 #include "../port/etherif.h"
  16
  17 /*
  18  * note: the 82575, 82576 and 82580 are operated using registers aliased
  19  * to the 82563-style architecture.  many features seen in the 82598
  20  * are also seen in the 82575 part.
  21  */
  22
  23 enum {
  24         /* General */
  25
  26         Ctrl            = 0x0000,       /* Device Control */
  27         Status          = 0x0008,       /* Device Status */
  28         Eec             = 0x0010,       /* EEPROM/Flash Control/Data */
  29         Eerd            = 0x0014,       /* EEPROM Read */
  30         Ctrlext         = 0x0018,       /* Extended Device Control */
  31         Fla             = 0x001c,       /* Flash Access */
  32         Mdic            = 0x0020,       /* MDI Control */
  33         Fcal            = 0x0028,       /* Flow Control Address Low */
  34         Fcah            = 0x002C,       /* Flow Control Address High */
  35         Fct             = 0x0030,       /* Flow Control Type */
  36         Kumctrlsta      = 0x0034,       /* Kumeran Control and Status Register */
  37         Connsw          = 0x0034,       /* copper / fiber switch control; 82575/82576 */
  38         Vet             = 0x0038,       /* VLAN EtherType */
  39         Fcttv           = 0x0170,       /* Flow Control Transmit Timer Value */
  40         Txcw            = 0x0178,       /* Transmit Configuration Word */
  41         Rxcw            = 0x0180,       /* Receive Configuration Word */
  42         Ledctl          = 0x0E00,       /* LED control */
  43         Pba             = 0x1000,       /* Packet Buffer Allocation */
  44         Pbs             = 0x1008,       /* Packet Buffer Size */
  45
  46         /* Interrupt */
  47
  48         Icr             = 0x00C0,       /* Interrupt Cause Read */
  49         Itr             = 0x00c4,       /* Interrupt Throttling Rate */
  50         Ics             = 0x00C8,       /* Interrupt Cause Set */
  51         Ims             = 0x00D0,       /* Interrupt Mask Set/Read */
  52         Imc             = 0x00D8,       /* Interrupt mask Clear */
  53         Iam             = 0x00E0,       /* Interrupt acknowledge Auto Mask */
  54         Eitr            = 0x1680,       /* Extended itr; 82575/6 80 only */
  55
  56         /* Receive */
  57
  58         Rctl            = 0x0100,       /* Control */
  59         Ert             = 0x2008,       /* Early Receive Threshold (573[EVL], 82578 only) */
  60         Fcrtl           = 0x2160,       /* Flow Control RX Threshold Low */
  61         Fcrth           = 0x2168,       /* Flow Control Rx Threshold High */
  62         Psrctl          = 0x2170,       /* Packet Split Receive Control */
  63         Drxmxod         = 0x2540,       /* dma max outstanding bytes (82575) */
  64         Rdbal           = 0x2800,       /* Rdesc Base Address Low Queue 0 */
  65         Rdbah           = 0x2804,       /* Rdesc Base Address High Queue 0 */
  66         Rdlen           = 0x2808,       /* Descriptor Length Queue 0 */
  67         Srrctl          = 0x280c,       /* split and replication rx control (82575) */
  68         Rdh             = 0x2810,       /* Descriptor Head Queue 0 */
  69         Rdt             = 0x2818,       /* Descriptor Tail Queue 0 */
  70         Rdtr            = 0x2820,       /* Descriptor Timer Ring */
  71         Rxdctl          = 0x2828,       /* Descriptor Control */
  72         Radv            = 0x282C,       /* Interrupt Absolute Delay Timer */
  73         Rsrpd           = 0x2c00,       /* Small Packet Detect */
  74         Raid            = 0x2c08,       /* ACK interrupt delay */
  75         Cpuvec          = 0x2c10,       /* CPU Vector */
  76         Rxcsum          = 0x5000,       /* Checksum Control */
  77         Rmpl            = 0x5004,       /* rx maximum packet length (82575) */
  78         Rfctl           = 0x5008,       /* Filter Control */
  79         Mta             = 0x5200,       /* Multicast Table Array */
  80         Ral             = 0x5400,       /* Receive Address Low */
  81         Rah             = 0x5404,       /* Receive Address High */
  82         Vfta            = 0x5600,       /* VLAN Filter Table Array */
  83         Mrqc            = 0x5818,       /* Multiple Receive Queues Command */
  84
  85         /* Transmit */
  86
  87         Tctl            = 0x0400,       /* Transmit Control */
  88         Tipg            = 0x0410,       /* Transmit IPG */
  89         Tkabgtxd        = 0x3004,       /* glci afe band gap transmit ref data, or something */
  90         Tdbal           = 0x3800,       /* Tdesc Base Address Low */
  91         Tdbah           = 0x3804,       /* Tdesc Base Address High */
  92         Tdlen           = 0x3808,       /* Descriptor Length */
  93         Tdh             = 0x3810,       /* Descriptor Head */
  94         Tdt             = 0x3818,       /* Descriptor Tail */
  95         Tidv            = 0x3820,       /* Interrupt Delay Value */
  96         Txdctl          = 0x3828,       /* Descriptor Control */
  97         Tadv            = 0x382C,       /* Interrupt Absolute Delay Timer */
  98         Tarc0           = 0x3840,       /* Arbitration Counter Queue 0 */
  99
 100         /* Statistics */
 101
 102         Statistics      = 0x4000,       /* Start of Statistics Area */
 103         Gorcl           = 0x88/4,       /* Good Octets Received Count */
 104         Gotcl           = 0x90/4,       /* Good Octets Transmitted Count */
 105         Torl            = 0xC0/4,       /* Total Octets Received */
 106         Totl            = 0xC8/4,       /* Total Octets Transmitted */
 107         Nstatistics     = 0x124/4,
 108
 109         /* iNVM (i211) */
 110         Invmdata0       = 0x12120,
 111 };
 112
 113 enum {                                  /* Ctrl */
 114         Lrst            = 1<<3,         /* link reset */
 115         Slu             = 1<<6,         /* Set Link Up */
 116         Devrst          = 1<<26,        /* Device Reset */
 117         Rfce            = 1<<27,        /* Receive Flow Control Enable */
 118         Tfce            = 1<<28,        /* Transmit Flow Control Enable */
 119         Phyrst          = 1<<31,        /* Phy Reset */
 120 };
 121
 122 enum {                                  /* Status */
 123         Lu              = 1<<1,         /* Link Up */
 124         Lanid           = 3<<2,         /* mask for Lan ID. */
 125         Txoff           = 1<<4,         /* Transmission Paused */
 126         Tbimode         = 1<<5,         /* TBI Mode Indication */
 127         Phyra           = 1<<10,        /* PHY Reset Asserted */
 128         GIOme           = 1<<19,        /* GIO Master Enable Status */
 129 };
 130
 131 enum {                                  /* Eec */
 132         Nvpres          = 1<<8,
 133         Autord          = 1<<9,
 134         Flupd           = 1<<19,
 135         Sec1val         = 1<<22,
 136 };
 137
 138 enum {                                  /* Eerd */
 139         EEstart         = 1<<0,         /* Start Read */
 140         EEdone          = 1<<1,         /* Read done */
 141 };
 142
 143 enum {                                  /* Ctrlext */
 144         Eerst           = 1<<13,        /* EEPROM Reset */
 145         Linkmode        = 3<<22,        /* linkmode */
 146         Internalphy     = 0<<22,        /* " internal phy (copper) */
 147         Sgmii           = 2<<22,        /* " sgmii */
 148         Serdes          = 3<<22,        /* " serdes */
 149 };
 150
 151 enum {
 152         /* Connsw */
 153         Enrgirq         = 1<<2, /* interrupt on power detect (enrgsrc) */
 154 };
 155
 156 enum {                                  /* EEPROM content offsets */
 157         Ea              = 0x00,         /* Ethernet Address */
 158 };
 159
 160 enum {                                  /* Mdic */
 161         MDIdMASK        = 0x0000FFFF,   /* Data */
 162         MDIdSHIFT       = 0,
 163         MDIrMASK        = 0x001F0000,   /* PHY Register Address */
 164         MDIrSHIFT       = 16,
 165         MDIpMASK        = 0x03E00000,   /* PHY Address */
 166         MDIpSHIFT       = 21,
 167         MDIwop          = 0x04000000,   /* Write Operation */
 168         MDIrop          = 0x08000000,   /* Read Operation */
 169         MDIready        = 0x10000000,   /* End of Transaction */
 170         MDIie           = 0x20000000,   /* Interrupt Enable */
 171         MDIe            = 0x40000000,   /* Error */
 172 };
 173
 174 enum {                                  /* phy interface */
 175         Phyctl          = 0,            /* phy ctl register */
 176         Physr           = 1,            /* phy status register */
 177         Phyid1          = 2,            /* phy id1 */
 178         Phyid2          = 3,            /* phy id2 */
 179         Phyisr          = 19,           /* 82563 phy interrupt status register */
 180         Phylhr          = 19,           /* 8257[12] link health register */
 181         Physsr          = 17,           /* phy secondary status register */
 182         Phyprst         = 193<<8 | 17,  /* 8256[34] phy port reset */
 183         Phyier          = 18,           /* 82573 phy interrupt enable register */
 184         Phypage         = 22,           /* 8256[34] page register */
 185         Phystat         = 26,           /* 82580 phy status */
 186         Phyapage        = 29,
 187         Rtlink          = 1<<10,        /* realtime link status */
 188         Phyan           = 1<<11,        /* phy has autonegotiated */
 189
 190         /* Phyctl bits */
 191         Ran             = 1<<9, /* restart auto negotiation */
 192         Ean             = 1<<12,        /* enable auto negotiation */
 193
 194         /* Phyprst bits */
 195         Prst            = 1<<0, /* reset the port */
 196
 197         /* 82573 Phyier bits */
 198         Lscie           = 1<<10,        /* link status changed ie */
 199         Ancie           = 1<<11,        /* auto negotiation complete ie */
 200         Spdie           = 1<<14,        /* speed changed ie */
 201         Panie           = 1<<15,        /* phy auto negotiation error ie */
 202
 203         /* Phylhr/Phyisr bits */
 204         Anf             = 1<<6, /* lhr: auto negotiation fault */
 205         Ane             = 1<<15,        /* isr: auto negotiation error */
 206
 207         /* 82580 Phystat bits */
 208         Ans     = 1<<14 | 1<<15,        /* 82580 autoneg. status */
 209         Link    = 1<<6,         /* 82580 Link */
 210
 211         /* Rxcw builtin serdes */
 212         Anc             = 1<<31,
 213         Rxsynch         = 1<<30,
 214         Rxcfg           = 1<<29,
 215         Rxcfgch         = 1<<28,
 216         Rxcfgbad        = 1<<27,
 217         Rxnc            = 1<<26,
 218
 219         /* Txcw */
 220         Txane           = 1<<31,
 221         Txcfg           = 1<<30,
 222 };
 223
 224 enum {                                  /* fiber (pcs) interface */
 225         Pcsctl  = 0x4208,               /* pcs control */
 226         Pcsstat = 0x420c,               /* pcs status */
 227
 228         /* Pcsctl bits */
 229         Pan     = 1<<16,                /* autoegotiate */
 230         Prestart        = 1<<17,                /* restart an (self clearing) */
 231
 232         /* Pcsstat bits */
 233         Linkok  = 1<<0,         /* link is okay */
 234         Andone  = 1<<16,                /* an phase is done see below for success */
 235         Anbad   = 1<<19 | 1<<20,        /* Anerror | Anremfault */
 236 };
 237
 238 enum {                                  /* Icr, Ics, Ims, Imc */
 239         Txdw            = 0x00000001,   /* Transmit Descriptor Written Back */
 240         Txqe            = 0x00000002,   /* Transmit Queue Empty */
 241         Lsc             = 0x00000004,   /* Link Status Change */
 242         Rxseq           = 0x00000008,   /* Receive Sequence Error */
 243         Rxdmt0          = 0x00000010,   /* Rdesc Minimum Threshold Reached */
 244         Rxo             = 0x00000040,   /* Receiver Overrun */
 245         Rxt0            = 0x00000080,   /* Receiver Timer Interrupt; !82575/6/80 only */
 246         Rxdw            = 0x00000080,   /* Rdesc write back; 82575/6/80 only */
 247         Mdac            = 0x00000200,   /* MDIO Access Completed */
 248         Rxcfgset                = 0x00000400,   /* Receiving /C/ ordered sets */
 249         Ack             = 0x00020000,   /* Receive ACK frame */
 250         Omed            = 1<<20,        /* media change; pcs interface */
 251 };
 252
 253 enum {                                  /* Txcw */
 254         TxcwFd          = 0x00000020,   /* Full Duplex */
 255         TxcwHd          = 0x00000040,   /* Half Duplex */
 256         TxcwPauseMASK   = 0x00000180,   /* Pause */
 257         TxcwPauseSHIFT  = 7,
 258         TxcwPs          = 1<<TxcwPauseSHIFT,    /* Pause Supported */
 259         TxcwAs          = 2<<TxcwPauseSHIFT,    /* Asymmetric FC desired */
 260         TxcwRfiMASK     = 0x00003000,   /* Remote Fault Indication */
 261         TxcwRfiSHIFT    = 12,
 262         TxcwNpr         = 0x00008000,   /* Next Page Request */
 263         TxcwConfig      = 0x40000000,   /* Transmit COnfig Control */
 264         TxcwAne         = 0x80000000,   /* Auto-Negotiation Enable */
 265 };
 266
 267 enum {                                  /* Rctl */
 268         Rrst            = 0x00000001,   /* Receiver Software Reset */
 269         Ren             = 0x00000002,   /* Receiver Enable */
 270         Sbp             = 0x00000004,   /* Store Bad Packets */
 271         Upe             = 0x00000008,   /* Unicast Promiscuous Enable */
 272         Mpe             = 0x00000010,   /* Multicast Promiscuous Enable */
 273         Lpe             = 0x00000020,   /* Long Packet Reception Enable */
 274         RdtmsMASK       = 0x00000300,   /* Rdesc Minimum Threshold Size */
 275         RdtmsHALF       = 0x00000000,   /* Threshold is 1/2 Rdlen */
 276         RdtmsQUARTER    = 0x00000100,   /* Threshold is 1/4 Rdlen */
 277         RdtmsEIGHTH     = 0x00000200,   /* Threshold is 1/8 Rdlen */
 278         MoMASK          = 0x00003000,   /* Multicast Offset */
 279         Bam             = 0x00008000,   /* Broadcast Accept Mode */
 280         BsizeMASK       = 0x00030000,   /* Receive Buffer Size */
 281         Bsize16384      = 0x00010000,   /* Bsex = 1 */
 282         Bsize8192       = 0x00020000,   /* Bsex = 1 */
 283         Bsize2048       = 0x00000000,
 284         Bsize1024       = 0x00010000,
 285         Bsize512        = 0x00020000,
 286         Bsize256        = 0x00030000,
 287         BsizeFlex       = 0x08000000,   /* Flexable Bsize in 1kb increments */
 288         Vfe             = 0x00040000,   /* VLAN Filter Enable */
 289         Cfien           = 0x00080000,   /* Canonical Form Indicator Enable */
 290         Cfi             = 0x00100000,   /* Canonical Form Indicator value */
 291         Dpf             = 0x00400000,   /* Discard Pause Frames */
 292         Pmcf            = 0x00800000,   /* Pass MAC Control Frames */
 293         Bsex            = 0x02000000,   /* Buffer Size Extension */
 294         Secrc           = 0x04000000,   /* Strip CRC from incoming packet */
 295 };
 296
 297 enum {                                  /* Srrctl */
 298         Dropen          = 1<<31,
 299 };
 300
 301 enum {                                  /* Tctl */
 302         Trst            = 0x00000001,   /* Transmitter Software Reset */
 303         Ten             = 0x00000002,   /* Transmit Enable */
 304         Psp             = 0x00000008,   /* Pad Short Packets */
 305         Mulr            = 0x10000000,   /* Allow multiple concurrent requests */
 306         CtMASK          = 0x00000FF0,   /* Collision Threshold */
 307         CtSHIFT         = 4,
 308         ColdMASK        = 0x003FF000,   /* Collision Distance */
 309         ColdSHIFT       = 12,
 310         Swxoff          = 0x00400000,   /* Sofware XOFF Transmission */
 311         Pbe             = 0x00800000,   /* Packet Burst Enable */
 312         Rtlc            = 0x01000000,   /* Re-transmit on Late Collision */
 313         Nrtu            = 0x02000000,   /* No Re-transmit on Underrrun */
 314 };
 315
 316 enum {                                  /* [RT]xdctl */
 317         PthreshMASK     = 0x0000003F,   /* Prefetch Threshold */
 318         PthreshSHIFT    = 0,
 319         HthreshMASK     = 0x00003F00,   /* Host Threshold */
 320         HthreshSHIFT    = 8,
 321         WthreshMASK     = 0x003F0000,   /* Writeback Threshold */
 322         WthreshSHIFT    = 16,
 323         Gran            = 0x01000000,   /* Granularity; not 82575 */
 324         Enable          = 0x02000000,
 325 };
 326
 327 enum {                                  /* Rxcsum */
 328         Ipofl           = 0x0100,       /* IP Checksum Off-load Enable */
 329         Tuofl           = 0x0200,       /* TCP/UDP Checksum Off-load Enable */
 330 };
 331
 332 typedef struct Rd {                     /* Receive Descriptor */
 333         u32int  addr[2];
 334         u16int  length;
 335         u16int  checksum;
 336         uchar   status;
 337         uchar   errors;
 338         u16int  special;
 339 } Rd;
 340
 341 enum {                                  /* Rd status */
 342         Rdd             = 0x01,         /* Descriptor Done */
 343         Reop            = 0x02,         /* End of Packet */
 344         Ixsm            = 0x04,         /* Ignore Checksum Indication */
 345         Vp              = 0x08,         /* Packet is 802.1Q (matched VET) */
 346         Tcpcs           = 0x20,         /* TCP Checksum Calculated on Packet */
 347         Ipcs            = 0x40,         /* IP Checksum Calculated on Packet */
 348         Pif             = 0x80,         /* Passed in-exact filter */
 349 };
 350
 351 enum {                                  /* Rd errors */
 352         Ce              = 0x01,         /* CRC Error or Alignment Error */
 353         Se              = 0x02,         /* Symbol Error */
 354         Seq             = 0x04,         /* Sequence Error */
 355         Cxe             = 0x10,         /* Carrier Extension Error */
 356         Tcpe            = 0x20,         /* TCP/UDP Checksum Error */
 357         Ipe             = 0x40,         /* IP Checksum Error */
 358         Rxe             = 0x80,         /* RX Data Error */
 359 };
 360
 361 typedef struct {                        /* Transmit Descriptor */
 362         u32int  addr[2];                /* Data */
 363         u32int  control;
 364         u32int  status;
 365 } Td;
 366
 367 enum {                                  /* Tdesc control */
 368         LenMASK         = 0x000FFFFF,   /* Data/Packet Length Field */
 369         LenSHIFT        = 0,
 370         DtypeCD         = 0x00000000,   /* Data Type 'Context Descriptor' */
 371         DtypeDD         = 0x00100000,   /* Data Type 'Data Descriptor' */
 372         PtypeTCP        = 0x01000000,   /* TCP/UDP Packet Type (CD) */
 373         Teop            = 0x01000000,   /* End of Packet (DD) */
 374         PtypeIP         = 0x02000000,   /* IP Packet Type (CD) */
 375         Ifcs            = 0x02000000,   /* Insert FCS (DD) */
 376         Tse             = 0x04000000,   /* TCP Segmentation Enable */
 377         Rs              = 0x08000000,   /* Report Status */
 378         Rps             = 0x10000000,   /* Report Status Sent */
 379         Dext            = 0x20000000,   /* Descriptor Extension */
 380         Vle             = 0x40000000,   /* VLAN Packet Enable */
 381         Ide             = 0x80000000,   /* Interrupt Delay Enable */
 382 };
 383
 384 enum {                                  /* Tdesc status */
 385         Tdd             = 0x0001,       /* Descriptor Done */
 386         Ec              = 0x0002,       /* Excess Collisions */
 387         Lc              = 0x0004,       /* Late Collision */
 388         Tu              = 0x0008,       /* Transmit Underrun */
 389         CssMASK         = 0xFF00,       /* Checksum Start Field */
 390         CssSHIFT        = 8,
 391 };
 392
 393 typedef struct {
 394         u16int  *reg;
 395         u32int  *reg32;
 396         uint    base;
 397         uint    lim;
 398 } Flash;
 399
 400 enum {
 401         /* 16 and 32-bit flash registers for ich flash parts */
 402         Bfpr    = 0x00/4,               /* flash base 0:12; lim 16:28 */
 403         Fsts    = 0x04/2,               /* flash status; Hsfsts */
 404         Fctl    = 0x06/2,               /* flash control; Hsfctl */
 405         Faddr   = 0x08/4,               /* flash address to r/w */
 406         Fdata   = 0x10/4,               /* data @ address */
 407
 408         /* status register */
 409         Fdone   = 1<<0,                 /* flash cycle done */
 410         Fcerr   = 1<<1,                 /* cycle error; write 1 to clear */
 411         Ael     = 1<<2,                 /* direct access error log; 1 to clear */
 412         Scip    = 1<<5,                 /* spi cycle in progress */
 413         Fvalid  = 1<<14,                /* flash descriptor valid */
 414
 415         /* control register */
 416         Fgo     = 1<<0,                 /* start cycle */
 417         Flcycle = 1<<1,                 /* two bits: r=0; w=2 */
 418         Fdbc    = 1<<8,                 /* bytes to read; 5 bits */
 419 };
 420
 421 enum {
 422         Nrd             = 256,          /* power of two */
 423         Ntd             = 128,          /* power of two */
 424         Nrb             = 512+512,      /* private receive buffers per Ctlr */
 425         Rbalign         = BY2PG,        /* rx buffer alignment */
 426 };
 427
 428 /*
 429  * cavet emptor: 82577/78 have been entered speculatitively.
 430  * awating datasheet from intel.
 431  */
 432 enum {
 433         i82563,
 434         i82566,
 435         i82567,
 436         i82567m,
 437         i82571,
 438         i82572,
 439         i82573,
 440         i82574,
 441         i82575,
 442         i82576,
 443         i82577,
 444         i82577m,
 445         i82578,
 446         i82578m,
 447         i82579,
 448         i82580,
 449         i82583,
 450         i210,
 451         i217,
 452         i218,
 453         i219,
 454         i350,
 455         Nctlrtype,
 456 };
 457
 458 enum {
 459         Fload   = 1<<0,
 460         Fert    = 1<<1,
 461         F75     = 1<<2,
 462         Fpba    = 1<<3,
 463         Fflashea= 1<<4,
 464         F79phy  = 1<<5,
 465         Fnofct  = 1<<6,
 466         Fbadcsum= 1<<7,
 467         Fnofca  = 1<<8,
 468 };
 469
 470 typedef struct Ctlrtype Ctlrtype;
 471 struct Ctlrtype {
 472         char    *name;
 473         int     mtu;
 474         int     flag;
 475 };
 476
 477 static Ctlrtype cttab[Nctlrtype] = {
 478 [i82563]        "i82563",       9014,   Fpba,
 479 [i82566]        "i82566",       1514,   Fload,
 480 [i82567]        "i82567",       9234,   Fload,
 481 [i82567m]       "i82567m",      1514,   Fload,
 482 [i82571]        "i82571",       9234,   Fpba,
 483 [i82572]        "i82572",       9234,   Fpba,
 484 [i82573]        "i82573",       8192,   Fert|Fbadcsum,          /* terrible perf above 8k */
 485 [i82574]        "i82574",       9018,   0,
 486 [i82575]        "i82575",       9728,   F75|Fflashea,
 487 [i82576]        "i82576",       9728,   F75,
 488 [i82577]        "i82577",       4096,   Fload|Fert,
 489 [i82577m]       "i82577",       1514,   Fload|Fert,
 490 [i82578]        "i82578",       4096,   Fload|Fert,
 491 [i82578m]       "i82578",       1514,   Fload|Fert,
 492 [i82579]        "i82579",       9018,   Fload|Fert|F79phy|Fnofct,
 493 [i82580]        "i82580",       9728,   F75|F79phy,
 494 [i82583]        "i82583",       1514,   0,
 495 [i210]          "i210",         9728,   F75|Fnofct|Fert,
 496 [i217]          "i217",         2048,   Fload|Fert|F79phy|Fnofct|Fnofca|Fbadcsum,/* 9018, but unstable above 2k */
 497 [i218]          "i218",         9018,   Fload|Fert|F79phy|Fnofct|Fnofca|Fbadcsum,
 498 [i219]          "i219",         9018,   Fload|Fert|F79phy|Fnofct|Fnofca|Fbadcsum,
 499 [i350]          "i350",         9728,   F75|F79phy|Fnofct,
 500 };
 501
 502 typedef void (*Freefn)(Block*);
 503
 504 typedef struct Ctlr Ctlr;
 505 struct Ctlr {
 506         uvlong  port;
 507         Pcidev  *pcidev;
 508         Ctlr    *next;
 509         int     active;
 510         int     type;
 511         u16int  eeprom[0x40];
 512
 513         QLock   alock;                  /* attach */
 514         void    *alloc;                 /* receive/transmit descriptors */
 515         int     nrd;
 516         int     ntd;
 517         int     rbsz;
 518
 519         u32int  *nic;
 520         Lock    imlock;
 521         int     im;                     /* interrupt mask */
 522
 523         Rendez  lrendez;
 524         int     lim;
 525
 526         QLock   slock;
 527         u32int  statistics[Nstatistics];
 528         uint    lsleep;
 529         uint    lintr;
 530         uint    rsleep;
 531         uint    rintr;
 532         uint    txdw;
 533         uint    tintr;
 534         uint    ixsm;
 535         uint    ipcs;
 536         uint    tcpcs;
 537         uint    speeds[4];
 538         uint    phyerrata;
 539
 540         uchar   ra[Eaddrlen];           /* receive address */
 541         u32int  mta[128];               /* multicast table array */
 542
 543         Rendez  rrendez;
 544         int     rim;
 545         int     rdfree;
 546         Rd      *rdba;                  /* receive descriptor base address */
 547         Block   **rb;                   /* receive buffers */
 548         uint    rdh;                    /* receive descriptor head */
 549         uint    rdt;                    /* receive descriptor tail */
 550         int     rdtr;                   /* receive delay timer ring value */
 551         int     radv;                   /* receive interrupt absolute delay timer */
 552
 553         Rendez  trendez;
 554         QLock   tlock;
 555         int     tbusy;
 556         Td      *tdba;                  /* transmit descriptor base address */
 557         Block   **tb;                   /* transmit buffers */
 558         int     tdh;                    /* transmit descriptor head */
 559         int     tdt;                    /* transmit descriptor tail */
 560
 561         int     fcrtl;
 562         int     fcrth;
 563
 564         u32int  pba;                    /* packet buffer allocation */
 565 };
 566
 567 #define csr32r(c, r)    (*((c)->nic+((r)/4)))
 568 #define csr32w(c, r, v) (*((c)->nic+((r)/4)) = (v))
 569
 570 static  Ctlr    *i82563ctlrhead;
 571 static  Ctlr    *i82563ctlrtail;
 572
 573 static char *statistics[Nstatistics] = {
 574         "CRC Error",
 575         "Alignment Error",
 576         "Symbol Error",
 577         "RX Error",
 578         "Missed Packets",
 579         "Single Collision",
 580         "Excessive Collisions",
 581         "Multiple Collision",
 582         "Late Collisions",
 583         nil,
 584         "Collision",
 585         "Transmit Underrun",
 586         "Defer",
 587         "Transmit - No CRS",
 588         "Sequence Error",
 589         "Carrier Extension Error",
 590         "Receive Error Length",
 591         nil,
 592         "XON Received",
 593         "XON Transmitted",
 594         "XOFF Received",
 595         "XOFF Transmitted",
 596         "FC Received Unsupported",
 597         "Packets Received (64 Bytes)",
 598         "Packets Received (65-127 Bytes)",
 599         "Packets Received (128-255 Bytes)",
 600         "Packets Received (256-511 Bytes)",
 601         "Packets Received (512-1023 Bytes)",
 602         "Packets Received (1024-mtu Bytes)",
 603         "Good Packets Received",
 604         "Broadcast Packets Received",
 605         "Multicast Packets Received",
 606         "Good Packets Transmitted",
 607         nil,
 608         "Good Octets Received",
 609         nil,
 610         "Good Octets Transmitted",
 611         nil,
 612         nil,
 613         nil,
 614         "Receive No Buffers",
 615         "Receive Undersize",
 616         "Receive Fragment",
 617         "Receive Oversize",
 618         "Receive Jabber",
 619         "Management Packets Rx",
 620         "Management Packets Drop",
 621         "Management Packets Tx",
 622         "Total Octets Received",
 623         nil,
 624         "Total Octets Transmitted",
 625         nil,
 626         "Total Packets Received",
 627         "Total Packets Transmitted",
 628         "Packets Transmitted (64 Bytes)",
 629         "Packets Transmitted (65-127 Bytes)",
 630         "Packets Transmitted (128-255 Bytes)",
 631         "Packets Transmitted (256-511 Bytes)",
 632         "Packets Transmitted (512-1023 Bytes)",
 633         "Packets Transmitted (1024-mtu Bytes)",
 634         "Multicast Packets Transmitted",
 635         "Broadcast Packets Transmitted",
 636         "TCP Segmentation Context Transmitted",
 637         "TCP Segmentation Context Fail",
 638         "Interrupt Assertion",
 639         "Interrupt Rx Pkt Timer",
 640         "Interrupt Rx Abs Timer",
 641         "Interrupt Tx Pkt Timer",
 642         "Interrupt Tx Abs Timer",
 643         "Interrupt Tx Queue Empty",
 644         "Interrupt Tx Desc Low",
 645         "Interrupt Rx Min",
 646         "Interrupt Rx Overrun",
 647 };
 648
 649 static char*
 650 cname(Ctlr *c)
 651 {
 652         return cttab[c->type].name;
 653 }
 654
 655 static long
 656 i82563ifstat(Ether *edev, void *a, long n, ulong offset)
 657 {
 658         char *s, *p, *e, *stat;
 659         int i, r;
 660         uvlong tuvl, ruvl;
 661         Ctlr *ctlr;
 662
 663         p = s = smalloc(READSTR);
 664         e = p + READSTR;
 665
 666         ctlr = edev->ctlr;
 667         qlock(&ctlr->slock);
 668
 669         for(i = 0; i < Nstatistics; i++){
 670                 r = csr32r(ctlr, Statistics + i*4);
 671                 if((stat = statistics[i]) == nil)
 672                         continue;
 673                 switch(i){
 674                 case Gorcl:
 675                 case Gotcl:
 676                 case Torl:
 677                 case Totl:
 678                         ruvl = r;
 679                         ruvl += (uvlong)csr32r(ctlr, Statistics+(i+1)*4) << 32;
 680                         tuvl = ruvl;
 681                         tuvl += ctlr->statistics[i];
 682                         tuvl += (uvlong)ctlr->statistics[i+1] << 32;
 683                         if(tuvl == 0)
 684                                 continue;
 685                         ctlr->statistics[i] = tuvl;
 686                         ctlr->statistics[i+1] = tuvl >> 32;
 687                         p = seprint(p, e, "%s: %llud %llud\n", stat, tuvl, ruvl);
 688                         i++;
 689                         break;
 690
 691                 default:
 692                         ctlr->statistics[i] += r;
 693                         if(ctlr->statistics[i] == 0)
 694                                 continue;
 695                         p = seprint(p, e, "%s: %ud %ud\n", stat,
 696                                 ctlr->statistics[i], r);
 697                         break;
 698                 }
 699         }
 700
 701         p = seprint(p, e, "lintr: %ud %ud\n", ctlr->lintr, ctlr->lsleep);
 702         p = seprint(p, e, "rintr: %ud %ud\n", ctlr->rintr, ctlr->rsleep);
 703         p = seprint(p, e, "tintr: %ud %ud\n", ctlr->tintr, ctlr->txdw);
 704         p = seprint(p, e, "ixcs: %ud %ud %ud\n", ctlr->ixsm, ctlr->ipcs, ctlr->tcpcs);
 705         p = seprint(p, e, "rdtr: %ud\n", ctlr->rdtr);
 706         p = seprint(p, e, "radv: %ud\n", ctlr->radv);
 707         p = seprint(p, e, "ctrl: %.8ux\n", csr32r(ctlr, Ctrl));
 708         p = seprint(p, e, "ctrlext: %.8ux\n", csr32r(ctlr, Ctrlext));
 709         p = seprint(p, e, "status: %.8ux\n", csr32r(ctlr, Status));
 710         p = seprint(p, e, "txcw: %.8ux\n", csr32r(ctlr, Txcw));
 711         p = seprint(p, e, "txdctl: %.8ux\n", csr32r(ctlr, Txdctl));
 712         p = seprint(p, e, "pba: %.8ux\n", ctlr->pba);
 713
 714         p = seprint(p, e, "speeds: 10:%ud 100:%ud 1000:%ud ?:%ud\n",
 715                 ctlr->speeds[0], ctlr->speeds[1], ctlr->speeds[2], ctlr->speeds[3]);
 716         p = seprint(p, e, "type: %s\n", cname(ctlr));
 717
 718         p = seprint(p, e, "eeprom:");
 719         for(i = 0; i < 0x40; i++){
 720                 if(i && ((i & 7) == 0))
 721                         p = seprint(p, e, "\n       ");
 722                 p = seprint(p, e, " %4.4ux", ctlr->eeprom[i]);
 723         }
 724         p = seprint(p, e, "\n");
 725
 726         USED(p);
 727         n = readstr(offset, a, n, s);
 728         free(s);
 729         qunlock(&ctlr->slock);
 730
 731         return n;
 732 }
 733
 734 static void
 735 i82563promiscuous(void *arg, int on)
 736 {
 737         int rctl;
 738         Ctlr *ctlr;
 739         Ether *edev;
 740
 741         edev = arg;
 742         ctlr = edev->ctlr;
 743
 744         rctl = csr32r(ctlr, Rctl);
 745         rctl &= ~MoMASK;
 746         if(on)
 747                 rctl |= Upe|Mpe;
 748         else
 749                 rctl &= ~(Upe|Mpe);
 750         csr32w(ctlr, Rctl, rctl);
 751 }
 752
 753 static void
 754 i82563multicast(void *arg, uchar *addr, int on)
 755 {
 756         int bit, x;
 757         Ctlr *ctlr;
 758         Ether *edev;
 759
 760         edev = arg;
 761         ctlr = edev->ctlr;
 762
 763         switch(ctlr->type){
 764         case i82566:
 765         case i82567:
 766         case i82567m:
 767         case i82577:
 768         case i82577m:
 769         case i82579:
 770         case i217:
 771         case i218:
 772         case i219:
 773                 bit = (addr[5]<<2)|(addr[4]>>6);
 774                 x = (bit>>5) & 31;
 775                 break;
 776         default:
 777                 bit = (addr[5]<<4)|(addr[4]>>4);
 778                 x = (bit>>5) & 127;
 779                 break;
 780         }
 781         bit &= 31;
 782
 783         /*
 784          * multiple ether addresses can hash to the same filter bit,
 785          * so it's never safe to clear a filter bit.
 786          * if we want to clear filter bits, we need to keep track of
 787          * all the multicast addresses in use, clear all the filter bits,
 788          * then set the ones corresponding to in-use addresses.
 789          */
 790         if(on)
 791                 ctlr->mta[x] |= 1<<bit;
 792
 793         csr32w(ctlr, Mta+x*4, ctlr->mta[x]);
 794 }
 795
 796 static void
 797 i82563im(Ctlr *ctlr, int im)
 798 {
 799         ilock(&ctlr->imlock);
 800         ctlr->im |= im;
 801         csr32w(ctlr, Ims, ctlr->im);
 802         iunlock(&ctlr->imlock);
 803 }
 804
 805 static void
 806 i82563txinit(Ctlr *ctlr)
 807 {
 808         u32int r;
 809         Block *b;
 810         int i;
 811
 812         if(cttab[ctlr->type].flag & F75)
 813                 csr32w(ctlr, Tctl, 0x0F<<CtSHIFT | Psp);
 814         else
 815                 csr32w(ctlr, Tctl, 0x0F<<CtSHIFT | Psp | 66<<ColdSHIFT | Mulr);
 816         csr32w(ctlr, Tipg, 6<<20 | 8<<10 | 8);          /* yb sez: 0x702008 */
 817         for(i = 0; i < ctlr->ntd; i++){
 818                 if((b = ctlr->tb[i]) != nil){
 819                         ctlr->tb[i] = nil;
 820                         freeb(b);
 821                 }
 822                 memset(&ctlr->tdba[i], 0, sizeof(Td));
 823         }
 824         csr32w(ctlr, Tdbal, PCIWADDR(ctlr->tdba));
 825         csr32w(ctlr, Tdbah, 0);
 826         csr32w(ctlr, Tdlen, ctlr->ntd * sizeof(Td));
 827         ctlr->tdh = PREV(0, ctlr->ntd);
 828         csr32w(ctlr, Tdh, 0);
 829         ctlr->tdt = 0;
 830         csr32w(ctlr, Tdt, 0);
 831         csr32w(ctlr, Tidv, 128);
 832         csr32w(ctlr, Tadv, 64);
 833         csr32w(ctlr, Tctl, csr32r(ctlr, Tctl) | Ten);
 834         r = csr32r(ctlr, Txdctl) & ~WthreshMASK;
 835         r |= 4<<WthreshSHIFT | 4<<PthreshSHIFT;
 836         if(cttab[ctlr->type].flag & F75)
 837                 r |= Enable;
 838         csr32w(ctlr, Txdctl, r);
 839 }
 840
 841 static uint
 842 i82563cleanup(Ctlr *c)
 843 {
 844         Block *b;
 845         uint tdh, n;
 846
 847         tdh = c->tdh;
 848         while(c->tdba[n = NEXT(tdh, c->ntd)].status & Tdd){
 849                 tdh = n;
 850                 if((b = c->tb[tdh]) != nil){
 851                         c->tb[tdh] = nil;
 852                         freeb(b);
 853                 }else
 854                         iprint("82563 tx underrun!\n");
 855                 c->tdba[tdh].status = 0;
 856         }
 857
 858         return c->tdh = tdh;
 859 }
 860
 861 static int
 862 notrim(void *v)
 863 {
 864         Ctlr *c;
 865
 866         c = v;
 867         return (c->im & Txdw) == 0;
 868 }
 869
 870 static void
 871 i82563tproc(void *v)
 872 {
 873         Td *td;
 874         Block *bp;
 875         Ether *edev;
 876         Ctlr *ctlr;
 877         uint tdt, n;
 878
 879         edev = v;
 880         ctlr = edev->ctlr;
 881         i82563txinit(ctlr);
 882
 883         tdt = ctlr->tdt;
 884         while(waserror())
 885                 ;
 886         for(;;){
 887                 n = NEXT(tdt, ctlr->ntd);
 888                 if(n == i82563cleanup(ctlr)){
 889                         ctlr->txdw++;
 890                         i82563im(ctlr, Txdw);
 891                         sleep(&ctlr->trendez, notrim, ctlr);
 892                         continue;
 893                 }
 894                 bp = qbread(edev->oq, 100000);
 895                 td = &ctlr->tdba[tdt];
 896                 td->addr[0] = PCIWADDR(bp->rp);
 897                 td->addr[1] = 0;
 898                 td->control = Ide|Rs|Ifcs|Teop|BLEN(bp);
 899                 coherence();
 900                 ctlr->tb[tdt] = bp;
 901                 ctlr->tdt = tdt = n;
 902                 csr32w(ctlr, Tdt, tdt);
 903         }
 904 }
 905
 906 static void
 907 i82563replenish(Ctlr *ctlr)
 908 {
 909         uint rdt, i;
 910         Block *bp;
 911         Rd *rd;
 912
 913         i = 0;
 914         for(rdt = ctlr->rdt; NEXT(rdt, ctlr->nrd) != ctlr->rdh; rdt = NEXT(rdt, ctlr->nrd)){
 915                 rd = &ctlr->rdba[rdt];
 916                 if(ctlr->rb[rdt] != nil){
 917                         iprint("82563: tx overrun\n");
 918                         break;
 919                 }
 920                 i++;
 921                 bp = allocb(ctlr->rbsz + Rbalign);
 922                 bp->rp = bp->wp = (uchar*)ROUND((uintptr)bp->base, Rbalign);
 923                 ctlr->rb[rdt] = bp;
 924                 rd->addr[0] = PCIWADDR(bp->rp);
 925                 rd->addr[1] = 0;
 926                 rd->status = 0;
 927                 ctlr->rdfree++;
 928         }
 929         if(i != 0){
 930                 coherence();
 931                 ctlr->rdt = rdt;
 932                 csr32w(ctlr, Rdt, rdt);
 933         }
 934 }
 935
 936 static void
 937 i82563rxinit(Ctlr *ctlr)
 938 {
 939         int i;
 940         Block *bp;
 941
 942         if(ctlr->rbsz <= 2048)
 943                 csr32w(ctlr, Rctl, Dpf|Bsize2048|Bam|RdtmsHALF);
 944         else{
 945                 i = ctlr->rbsz / 1024;
 946                 if(cttab[ctlr->type].flag & F75){
 947                         csr32w(ctlr, Rctl, Lpe|Dpf|Bsize2048|Bam|RdtmsHALF|Secrc);
 948                         if(ctlr->type != i82575)
 949                                 i |= (ctlr->nrd/2>>4)<<20;              /* RdmsHalf */
 950                         csr32w(ctlr, Srrctl, i | Dropen);
 951                         csr32w(ctlr, Rmpl, ctlr->rbsz);
 952 //                      csr32w(ctlr, Drxmxod, 0x7ff);
 953                 }else
 954                         csr32w(ctlr, Rctl, Lpe|Dpf|BsizeFlex*i|Bam|RdtmsHALF|Secrc);
 955         }
 956
 957         if(cttab[ctlr->type].flag & Fert)
 958                 csr32w(ctlr, Ert, 1024/8);
 959
 960         if(ctlr->type == i82566)
 961                 csr32w(ctlr, Pbs, 16);
 962
 963         csr32w(ctlr, Rdbal, PCIWADDR(ctlr->rdba));
 964         csr32w(ctlr, Rdbah, 0);
 965         csr32w(ctlr, Rdlen, ctlr->nrd * sizeof(Rd));
 966         ctlr->rdh = 0;
 967         csr32w(ctlr, Rdh, 0);
 968         ctlr->rdt = 0;
 969         csr32w(ctlr, Rdt, 0);
 970         ctlr->rdtr = 25;
 971         ctlr->radv = 500;
 972         csr32w(ctlr, Rdtr, ctlr->rdtr);
 973         csr32w(ctlr, Radv, ctlr->radv);
 974
 975         for(i = 0; i < ctlr->nrd; i++)
 976                 if((bp = ctlr->rb[i]) != nil){
 977                         ctlr->rb[i] = nil;
 978                         freeb(bp);
 979                 }
 980         if(cttab[ctlr->type].flag & F75)
 981                 csr32w(ctlr, Rxdctl, 1<<WthreshSHIFT | 8<<PthreshSHIFT | 1<<HthreshSHIFT | Enable);
 982         else
 983                 csr32w(ctlr, Rxdctl, 2<<WthreshSHIFT | 2<<PthreshSHIFT);
 984
 985         /*
 986          * Enable checksum offload.
 987          */
 988         csr32w(ctlr, Rxcsum, Tuofl | Ipofl | ETHERHDRSIZE);
 989 }
 990
 991 static int
 992 i82563rim(void *v)
 993 {
 994         return ((Ctlr*)v)->rim != 0;
 995 }
 996
 997 static void
 998 i82563rproc(void *arg)
 999 {
1000         uint rdh, rim, im;
1001         Block *bp;
1002         Ctlr *ctlr;
1003         Ether *edev;
1004         Rd *rd;
1005
1006         edev = arg;
1007         ctlr = edev->ctlr;
1008
1009         i82563rxinit(ctlr);
1010         csr32w(ctlr, Rctl, csr32r(ctlr, Rctl) | Ren);
1011         if(cttab[ctlr->type].flag & F75){
1012                 csr32w(ctlr, Rxdctl, csr32r(ctlr, Rxdctl) | Enable);
1013                 im = Rxt0|Rxo|Rxdmt0|Rxseq|Ack;
1014         }else
1015                 im = Rxt0|Rxo|Rxdmt0|Rxseq|Ack;
1016
1017         while(waserror())
1018                 ;
1019         for(;;){
1020                 i82563im(ctlr, im);
1021                 ctlr->rsleep++;
1022                 i82563replenish(ctlr);
1023                 sleep(&ctlr->rrendez, i82563rim, ctlr);
1024
1025                 rdh = ctlr->rdh;
1026                 for(;;){
1027                         rim = ctlr->rim;
1028                         ctlr->rim = 0;
1029                         rd = &ctlr->rdba[rdh];
1030                         if(!(rd->status & Rdd))
1031                                 break;
1032
1033                         /*
1034                          * Accept eop packets with no errors.
1035                          * With no errors and the Ixsm bit set,
1036                          * the descriptor status Tpcs and Ipcs bits give
1037                          * an indication of whether the checksums were
1038                          * calculated and valid.
1039                          */
1040                         bp = ctlr->rb[rdh];
1041                         if((rd->status & Reop) && rd->errors == 0){
1042                                 bp->wp += rd->length;
1043                                 if(!(rd->status & Ixsm)){
1044                                         ctlr->ixsm++;
1045                                         if(rd->status & Ipcs){
1046                                                 /*
1047                                                  * IP checksum calculated
1048                                                  * (and valid as errors == 0).
1049                                                  */
1050                                                 ctlr->ipcs++;
1051                                                 bp->flag |= Bipck;
1052                                         }
1053                                         if(rd->status & Tcpcs){
1054                                                 /*
1055                                                  * TCP/UDP checksum calculated
1056                                                  * (and valid as errors == 0).
1057                                                  */
1058                                                 ctlr->tcpcs++;
1059                                                 bp->flag |= Btcpck|Budpck;
1060                                         }
1061                                         bp->checksum = rd->checksum;
1062                                         bp->flag |= Bpktck;
1063                                 }
1064                                 etheriq(edev, bp);
1065                         } else
1066                                 freeb(bp);
1067                         ctlr->rb[rdh] = nil;
1068                         ctlr->rdfree--;
1069                         ctlr->rdh = rdh = NEXT(rdh, ctlr->nrd);
1070                         if(ctlr->nrd-ctlr->rdfree >= 32 || (rim & Rxdmt0))
1071                                 i82563replenish(ctlr);
1072                 }
1073         }
1074 }
1075
1076 static int
1077 i82563lim(void *v)
1078 {
1079         return ((Ctlr*)v)->lim != 0;
1080 }
1081
1082 static int speedtab[] = {
1083         10, 100, 1000, 0
1084 };
1085
1086 static uint
1087 phyread(Ctlr *c, int phyno, int reg)
1088 {
1089         uint phy, i;
1090
1091         csr32w(c, Mdic, MDIrop | phyno<<MDIpSHIFT | reg<<MDIrSHIFT);
1092         phy = 0;
1093         for(i = 0; i < 64; i++){
1094                 phy = csr32r(c, Mdic);
1095                 if(phy & (MDIe|MDIready))
1096                         break;
1097                 microdelay(1);
1098         }
1099         if((phy & (MDIe|MDIready)) != MDIready){
1100                 print("%s: phy %d wedged %.8ux\n", cname(c), phyno, phy);
1101                 return ~0;
1102         }
1103         return phy & 0xffff;
1104 }
1105
1106 static uint
1107 phywrite0(Ctlr *c, int phyno, int reg, ushort val)
1108 {
1109         uint phy, i;
1110
1111         csr32w(c, Mdic, MDIwop | phyno<<MDIpSHIFT | reg<<MDIrSHIFT | val);
1112         phy = 0;
1113         for(i = 0; i < 64; i++){
1114                 phy = csr32r(c, Mdic);
1115                 if(phy & (MDIe|MDIready))
1116                         break;
1117                 microdelay(1);
1118         }
1119         if((phy & (MDIe|MDIready)) != MDIready)
1120                 return ~0;
1121         return 0;
1122 }
1123
1124 static uint
1125 setpage(Ctlr *c, uint phyno, uint p, uint r)
1126 {
1127         uint pr;
1128
1129         if(c->type == i82563){
1130                 if(r >= 16 && r <= 28 && r != 22)
1131                         pr = Phypage;
1132                 else if(r == 30 || r == 31)
1133                         pr = Phyapage;
1134                 else
1135                         return 0;
1136                 return phywrite0(c, phyno, pr, p);
1137         }else if(p == 0)
1138                 return 0;
1139         return ~0;
1140 }
1141
1142 static uint
1143 phywrite(Ctlr *c, uint phyno, uint reg, ushort v)
1144 {
1145         if(setpage(c, phyno, reg>>8, reg & 0xff) == ~0)
1146                 panic("%s: bad phy reg %.4ux", cname(c), reg);
1147         return phywrite0(c, phyno, reg & 0xff, v);
1148 }
1149
1150 static void
1151 phyerrata(Ether *e, Ctlr *c, uint phyno)
1152 {
1153         if(e->mbps == 0){
1154                 if(c->phyerrata == 0){
1155                         c->phyerrata++;
1156                         phywrite(c, phyno, Phyprst, Prst);      /* try a port reset */
1157                         print("%s: phy port reset\n", cname(c));
1158                 }
1159         }else
1160                 c->phyerrata = 0;
1161 }
1162
1163 static uint
1164 phyprobe(Ctlr *c, uint mask)
1165 {
1166         uint phy, phyno;
1167
1168         for(phyno=0; mask != 0; phyno++, mask>>=1){
1169                 if((mask & 1) == 0)
1170                         continue;
1171                 if(phyread(c, phyno, Physr) == ~0)
1172                         continue;
1173                 phy = (phyread(c, phyno, Phyid1) & 0x3FFF)<<6;
1174                 phy |= phyread(c, phyno, Phyid2) >> 10;
1175                 if(phy == 0xFFFFF || phy == 0)
1176                         continue;
1177                 print("%s: phy%d oui %#ux\n", cname(c), phyno, phy);
1178                 return phyno;
1179         }
1180         print("%s: no phy\n", cname(c));
1181         return ~0;
1182 }
1183
1184 static void
1185 lsleep(Ctlr *c, uint m)
1186 {
1187         c->lim = 0;
1188         i82563im(c, m);
1189         c->lsleep++;
1190         sleep(&c->lrendez, i82563lim, c);
1191 }
1192
1193 static void
1194 phyl79proc(void *v)
1195 {
1196         uint i, r, phy, phyno;
1197         Ctlr *c;
1198         Ether *e;
1199
1200         e = v;
1201         c = e->ctlr;
1202         while(waserror())
1203                 ;
1204
1205         while((phyno = phyprobe(c, 3<<1)) == ~0)
1206                 lsleep(c, Lsc);
1207
1208         for(;;){
1209                 phy = 0;
1210                 for(i=0; i<4; i++){
1211                         tsleep(&up->sleep, return0, 0, 150);
1212                         phy = phyread(c, phyno, Phystat);
1213                         if(phy == ~0)
1214                                 continue;
1215                         if(phy & Ans){
1216                                 r = phyread(c, phyno, Phyctl);
1217                                 if(r == ~0)
1218                                         continue;
1219                                 phywrite(c, phyno, Phyctl, r | Ran | Ean);
1220                         }
1221                         break;
1222                 }
1223                 i = (phy>>8) & 3;
1224                 e->link = i != 3 && (phy & Link) != 0;
1225                 if(e->link == 0)
1226                         i = 3;
1227                 c->speeds[i]++;
1228                 e->mbps = speedtab[i];
1229                 lsleep(c, Lsc);
1230         }
1231 }
1232
1233 static void
1234 phylproc(void *v)
1235 {
1236         uint a, i, phy, phyno;
1237         Ctlr *c;
1238         Ether *e;
1239
1240         e = v;
1241         c = e->ctlr;
1242         while(waserror())
1243                 ;
1244
1245         while((phyno = phyprobe(c, 3<<1)) == ~0)
1246                 lsleep(c, Lsc);
1247
1248         if(c->type == i82573 && (phy = phyread(c, phyno, Phyier)) != ~0)
1249                 phywrite(c, phyno, Phyier, phy | Lscie | Ancie | Spdie | Panie);
1250
1251         for(;;){
1252                 phy = phyread(c, phyno, Physsr);
1253                 if(phy == ~0){
1254                         phy = 0;
1255                         i = 3;
1256                         goto next;
1257                 }
1258                 i = (phy>>14) & 3;
1259                 switch(c->type){
1260                 default:
1261                         a = 0;
1262                         break;
1263                 case i82563:
1264                 case i82578:
1265                 case i82578m:
1266                 case i82583:
1267                         a = phyread(c, phyno, Phyisr) & Ane;
1268                         break;
1269                 case i82571:
1270                 case i82572:
1271                 case i82575:
1272                 case i82576:
1273                         a = phyread(c, phyno, Phylhr) & Anf;
1274                         i = (i-1) & 3;
1275                         break;
1276                 }
1277                 if(a)
1278                         phywrite(c, phyno, Phyctl, phyread(c, phyno, Phyctl) | Ran | Ean);
1279 next:
1280                 e->link = (phy & Rtlink) != 0;
1281                 if(e->link == 0)
1282                         i = 3;
1283                 c->speeds[i]++;
1284                 e->mbps = speedtab[i];
1285                 if(c->type == i82563)
1286                         phyerrata(e, c, phyno);
1287                 lsleep(c, Lsc);
1288         }
1289 }
1290
1291 static void
1292 pcslproc(void *v)
1293 {
1294         uint i, phy;
1295         Ctlr *c;
1296         Ether *e;
1297
1298         e = v;
1299         c = e->ctlr;
1300         while(waserror())
1301                 ;
1302
1303         if(c->type == i82575 || c->type == i82576)
1304                 csr32w(c, Connsw, Enrgirq);
1305         for(;;){
1306                 phy = csr32r(c, Pcsstat);
1307                 e->link = phy & Linkok;
1308                 i = 3;
1309                 if(e->link)
1310                         i = (phy & 6) >> 1;
1311                 else if(phy & Anbad)
1312                         csr32w(c, Pcsctl, csr32r(c, Pcsctl) | Pan | Prestart);
1313                 c->speeds[i]++;
1314                 e->mbps = speedtab[i];
1315                 lsleep(c, Lsc | Omed);
1316         }
1317 }
1318
1319 static void
1320 serdeslproc(void *v)
1321 {
1322         uint i, tx, rx;
1323         Ctlr *c;
1324         Ether *e;
1325
1326         e = v;
1327         c = e->ctlr;
1328         while(waserror())
1329                 ;
1330         for(;;){
1331                 rx = csr32r(c, Rxcw);
1332                 tx = csr32r(c, Txcw);
1333                 USED(tx);
1334                 e->link = (rx & 1<<31) != 0;
1335 //              e->link = (csr32r(c, Status) & Lu) != 0;
1336                 i = 3;
1337                 if(e->link)
1338                         i = 2;
1339                 c->speeds[i]++;
1340                 e->mbps = speedtab[i];
1341                 lsleep(c, Lsc);
1342         }
1343 }
1344
1345 static void
1346 i82563attach(Ether *edev)
1347 {
1348         char name[KNAMELEN];
1349         Ctlr *ctlr;
1350
1351         ctlr = edev->ctlr;
1352         qlock(&ctlr->alock);
1353         if(ctlr->alloc != nil){
1354                 qunlock(&ctlr->alock);
1355                 return;
1356         }
1357
1358         ctlr->nrd = Nrd;
1359         ctlr->ntd = Ntd;
1360         ctlr->alloc = malloc(ctlr->nrd*sizeof(Rd)+ctlr->ntd*sizeof(Td) + 255);
1361         ctlr->rb = malloc(ctlr->nrd * sizeof(Block*));
1362         ctlr->tb = malloc(ctlr->ntd * sizeof(Block*));
1363         if(ctlr->alloc == nil || ctlr->rb == nil || ctlr->tb == nil){
1364                 free(ctlr->rb);
1365                 ctlr->rb = nil;
1366                 free(ctlr->tb);
1367                 ctlr->tb = nil;
1368                 free(ctlr->alloc);
1369                 ctlr->alloc = nil;
1370                 qunlock(&ctlr->alock);
1371                 error(Enomem);
1372         }
1373         ctlr->rdba = (Rd*)ROUNDUP((uintptr)ctlr->alloc, 256);
1374         ctlr->tdba = (Td*)(ctlr->rdba + ctlr->nrd);
1375
1376         if(waserror()){
1377                 free(ctlr->tb);
1378                 ctlr->tb = nil;
1379                 free(ctlr->rb);
1380                 ctlr->rb = nil;
1381                 free(ctlr->alloc);
1382                 ctlr->alloc = nil;
1383                 qunlock(&ctlr->alock);
1384                 nexterror();
1385         }
1386
1387         snprint(name, sizeof name, "#l%dl", edev->ctlrno);
1388         if(csr32r(ctlr, Status) & Tbimode)
1389                 kproc(name, serdeslproc, edev);         /* mac based serdes */
1390         else if((csr32r(ctlr, Ctrlext) & Linkmode) == Serdes)
1391                 kproc(name, pcslproc, edev);            /* phy based serdes */
1392         else if(cttab[ctlr->type].flag & F79phy)
1393                 kproc(name, phyl79proc, edev);
1394         else
1395                 kproc(name, phylproc, edev);
1396
1397         snprint(name, sizeof name, "#l%dr", edev->ctlrno);
1398         kproc(name, i82563rproc, edev);
1399
1400         snprint(name, sizeof name, "#l%dt", edev->ctlrno);
1401         kproc(name, i82563tproc, edev);
1402
1403         qunlock(&ctlr->alock);
1404         poperror();
1405 }
1406
1407 static void
1408 i82563interrupt(Ureg*, void *arg)
1409 {
1410         Ctlr *ctlr;
1411         Ether *edev;
1412         int icr, im;
1413
1414         edev = arg;
1415         ctlr = edev->ctlr;
1416
1417         ilock(&ctlr->imlock);
1418         csr32w(ctlr, Imc, ~0);
1419         im = ctlr->im;
1420
1421         while(icr = csr32r(ctlr, Icr) & ctlr->im){
1422                 if(icr & (Lsc | Omed)){
1423                         im &= ~(Lsc | Omed);
1424                         ctlr->lim = icr & (Lsc | Omed);
1425                         wakeup(&ctlr->lrendez);
1426                         ctlr->lintr++;
1427                 }
1428                 if(icr & (Rxt0|Rxo|Rxdmt0|Rxseq|Ack)){
1429                         ctlr->rim = icr & (Rxt0|Rxo|Rxdmt0|Rxseq|Ack);
1430                         im &= ~(Rxt0|Rxo|Rxdmt0|Rxseq|Ack);
1431                         wakeup(&ctlr->rrendez);
1432                         ctlr->rintr++;
1433                 }
1434                 if(icr & Txdw){
1435                         im &= ~Txdw;
1436                         ctlr->tintr++;
1437                         wakeup(&ctlr->trendez);
1438                 }
1439         }
1440
1441         ctlr->im = im;
1442         csr32w(ctlr, Ims, im);
1443         iunlock(&ctlr->imlock);
1444 }
1445
1446 static int
1447 i82563detach(Ctlr *ctlr)
1448 {
1449         int r, timeo;
1450
1451         /* balance rx/tx packet buffer; survives reset */
1452         if(ctlr->rbsz > 8192 && cttab[ctlr->type].flag & Fpba){
1453                 ctlr->pba = csr32r(ctlr, Pba);
1454                 r = ctlr->pba >> 16;
1455                 r += ctlr->pba & 0xffff;
1456                 r >>= 1;
1457                 csr32w(ctlr, Pba, r);
1458         }else if(ctlr->type == i82573 && ctlr->rbsz > 1514)
1459                 csr32w(ctlr, Pba, 14);
1460         ctlr->pba = csr32r(ctlr, Pba);
1461
1462         /*
1463          * Perform a device reset to get the chip back to the
1464          * power-on state, followed by an EEPROM reset to read
1465          * the defaults for some internal registers.
1466          */
1467         csr32w(ctlr, Imc, ~0);
1468         csr32w(ctlr, Rctl, 0);
1469         csr32w(ctlr, Tctl, csr32r(ctlr, Tctl) & ~Ten);
1470
1471         delay(10);
1472
1473         r = csr32r(ctlr, Ctrl);
1474         if(ctlr->type == i82566 || ctlr->type == i82579)
1475                 r |= Phyrst;
1476         /*
1477          * hack: 82579LM on lenovo X230 is stuck at 10mbps after
1478          * reseting the phy, but works fine if we dont reset.
1479          */
1480         if(ctlr->pcidev->did == 0x1502)
1481                 r &= ~Phyrst;
1482         csr32w(ctlr, Ctrl, Devrst | r);
1483         delay(1);
1484         for(timeo = 0;; timeo++){
1485                 if((csr32r(ctlr, Ctrl) & (Devrst|Phyrst)) == 0)
1486                         break;
1487                 if(timeo >= 1000)
1488                         return -1;
1489                 delay(1);
1490         }
1491
1492         r = csr32r(ctlr, Ctrl);
1493         csr32w(ctlr, Ctrl, Slu|r);
1494
1495         r = csr32r(ctlr, Ctrlext);
1496         csr32w(ctlr, Ctrlext, r|Eerst);
1497         delay(1);
1498         for(timeo = 0; timeo < 1000; timeo++){
1499                 if(!(csr32r(ctlr, Ctrlext) & Eerst))
1500                         break;
1501                 delay(1);
1502         }
1503         if(csr32r(ctlr, Ctrlext) & Eerst)
1504                 return -1;
1505
1506         csr32w(ctlr, Imc, ~0);
1507         delay(1);
1508         for(timeo = 0; timeo < 1000; timeo++){
1509                 if((csr32r(ctlr, Icr) & ~Rxcfg) == 0)
1510                         break;
1511                 delay(1);
1512         }
1513         if(csr32r(ctlr, Icr) & ~Rxcfg)
1514                 return -1;
1515
1516         return 0;
1517 }
1518
1519 static void
1520 i82563shutdown(Ether *edev)
1521 {
1522         i82563detach(edev->ctlr);
1523 }
1524
1525 static int
1526 eeread(Ctlr *ctlr, int adr)
1527 {
1528         int timeout;
1529
1530         csr32w(ctlr, Eerd, EEstart | adr << 2);
1531         timeout = 1000;
1532         while ((csr32r(ctlr, Eerd) & EEdone) == 0 && timeout--)
1533                 microdelay(5);
1534         if (timeout < 0) {
1535                 print("%s: eeread timeout\n", cname(ctlr));
1536                 return -1;
1537         }
1538         return (csr32r(ctlr, Eerd) >> 16) & 0xffff;
1539 }
1540
1541 static int
1542 eeload(Ctlr *ctlr)
1543 {
1544         u16int sum;
1545         int data, adr;
1546
1547         sum = 0;
1548         for (adr = 0; adr < 0x40; adr++) {
1549                 data = eeread(ctlr, adr);
1550                 if(data == -1) return -1;
1551                 ctlr->eeprom[adr] = data;
1552                 sum += data;
1553         }
1554         return sum;
1555 }
1556
1557 static int
1558 fread16(Ctlr *c, Flash *f, int ladr)
1559 {
1560         u16int s;
1561         int timeout;
1562
1563         delay(1);
1564         s = f->reg[Fsts];
1565         if((s&Fvalid) == 0)
1566                 return -1;
1567         f->reg[Fsts] |= Fcerr | Ael;
1568         for(timeout = 0; timeout < 10; timeout++){
1569                 if((s&Scip) == 0)
1570                         goto done;
1571                 delay(1);
1572                 s = f->reg[Fsts];
1573         }
1574         return -1;
1575 done:
1576         f->reg[Fsts] |= Fdone;
1577         f->reg32[Faddr] = ladr;
1578
1579         /* setup flash control register */
1580         s = f->reg[Fctl] & ~0x3ff;
1581         f->reg[Fctl] = s | 1<<8 | Fgo;  /* 2 byte read */
1582         timeout = 1000;
1583         while((f->reg[Fsts] & Fdone) == 0 && timeout--)
1584                 microdelay(5);
1585         if(timeout < 0){
1586                 print("%s: fread timeout\n", cname(c));
1587                 return -1;
1588         }
1589         if(f->reg[Fsts] & (Fcerr|Ael))
1590                 return -1;
1591         return f->reg32[Fdata] & 0xffff;
1592 }
1593
1594 static int
1595 fread32(Ctlr *c, Flash *f, int ladr, u32int *data)
1596 {
1597         u32int s;
1598         int timeout;
1599
1600         delay(1);
1601         s = f->reg32[Fsts/2];
1602         if((s&Fvalid) == 0)
1603                 return -1;
1604         f->reg32[Fsts/2] |= Fcerr | Ael;
1605         for(timeout = 0; timeout < 10; timeout++){
1606                 if((s&Scip) == 0)
1607                         goto done;
1608                 delay(1);
1609                 s = f->reg32[Fsts/2];
1610         }
1611         return -1;
1612 done:
1613         f->reg32[Fsts/2] |= Fdone;
1614         f->reg32[Faddr] = ladr;
1615
1616         /* setup flash control register */
1617         s = (f->reg32[Fctl/2] >> 16) & ~0x3ff;
1618         f->reg32[Fctl/2] = (s | 3<<8 | Fgo) << 16;      /* 4 byte read */
1619         timeout = 1000;
1620         while((f->reg32[Fsts/2] & Fdone) == 0 && timeout--)
1621                 microdelay(5);
1622         if(timeout < 0){
1623                 print("%s: fread timeout\n", cname(c));
1624                 return -1;
1625         }
1626         if(f->reg32[Fsts/2] & (Fcerr|Ael))
1627                 return -1;
1628         *data = f->reg32[Fdata];
1629         return 0;
1630 }
1631
1632 static int
1633 fload32(Ctlr *c)
1634 {
1635         int r, adr;
1636         u16int sum;
1637         u32int w;
1638         Flash f;
1639
1640         f.reg32 = &c->nic[0xe000/4];
1641         f.reg = nil;
1642         f.base = 0;
1643         f.lim = (((csr32r(c, 0xC) >> 1) & 0x1F) + 1) << 12;
1644         r = f.lim >> 1;
1645         if(fread32(c, &f, r + 0x24, &w) == -1  || (w & 0xC000) != 0x8000)
1646                 r = 0;
1647         sum = 0;
1648         for(adr = 0; adr < 0x20; adr++) {
1649                 if(fread32(c, &f, r + adr*4, &w) == -1)
1650                         return -1;
1651                 c->eeprom[adr*2+0] = w;
1652                 c->eeprom[adr*2+1] = w>>16;
1653                 sum += w & 0xFFFF;
1654                 sum += w >> 16;
1655         }
1656         return sum;
1657 }
1658
1659 static int
1660 fload(Ctlr *c)
1661 {
1662         int data, r, adr;
1663         u16int sum;
1664         void *va;
1665         Flash f;
1666
1667         memset(c->eeprom, 0xFF, sizeof(c->eeprom));
1668         if(c->pcidev->mem[1].bar == 0)
1669                 return fload32(c);      /* i219 */
1670
1671         if(c->pcidev->mem[1].bar & 1)
1672                 return -1;
1673
1674         va = vmap(c->pcidev->mem[1].bar & ~0xF, c->pcidev->mem[1].size);
1675         if(va == nil)
1676                 return -1;
1677         f.reg = va;
1678         f.reg32 = va;
1679         f.base = f.reg32[Bfpr] & 0x1fff;
1680         f.lim = f.reg32[Bfpr]>>16 & 0x1fff;
1681         if(csr32r(c, Eec) & Sec1val)
1682                 f.base += f.lim+1 - f.base >> 1;
1683         r = f.base << 12;
1684         sum = 0;
1685         for(adr = 0; adr < 0x40; adr++) {
1686                 data = fread16(c, &f, r + adr*2);
1687                 if(data == -1)
1688                         goto out;
1689                 c->eeprom[adr] = data;
1690                 sum += data;
1691         }
1692 out:
1693         vunmap(va, c->pcidev->mem[1].size);
1694         return sum;
1695 }
1696
1697 static int
1698 invmload(Ctlr *c)
1699 {
1700         int i, a;
1701         u32int w;
1702
1703         memset(c->eeprom, 0xFF, sizeof(c->eeprom));
1704         for(i=0; i<64; i++){
1705                 w = csr32r(c, Invmdata0 + i*4);
1706                 switch(w & 7){
1707                 case 0: // uninitialized structure
1708                         break;
1709                 case 1: // word auto load
1710                         a = (w & 0xFE00) >> 9;
1711                         if(a < nelem(c->eeprom))
1712                                 c->eeprom[a] = w >> 16;
1713                         continue;
1714                 case 2: // csr auto load
1715                         i++;
1716                 case 3: // phy auto load
1717                         continue;
1718                 case 4: // rsa key sha256
1719                         i += 256/32;
1720                 case 5: // invalidated structure
1721                         continue;
1722                 default:
1723                         print("invm: %.2x %.8ux\n", i, w);
1724                         continue;
1725                 }
1726                 break;
1727         }
1728         return 0;
1729 }
1730
1731 static void
1732 defaultea(Ctlr *ctlr, uchar *ra)
1733 {
1734         uint i, r;
1735         uvlong u;
1736         static uchar nilea[Eaddrlen];
1737
1738         if(memcmp(ra, nilea, Eaddrlen) != 0)
1739                 return;
1740         if(cttab[ctlr->type].flag & Fflashea){
1741                 /* intel mb bug */
1742                 u = (uvlong)csr32r(ctlr, Rah)<<32u | (ulong)csr32r(ctlr, Ral);
1743                 for(i = 0; i < Eaddrlen; i++)
1744                         ra[i] = u >> 8*i;
1745         }
1746         if(memcmp(ra, nilea, Eaddrlen) != 0)
1747                 return;
1748         for(i = 0; i < Eaddrlen/2; i++){
1749                 ra[2*i] = ctlr->eeprom[Ea+i];
1750                 ra[2*i+1] = ctlr->eeprom[Ea+i] >> 8;
1751         }
1752         r = (csr32r(ctlr, Status) & Lanid) >> 2;
1753         ra[5] += r;                             /* ea ctlr[n] = ea ctlr[0]+n */
1754 }
1755
1756 static int
1757 i82563reset(Ctlr *ctlr)
1758 {
1759         uchar *ra;
1760         int i, r, flag;
1761
1762         if(i82563detach(ctlr))
1763                 return -1;
1764         flag = cttab[ctlr->type].flag;
1765
1766         if(ctlr->type == i210 && (csr32r(ctlr, Eec) & Flupd) == 0)
1767                 r = invmload(ctlr);
1768         else if(flag & Fload)
1769                 r = fload(ctlr);
1770         else
1771                 r = eeload(ctlr);
1772
1773         if(r != 0 && r != 0xbaba){
1774                 print("%s: bad eeprom checksum - %#.4ux", cname(ctlr), r);
1775                 if(flag & Fbadcsum)
1776                         print("; ignored\n");
1777                 else {
1778                         print("\n");
1779                         return -1;
1780                 }
1781         }
1782
1783         ra = ctlr->ra;
1784         defaultea(ctlr, ra);
1785         csr32w(ctlr, Ral, ra[3]<<24 | ra[2]<<16 | ra[1]<<8 | ra[0]);
1786         csr32w(ctlr, Rah, 1<<31 | ra[5]<<8 | ra[4]);
1787         for(i = 1; i < 16; i++){
1788                 csr32w(ctlr, Ral+i*8, 0);
1789                 csr32w(ctlr, Rah+i*8, 0);
1790         }
1791         memset(ctlr->mta, 0, sizeof(ctlr->mta));
1792         for(i = 0; i < 128; i++)
1793                 csr32w(ctlr, Mta + i*4, 0);
1794         if((flag & Fnofca) == 0){
1795                 csr32w(ctlr, Fcal, 0x00C28001);
1796                 csr32w(ctlr, Fcah, 0x0100);
1797         }
1798         if((flag & Fnofct) == 0)
1799                 csr32w(ctlr, Fct, 0x8808);
1800         csr32w(ctlr, Fcttv, 0x0100);
1801         csr32w(ctlr, Fcrtl, ctlr->fcrtl);
1802         csr32w(ctlr, Fcrth, ctlr->fcrth);
1803         if(flag & F75)
1804                 csr32w(ctlr, Eitr, 128<<2);             /* 128 ¼ microsecond intervals */
1805         return 0;
1806 }
1807
1808 enum {
1809         CMrdtr,
1810         CMradv,
1811         CMpause,
1812         CMan,
1813 };
1814
1815 static Cmdtab i82563ctlmsg[] = {
1816         CMrdtr, "rdtr", 2,
1817         CMradv, "radv", 2,
1818         CMpause, "pause", 1,
1819         CMan,   "an",   1,
1820 };
1821
1822 static long
1823 i82563ctl(Ether *edev, void *buf, long n)
1824 {
1825         char *p;
1826         ulong v;
1827         Ctlr *ctlr;
1828         Cmdbuf *cb;
1829         Cmdtab *ct;
1830
1831         if((ctlr = edev->ctlr) == nil)
1832                 error(Enonexist);
1833
1834         cb = parsecmd(buf, n);
1835         if(waserror()){
1836                 free(cb);
1837                 nexterror();
1838         }
1839
1840         ct = lookupcmd(cb, i82563ctlmsg, nelem(i82563ctlmsg));
1841         switch(ct->index){
1842         case CMrdtr:
1843                 v = strtoul(cb->f[1], &p, 0);
1844                 if(*p || v > 0xffff)
1845                         error(Ebadarg);
1846                 ctlr->rdtr = v;
1847                 csr32w(ctlr, Rdtr, v);
1848                 break;
1849         case CMradv:
1850                 v = strtoul(cb->f[1], &p, 0);
1851                 if(*p || v > 0xffff)
1852                         error(Ebadarg);
1853                 ctlr->radv = v;
1854                 csr32w(ctlr, Radv, v);
1855                 break;
1856         case CMpause:
1857                 csr32w(ctlr, Ctrl, csr32r(ctlr, Ctrl) ^ (Rfce | Tfce));
1858                 break;
1859         case CMan:
1860                 csr32w(ctlr, Ctrl, csr32r(ctlr, Ctrl) | Lrst | Phyrst);
1861                 break;
1862         }
1863         free(cb);
1864         poperror();
1865
1866         return n;
1867 }
1868
1869 static int
1870 didtype(int d)
1871 {
1872         /*
1873          * Some names and did values are from
1874          * OpenBSD's em(4) Intel driver.
1875          */
1876         switch(d){
1877         case 0x1096:            /* copper */
1878         case 0x10ba:            /* copper “gilgal” */
1879         case 0x1098:            /* serdes; not seen */
1880         case 0x10bb:            /* serdes */
1881                 return i82563;
1882         case 0x1049:            /* ich8; mm */
1883         case 0x104a:            /* ich8; dm */
1884         case 0x104b:            /* ich8; dc */
1885         case 0x104d:            /* ich8; v “ninevah” */
1886         case 0x10bd:            /* ich9; dm-2 */
1887         case 0x294c:            /* ich9 */
1888         case 0x104c:            /* ich8; untested */
1889         case 0x10c4:            /* ich8; untested */
1890         case 0x10c5:            /* ich8; untested */
1891                 return i82566;
1892         case 0x10de:            /* lm ich10d */
1893         case 0x10df:            /* lf ich10 */
1894         case 0x10e5:            /* lm ich9 */
1895         case 0x10f5:            /* lm ich9m; “boazman” */
1896         case 0x10ce:            /* v ich10 */
1897         case 0x10c0:            /* ich9 */
1898         case 0x10c2:            /* ich9; untested */
1899         case 0x10c3:            /* ich9; untested */
1900         case 0x1501:            /* ich8; untested */
1901                 return i82567;
1902         case 0x10bf:            /* lf ich9m */
1903         case 0x10cb:            /* v ich9m */
1904         case 0x10cd:            /* lf ich10 */
1905         case 0x10cc:            /* lm ich10 */
1906                 return i82567m;
1907         case 0x105e:            /* eb copper */
1908         case 0x105f:            /* eb fiber */
1909         case 0x1060:            /* eb serdes */
1910         case 0x10a4:            /* eb copper */
1911         case 0x10a5:            /* eb fiber */
1912         case 0x10bc:            /* eb copper */
1913         case 0x10d9:            /* eb serdes */
1914         case 0x10da:            /* eb serdes “ophir” */
1915         case 0x10a0:            /* eb; untested */
1916         case 0x10a1:            /* eb; untested */
1917         case 0x10d5:            /* copper; untested */
1918                 return i82571;
1919         case 0x107d:            /* ei copper */
1920         case 0x107e:            /* ei fiber */
1921         case 0x107f:            /* ei serdes */
1922         case 0x10b9:            /* ei “rimon” */
1923                 return i82572;
1924         case 0x108b:            /* e “vidalia” */
1925         case 0x108c:            /* e (iamt) */
1926         case 0x109a:            /* l “tekoa” */
1927         case 0x10b0:            /* l; untested */
1928         case 0x10b2:            /* v; untested */
1929         case 0x10b3:            /* e; untested */
1930         case 0x10b4:            /* l; untested */
1931                 return i82573;
1932         case 0x10d3:            /* l or it; “hartwell” */
1933         case 0x10f6:            /* la; untested */
1934                 return i82574;
1935         case 0x10a7:            /* eb */
1936         case 0x10a9:            /* eb fiber/serdes */
1937         case 0x10d6:            /* untested */
1938         case 0x10e2:            /* untested */
1939                 return i82575;
1940         case 0x10c9:            /* copper */
1941         case 0x10e6:            /* fiber */
1942         case 0x10e7:            /* serdes; “kawela” */
1943         case 0x10e8:            /* copper; untested */
1944         case 0x150a:            /* untested */
1945         case 0x150d:            /* serdes backplane */
1946         case 0x1518:            /* serdes; untested */
1947         case 0x1526:            /* untested */
1948                 return i82576;
1949         case 0x10ea:            /* lc “calpella”; aka pch lan */
1950                 return i82577;
1951         case 0x10eb:            /* lm “calpella” */
1952                 return i82577m;
1953         case 0x10ef:            /* dc “piketon” */
1954                 return i82578;
1955         case 0x1502:            /* lm */
1956         case 0x1503:            /* v “lewisville” */
1957                 return i82579;
1958         case 0x10f0:            /* dm “king's creek” */
1959                 return i82578m;
1960         case 0x150e:            /* copper “barton hills” */
1961         case 0x150f:            /* fiber */
1962         case 0x1510:            /* serdes backplane */
1963         case 0x1511:            /* sgmii sfp */
1964         case 0x1516:            /* copper */
1965                 return i82580;
1966         case 0x1506:            /* v */
1967         case 0x150c:            /* untested */
1968                 return i82583;
1969         case 0x1533:            /* i210-t1 */
1970         case 0x1534:            /* i210 */
1971         case 0x1536:            /* i210-fiber */
1972         case 0x1537:            /* i210-backplane */
1973         case 0x1538:            /* i210 sgmii */
1974         case 0x1539:            /* i211 copper */
1975         case 0x157b:            /* i210 copper flashless */
1976         case 0x157c:            /* i210 serdes flashless */
1977                 return i210;
1978         case 0x153a:            /* i217-lm */
1979         case 0x153b:            /* i217-v */
1980                 return i217;
1981         case 0x1559:            /* i218-v */
1982         case 0x155a:            /* i218-lm */
1983         case 0x15a0:            /* i218-lm */
1984         case 0x15a1:            /* i218-v */
1985         case 0x15a2:            /* i218-lm */
1986         case 0x15a3:            /* i218-v */
1987                 return i218;
1988         case 0x156f:            /* i219-lm */
1989         case 0x15b7:            /* i219-lm */
1990         case 0x1570:            /* i219-v */
1991         case 0x15b8:            /* i219-v */
1992         case 0x15b9:            /* i219-lm */
1993         case 0x15bb:            /* i219-lm */
1994         case 0x15d6:            /* i219-v */
1995         case 0x15d7:            /* i219-lm */
1996         case 0x15d8:            /* i219-v */
1997         case 0x15e3:            /* i219-lm */
1998         case 0x0d4c:            /* i219-lm */
1999                 return i219;
2000         case 0x151f:            /* i350 “powerville” eeprom-less */
2001         case 0x1521:            /* i350 copper */
2002         case 0x1522:            /* i350 fiber */
2003         case 0x1523:            /* i350 serdes */
2004         case 0x1524:            /* i350 sgmii */
2005         case 0x1546:            /* i350 DA4 (untested) */
2006         case 0x1f40:            /* i354 backplane */
2007         case 0x1f41:            /* i354 sgmii */
2008         case 0x1f42:            /* i354 sgmii (c2000) */
2009         case 0x1f45:            /* i354 backplane 2.5 */
2010                 return i350;
2011         }
2012         return -1;
2013 }
2014
2015 static void
2016 hbafixup(Pcidev *p)
2017 {
2018         uint i;
2019
2020         i = pcicfgr32(p, PciSVID);
2021         if((i & 0xffff) == 0x1b52 && p->did == 1)
2022                 p->did = i>>16;
2023 }
2024
2025 static void
2026 i82563pci(void)
2027 {
2028         int type;
2029         Ctlr *ctlr;
2030         Pcidev *p;
2031
2032         for(p = nil; p = pcimatch(p, 0x8086, 0);){
2033                 hbafixup(p);
2034                 if(p->mem[0].bar & 1)
2035                         continue;
2036                 if((type = didtype(p->did)) == -1)
2037                         continue;
2038                 ctlr = malloc(sizeof(Ctlr));
2039                 if(ctlr == nil){
2040                         print("%s: can't allocate memory\n", cttab[type].name);
2041                         continue;
2042                 }
2043                 ctlr->type = type;
2044                 ctlr->pcidev = p;
2045                 ctlr->rbsz = ROUND(cttab[type].mtu, 1024);
2046                 ctlr->port = p->mem[0].bar & ~0xF;
2047                 if(i82563ctlrhead != nil)
2048                         i82563ctlrtail->next = ctlr;
2049                 else
2050                         i82563ctlrhead = ctlr;
2051                 i82563ctlrtail = ctlr;
2052         }
2053 }
2054
2055 static int
2056 setup(Ctlr *ctlr)
2057 {
2058         Pcidev *p;
2059
2060         p = ctlr->pcidev;
2061         ctlr->nic = vmap(ctlr->port, p->mem[0].size);
2062         if(ctlr->nic == nil){
2063                 print("%s: can't map %llux\n", cname(ctlr), ctlr->port);
2064                 return -1;
2065         }
2066         pcienable(p);
2067         if(i82563reset(ctlr)){
2068                 pcidisable(p);
2069                 vunmap(ctlr->nic, p->mem[0].size);
2070                 return -1;
2071         }
2072         pcisetbme(p);
2073         return 0;
2074 }
2075
2076 static int
2077 pnp(Ether *edev, int type)
2078 {
2079         Ctlr *ctlr;
2080         static int done;
2081
2082         if(!done) {
2083                 i82563pci();
2084                 done = 1;
2085         }
2086
2087         /*
2088          * Any adapter matches if no edev->port is supplied,
2089          * otherwise the ports must match.
2090          */
2091         for(ctlr = i82563ctlrhead; ; ctlr = ctlr->next){
2092                 if(ctlr == nil)
2093                         return -1;
2094                 if(ctlr->active)
2095                         continue;
2096                 if(type != -1 && ctlr->type != type)
2097                         continue;
2098                 if(edev->port == 0 || edev->port == ctlr->port){
2099                         ctlr->active = 1;
2100                         memmove(ctlr->ra, edev->ea, Eaddrlen);
2101                         if(setup(ctlr) == 0)
2102                                 break;
2103                 }
2104         }
2105
2106         edev->ctlr = ctlr;
2107         edev->port = ctlr->port;
2108         edev->irq = ctlr->pcidev->intl;
2109         edev->tbdf = ctlr->pcidev->tbdf;
2110         edev->mbps = 1000;
2111         edev->maxmtu = cttab[ctlr->type].mtu;
2112         memmove(edev->ea, ctlr->ra, Eaddrlen);
2113
2114         /*
2115          * Linkage to the generic ethernet driver.
2116          */
2117         edev->attach = i82563attach;
2118 //      edev->transmit = i82563transmit;
2119         edev->ifstat = i82563ifstat;
2120         edev->ctl = i82563ctl;
2121
2122         edev->arg = edev;
2123         edev->promiscuous = i82563promiscuous;
2124         edev->shutdown = i82563shutdown;
2125         edev->multicast = i82563multicast;
2126
2127         intrenable(edev->irq, i82563interrupt, edev, edev->tbdf, edev->name);
2128
2129         return 0;
2130 }
2131
2132 static int
2133 anypnp(Ether *e)
2134 {
2135         return pnp(e, -1);
2136 }
2137
2138 static int
2139 i82563pnp(Ether *e)
2140 {
2141         return pnp(e, i82563);
2142 }
2143
2144 static int
2145 i82566pnp(Ether *e)
2146 {
2147         return pnp(e, i82566);
2148 }
2149
2150 static int
2151 i82567pnp(Ether *e)
2152 {
2153         return pnp(e, i82567m) & pnp(e, i82567);
2154 }
2155
2156 static int
2157 i82571pnp(Ether *e)
2158 {
2159         return pnp(e, i82571);
2160 }
2161
2162 static int
2163 i82572pnp(Ether *e)
2164 {
2165         return pnp(e, i82572);
2166 }
2167
2168 static int
2169 i82573pnp(Ether *e)
2170 {
2171         return pnp(e, i82573);
2172 }
2173
2174 static int
2175 i82574pnp(Ether *e)
2176 {
2177         return pnp(e, i82574);
2178 }
2179
2180 static int
2181 i82575pnp(Ether *e)
2182 {
2183         return pnp(e, i82575);
2184 }
2185
2186 static int
2187 i82576pnp(Ether *e)
2188 {
2189         return pnp(e, i82576);
2190 }
2191
2192 static int
2193 i82577pnp(Ether *e)
2194 {
2195         return pnp(e, i82577m) & pnp(e, i82577);
2196 }
2197
2198 static int
2199 i82578pnp(Ether *e)
2200 {
2201         return pnp(e, i82578m) & pnp(e, i82578);
2202 }
2203
2204 static int
2205 i82579pnp(Ether *e)
2206 {
2207         return pnp(e, i82579);
2208 }
2209
2210 static int
2211 i82580pnp(Ether *e)
2212 {
2213         return pnp(e, i82580);
2214 }
2215
2216 static int
2217 i82583pnp(Ether *e)
2218 {
2219         return pnp(e, i82583);
2220 }
2221
2222 static int
2223 i210pnp(Ether *e)
2224 {
2225         return pnp(e, i210);
2226 }
2227
2228 static int
2229 i217pnp(Ether *e)
2230 {
2231         return pnp(e, i217);
2232 }
2233
2234 static int
2235 i218pnp(Ether *e)
2236 {
2237         return pnp(e, i218);
2238 }
2239
2240 static int
2241 i219pnp(Ether *e)
2242 {
2243         return pnp(e, i219);
2244 }
2245
2246 static int
2247 i350pnp(Ether *e)
2248 {
2249         return pnp(e, i350);
2250 }
2251
2252 void
2253 ether82563link(void)
2254 {
2255         /*
2256          * recognise lots of model numbers for debugging
2257          * also good for forcing onboard nic(s) as ether0
2258          * try to make that unnecessary by listing lom first.
2259          */
2260         addethercard("i82563", i82563pnp);
2261         addethercard("i82566", i82566pnp);
2262         addethercard("i82574", i82574pnp);
2263         addethercard("i82576", i82576pnp);
2264         addethercard("i82567", i82567pnp);
2265         addethercard("i82573", i82573pnp);
2266
2267         addethercard("i82571", i82571pnp);
2268         addethercard("i82572", i82572pnp);
2269         addethercard("i82575", i82575pnp);
2270         addethercard("i82577", i82577pnp);
2271         addethercard("i82578", i82578pnp);
2272         addethercard("i82579", i82579pnp);
2273         addethercard("i82580", i82580pnp);
2274         addethercard("i82583", i82583pnp);
2275         addethercard("i210", i210pnp);
2276         addethercard("i217", i217pnp);
2277         addethercard("i218", i218pnp);
2278         addethercard("i219", i219pnp);
2279         addethercard("i350", i350pnp);
2280         addethercard("igbepcie", anypnp);
2281 }