sys/src/9/pc/ether82563.c

   1 /*
   2  * Intel 8256[367], 8257[1-9], 8258[03]
   3  *      Gigabit Ethernet PCI-Express Controllers
   4  * Coraid EtherDrive® hba
   5  */
   6 #include "u.h"
   7 #include "../port/lib.h"
   8 #include "mem.h"
   9 #include "dat.h"
  10 #include "fns.h"
  11 #include "io.h"
  12 #include "../port/error.h"
  13 #include "../port/netif.h"
  14
  15 #include "etherif.h"
  16
  17 /*
  18  * note: the 82575, 82576 and 82580 are operated using registers aliased
  19  * to the 82563-style architecture.  many features seen in the 82598
  20  * are also seen in the 82575 part.
  21  */
  22
  23 enum {
  24         /* General */
  25
  26         Ctrl            = 0x0000,       /* Device Control */
  27         Status          = 0x0008,       /* Device Status */
  28         Eec             = 0x0010,       /* EEPROM/Flash Control/Data */
  29         Eerd            = 0x0014,       /* EEPROM Read */
  30         Ctrlext         = 0x0018,       /* Extended Device Control */
  31         Fla             = 0x001c,       /* Flash Access */
  32         Mdic            = 0x0020,       /* MDI Control */
  33         Fcal            = 0x0028,       /* Flow Control Address Low */
  34         Fcah            = 0x002C,       /* Flow Control Address High */
  35         Fct             = 0x0030,       /* Flow Control Type */
  36         Kumctrlsta      = 0x0034,       /* Kumeran Control and Status Register */
  37         Vet             = 0x0038,       /* VLAN EtherType */
  38         Fcttv           = 0x0170,       /* Flow Control Transmit Timer Value */
  39         Txcw            = 0x0178,       /* Transmit Configuration Word */
  40         Rxcw            = 0x0180,       /* Receive Configuration Word */
  41         Ledctl          = 0x0E00,       /* LED control */
  42         Pba             = 0x1000,       /* Packet Buffer Allocation */
  43         Pbs             = 0x1008,       /* Packet Buffer Size */
  44
  45         /* Interrupt */
  46
  47         Icr             = 0x00C0,       /* Interrupt Cause Read */
  48         Itr             = 0x00c4,       /* Interrupt Throttling Rate */
  49         Ics             = 0x00C8,       /* Interrupt Cause Set */
  50         Ims             = 0x00D0,       /* Interrupt Mask Set/Read */
  51         Imc             = 0x00D8,       /* Interrupt mask Clear */
  52         Iam             = 0x00E0,       /* Interrupt acknowledge Auto Mask */
  53         Eitr            = 0x1680,       /* Extended itr; 82575/6 80 only */
  54
  55         /* Receive */
  56
  57         Rctl            = 0x0100,       /* Control */
  58         Ert             = 0x2008,       /* Early Receive Threshold (573[EVL], 82578 only) */
  59         Fcrtl           = 0x2160,       /* Flow Control RX Threshold Low */
  60         Fcrth           = 0x2168,       /* Flow Control Rx Threshold High */
  61         Psrctl          = 0x2170,       /* Packet Split Receive Control */
  62         Drxmxod         = 0x2540,       /* dma max outstanding bytes (82575) */
  63         Rdbal           = 0x2800,       /* Rdesc Base Address Low Queue 0 */
  64         Rdbah           = 0x2804,       /* Rdesc Base Address High Queue 0 */
  65         Rdlen           = 0x2808,       /* Descriptor Length Queue 0 */
  66         Srrctl          = 0x280c,       /* split and replication rx control (82575) */
  67         Rdh             = 0x2810,       /* Descriptor Head Queue 0 */
  68         Rdt             = 0x2818,       /* Descriptor Tail Queue 0 */
  69         Rdtr            = 0x2820,       /* Descriptor Timer Ring */
  70         Rxdctl          = 0x2828,       /* Descriptor Control */
  71         Radv            = 0x282C,       /* Interrupt Absolute Delay Timer */
  72         Rsrpd           = 0x2c00,       /* Small Packet Detect */
  73         Raid            = 0x2c08,       /* ACK interrupt delay */
  74         Cpuvec          = 0x2c10,       /* CPU Vector */
  75         Rxcsum          = 0x5000,       /* Checksum Control */
  76         Rmpl            = 0x5004,       /* rx maximum packet length (82575) */
  77         Rfctl           = 0x5008,       /* Filter Control */
  78         Mta             = 0x5200,       /* Multicast Table Array */
  79         Ral             = 0x5400,       /* Receive Address Low */
  80         Rah             = 0x5404,       /* Receive Address High */
  81         Vfta            = 0x5600,       /* VLAN Filter Table Array */
  82         Mrqc            = 0x5818,       /* Multiple Receive Queues Command */
  83
  84         /* Transmit */
  85
  86         Tctl            = 0x0400,       /* Transmit Control */
  87         Tipg            = 0x0410,       /* Transmit IPG */
  88         Tkabgtxd        = 0x3004,       /* glci afe band gap transmit ref data, or something */
  89         Tdbal           = 0x3800,       /* Tdesc Base Address Low */
  90         Tdbah           = 0x3804,       /* Tdesc Base Address High */
  91         Tdlen           = 0x3808,       /* Descriptor Length */
  92         Tdh             = 0x3810,       /* Descriptor Head */
  93         Tdt             = 0x3818,       /* Descriptor Tail */
  94         Tidv            = 0x3820,       /* Interrupt Delay Value */
  95         Txdctl          = 0x3828,       /* Descriptor Control */
  96         Tadv            = 0x382C,       /* Interrupt Absolute Delay Timer */
  97         Tarc0           = 0x3840,       /* Arbitration Counter Queue 0 */
  98
  99         /* Statistics */
 100
 101         Statistics      = 0x4000,       /* Start of Statistics Area */
 102         Gorcl           = 0x88/4,       /* Good Octets Received Count */
 103         Gotcl           = 0x90/4,       /* Good Octets Transmitted Count */
 104         Torl            = 0xC0/4,       /* Total Octets Received */
 105         Totl            = 0xC8/4,       /* Total Octets Transmitted */
 106         Nstatistics     = 0x124/4,
 107 };
 108
 109 enum {                                  /* Ctrl */
 110         Lrst            = 1<<3,         /* link reset */
 111         Slu             = 1<<6,         /* Set Link Up */
 112         Devrst          = 1<<26,        /* Device Reset */
 113         Rfce            = 1<<27,        /* Receive Flow Control Enable */
 114         Tfce            = 1<<28,        /* Transmit Flow Control Enable */
 115         Phyrst          = 1<<31,        /* Phy Reset */
 116 };
 117
 118 enum {                                  /* Status */
 119         Lu              = 1<<1,         /* Link Up */
 120         Lanid           = 3<<2,         /* mask for Lan ID. */
 121         Txoff           = 1<<4,         /* Transmission Paused */
 122         Tbimode         = 1<<5,         /* TBI Mode Indication */
 123         Phyra           = 1<<10,        /* PHY Reset Asserted */
 124         GIOme           = 1<<19,        /* GIO Master Enable Status */
 125 };
 126
 127 enum {                                  /* Eerd */
 128         EEstart         = 1<<0,         /* Start Read */
 129         EEdone          = 1<<1,         /* Read done */
 130 };
 131
 132 enum {                                  /* Ctrlext */
 133         Eerst           = 1<<13,        /* EEPROM Reset */
 134         Linkmode        = 3<<23,        /* linkmode */
 135         Serdes          = 3<<23,        /* " serdes */
 136 };
 137
 138 enum {                                  /* EEPROM content offsets */
 139         Ea              = 0x00,         /* Ethernet Address */
 140 };
 141
 142 enum {                                  /* Mdic */
 143         MDIdMASK        = 0x0000FFFF,   /* Data */
 144         MDIdSHIFT       = 0,
 145         MDIrMASK        = 0x001F0000,   /* PHY Register Address */
 146         MDIrSHIFT       = 16,
 147         MDIpMASK        = 0x03E00000,   /* PHY Address */
 148         MDIpSHIFT       = 21,
 149         MDIwop          = 0x04000000,   /* Write Operation */
 150         MDIrop          = 0x08000000,   /* Read Operation */
 151         MDIready        = 0x10000000,   /* End of Transaction */
 152         MDIie           = 0x20000000,   /* Interrupt Enable */
 153         MDIe            = 0x40000000,   /* Error */
 154 };
 155
 156 enum {                                  /* phy interface */
 157         Phyctl          = 0,            /* phy ctl register */
 158         Phyisr          = 19,           /* 82563 phy interrupt status register */
 159         Phylhr          = 19,           /* 8257[12] link health register */
 160         Physsr          = 17,           /* phy secondary status register */
 161         Phyprst         = 193<<8 | 17,  /* 8256[34] phy port reset */
 162         Phyier          = 18,           /* 82573 phy interrupt enable register */
 163         Phypage         = 22,           /* 8256[34] page register */
 164         Phystat         = 26,           /* 82580 phy status */
 165         Phyapage        = 29,
 166         Rtlink          = 1<<10,        /* realtime link status */
 167         Phyan           = 1<<11,        /* phy has autonegotiated */
 168
 169         /* Phyctl bits */
 170         Ran             = 1<<9, /* restart auto negotiation */
 171         Ean             = 1<<12,        /* enable auto negotiation */
 172
 173         /* Phyprst bits */
 174         Prst            = 1<<0, /* reset the port */
 175
 176         /* 82573 Phyier bits */
 177         Lscie           = 1<<10,        /* link status changed ie */
 178         Ancie           = 1<<11,        /* auto negotiation complete ie */
 179         Spdie           = 1<<14,        /* speed changed ie */
 180         Panie           = 1<<15,        /* phy auto negotiation error ie */
 181
 182         /* Phylhr/Phyisr bits */
 183         Anf             = 1<<6, /* lhr: auto negotiation fault */
 184         Ane             = 1<<15,        /* isr: auto negotiation error */
 185
 186         /* 82580 Phystat bits */
 187         Ans     = 1<<14 | 1<<15,        /* 82580 autoneg. status */
 188         Link    = 1<<6,         /* 82580 Link */
 189
 190         /* Rxcw builtin serdes */
 191         Anc             = 1<<31,
 192         Rxsynch         = 1<<30,
 193         Rxcfg           = 1<<29,
 194         Rxcfgch         = 1<<28,
 195         Rxcfgbad        = 1<<27,
 196         Rxnc            = 1<<26,
 197
 198         /* Txcw */
 199         Txane           = 1<<31,
 200         Txcfg           = 1<<30,
 201 };
 202
 203 enum {                                  /* fiber (pcs) interface */
 204         Pcsctl  = 0x4208,               /* pcs control */
 205         Pcsstat = 0x420c,               /* pcs status */
 206
 207         /* Pcsctl bits */
 208         Pan     = 1<<16,                /* autoegotiate */
 209         Prestart        = 1<<17,                /* restart an (self clearing) */
 210
 211         /* Pcsstat bits */
 212         Linkok  = 1<<0,         /* link is okay */
 213         Andone  = 1<<16,                /* an phase is done see below for success */
 214         Anbad   = 1<<19 | 1<<20,        /* Anerror | Anremfault */
 215 };
 216
 217 enum {                                  /* Icr, Ics, Ims, Imc */
 218         Txdw            = 0x00000001,   /* Transmit Descriptor Written Back */
 219         Txqe            = 0x00000002,   /* Transmit Queue Empty */
 220         Lsc             = 0x00000004,   /* Link Status Change */
 221         Rxseq           = 0x00000008,   /* Receive Sequence Error */
 222         Rxdmt0          = 0x00000010,   /* Rdesc Minimum Threshold Reached */
 223         Rxo             = 0x00000040,   /* Receiver Overrun */
 224         Rxt0            = 0x00000080,   /* Receiver Timer Interrupt; !82575/6/80 only */
 225         Rxdw            = 0x00000080,   /* Rdesc write back; 82575/6/80 only */
 226         Mdac            = 0x00000200,   /* MDIO Access Completed */
 227         Rxcfgset                = 0x00000400,   /* Receiving /C/ ordered sets */
 228         Ack             = 0x00020000,   /* Receive ACK frame */
 229 };
 230
 231 enum {                                  /* Txcw */
 232         TxcwFd          = 0x00000020,   /* Full Duplex */
 233         TxcwHd          = 0x00000040,   /* Half Duplex */
 234         TxcwPauseMASK   = 0x00000180,   /* Pause */
 235         TxcwPauseSHIFT  = 7,
 236         TxcwPs          = 1<<TxcwPauseSHIFT,    /* Pause Supported */
 237         TxcwAs          = 2<<TxcwPauseSHIFT,    /* Asymmetric FC desired */
 238         TxcwRfiMASK     = 0x00003000,   /* Remote Fault Indication */
 239         TxcwRfiSHIFT    = 12,
 240         TxcwNpr         = 0x00008000,   /* Next Page Request */
 241         TxcwConfig      = 0x40000000,   /* Transmit COnfig Control */
 242         TxcwAne         = 0x80000000,   /* Auto-Negotiation Enable */
 243 };
 244
 245 enum {                                  /* Rctl */
 246         Rrst            = 0x00000001,   /* Receiver Software Reset */
 247         Ren             = 0x00000002,   /* Receiver Enable */
 248         Sbp             = 0x00000004,   /* Store Bad Packets */
 249         Upe             = 0x00000008,   /* Unicast Promiscuous Enable */
 250         Mpe             = 0x00000010,   /* Multicast Promiscuous Enable */
 251         Lpe             = 0x00000020,   /* Long Packet Reception Enable */
 252         RdtmsMASK       = 0x00000300,   /* Rdesc Minimum Threshold Size */
 253         RdtmsHALF       = 0x00000000,   /* Threshold is 1/2 Rdlen */
 254         RdtmsQUARTER    = 0x00000100,   /* Threshold is 1/4 Rdlen */
 255         RdtmsEIGHTH     = 0x00000200,   /* Threshold is 1/8 Rdlen */
 256         MoMASK          = 0x00003000,   /* Multicast Offset */
 257         Bam             = 0x00008000,   /* Broadcast Accept Mode */
 258         BsizeMASK       = 0x00030000,   /* Receive Buffer Size */
 259         Bsize16384      = 0x00010000,   /* Bsex = 1 */
 260         Bsize8192       = 0x00020000,   /* Bsex = 1 */
 261         Bsize2048       = 0x00000000,
 262         Bsize1024       = 0x00010000,
 263         Bsize512        = 0x00020000,
 264         Bsize256        = 0x00030000,
 265         BsizeFlex       = 0x08000000,   /* Flexable Bsize in 1kb increments */
 266         Vfe             = 0x00040000,   /* VLAN Filter Enable */
 267         Cfien           = 0x00080000,   /* Canonical Form Indicator Enable */
 268         Cfi             = 0x00100000,   /* Canonical Form Indicator value */
 269         Dpf             = 0x00400000,   /* Discard Pause Frames */
 270         Pmcf            = 0x00800000,   /* Pass MAC Control Frames */
 271         Bsex            = 0x02000000,   /* Buffer Size Extension */
 272         Secrc           = 0x04000000,   /* Strip CRC from incoming packet */
 273 };
 274
 275 enum {                                  /* Srrctl */
 276         Dropen          = 1<<31,
 277 };
 278
 279 enum {                                  /* Tctl */
 280         Trst            = 0x00000001,   /* Transmitter Software Reset */
 281         Ten             = 0x00000002,   /* Transmit Enable */
 282         Psp             = 0x00000008,   /* Pad Short Packets */
 283         Mulr            = 0x10000000,   /* Allow multiple concurrent requests */
 284         CtMASK          = 0x00000FF0,   /* Collision Threshold */
 285         CtSHIFT         = 4,
 286         ColdMASK        = 0x003FF000,   /* Collision Distance */
 287         ColdSHIFT       = 12,
 288         Swxoff          = 0x00400000,   /* Sofware XOFF Transmission */
 289         Pbe             = 0x00800000,   /* Packet Burst Enable */
 290         Rtlc            = 0x01000000,   /* Re-transmit on Late Collision */
 291         Nrtu            = 0x02000000,   /* No Re-transmit on Underrrun */
 292 };
 293
 294 enum {                                  /* [RT]xdctl */
 295         PthreshMASK     = 0x0000003F,   /* Prefetch Threshold */
 296         PthreshSHIFT    = 0,
 297         HthreshMASK     = 0x00003F00,   /* Host Threshold */
 298         HthreshSHIFT    = 8,
 299         WthreshMASK     = 0x003F0000,   /* Writeback Threshold */
 300         WthreshSHIFT    = 16,
 301         Gran            = 0x01000000,   /* Granularity; not 82575 */
 302         Enable          = 0x02000000,
 303 };
 304
 305 enum {                                  /* Rxcsum */
 306         Ipofl           = 0x0100,       /* IP Checksum Off-load Enable */
 307         Tuofl           = 0x0200,       /* TCP/UDP Checksum Off-load Enable */
 308 };
 309
 310 typedef struct Rd {                     /* Receive Descriptor */
 311         uint    addr[2];
 312         ushort  length;
 313         ushort  checksum;
 314         uchar   status;
 315         uchar   errors;
 316         ushort  special;
 317 } Rd;
 318
 319 enum {                                  /* Rd status */
 320         Rdd             = 0x01,         /* Descriptor Done */
 321         Reop            = 0x02,         /* End of Packet */
 322         Ixsm            = 0x04,         /* Ignore Checksum Indication */
 323         Vp              = 0x08,         /* Packet is 802.1Q (matched VET) */
 324         Tcpcs           = 0x20,         /* TCP Checksum Calculated on Packet */
 325         Ipcs            = 0x40,         /* IP Checksum Calculated on Packet */
 326         Pif             = 0x80,         /* Passed in-exact filter */
 327 };
 328
 329 enum {                                  /* Rd errors */
 330         Ce              = 0x01,         /* CRC Error or Alignment Error */
 331         Se              = 0x02,         /* Symbol Error */
 332         Seq             = 0x04,         /* Sequence Error */
 333         Cxe             = 0x10,         /* Carrier Extension Error */
 334         Tcpe            = 0x20,         /* TCP/UDP Checksum Error */
 335         Ipe             = 0x40,         /* IP Checksum Error */
 336         Rxe             = 0x80,         /* RX Data Error */
 337 };
 338
 339 typedef struct {                        /* Transmit Descriptor */
 340         uint    addr[2];                /* Data */
 341         uint    control;
 342         uint    status;
 343 } Td;
 344
 345 enum {                                  /* Tdesc control */
 346         LenMASK         = 0x000FFFFF,   /* Data/Packet Length Field */
 347         LenSHIFT        = 0,
 348         DtypeCD         = 0x00000000,   /* Data Type 'Context Descriptor' */
 349         DtypeDD         = 0x00100000,   /* Data Type 'Data Descriptor' */
 350         PtypeTCP        = 0x01000000,   /* TCP/UDP Packet Type (CD) */
 351         Teop            = 0x01000000,   /* End of Packet (DD) */
 352         PtypeIP         = 0x02000000,   /* IP Packet Type (CD) */
 353         Ifcs            = 0x02000000,   /* Insert FCS (DD) */
 354         Tse             = 0x04000000,   /* TCP Segmentation Enable */
 355         Rs              = 0x08000000,   /* Report Status */
 356         Rps             = 0x10000000,   /* Report Status Sent */
 357         Dext            = 0x20000000,   /* Descriptor Extension */
 358         Vle             = 0x40000000,   /* VLAN Packet Enable */
 359         Ide             = 0x80000000,   /* Interrupt Delay Enable */
 360 };
 361
 362 enum {                                  /* Tdesc status */
 363         Tdd             = 0x0001,       /* Descriptor Done */
 364         Ec              = 0x0002,       /* Excess Collisions */
 365         Lc              = 0x0004,       /* Late Collision */
 366         Tu              = 0x0008,       /* Transmit Underrun */
 367         CssMASK         = 0xFF00,       /* Checksum Start Field */
 368         CssSHIFT        = 8,
 369 };
 370
 371 typedef struct {
 372         ushort  *reg;
 373         ulong   *reg32;
 374         int     sz;
 375 } Flash;
 376
 377 enum {
 378         /* 16 and 32-bit flash registers for ich flash parts */
 379         Bfpr    = 0x00/4,               /* flash base 0:12; lim 16:28 */
 380         Fsts    = 0x04/2,               /* flash status; Hsfsts */
 381         Fctl    = 0x06/2,               /* flash control; Hsfctl */
 382         Faddr   = 0x08/4,               /* flash address to r/w */
 383         Fdata   = 0x10/4,               /* data @ address */
 384
 385         /* status register */
 386         Fdone   = 1<<0,                 /* flash cycle done */
 387         Fcerr   = 1<<1,                 /* cycle error; write 1 to clear */
 388         Ael     = 1<<2,                 /* direct access error log; 1 to clear */
 389         Scip    = 1<<5,                 /* spi cycle in progress */
 390         Fvalid  = 1<<14,                /* flash descriptor valid */
 391
 392         /* control register */
 393         Fgo     = 1<<0,                 /* start cycle */
 394         Flcycle = 1<<1,                 /* two bits: r=0; w=2 */
 395         Fdbc    = 1<<8,                 /* bytes to read; 5 bits */
 396 };
 397
 398 enum {
 399         Nrd             = 256,          /* power of two */
 400         Ntd             = 128,          /* power of two */
 401         Nrb             = 512+512,      /* private receive buffers per Ctlr */
 402         Rbalign         = BY2PG,        /* rx buffer alignment */
 403         Npool           = 8,
 404 };
 405
 406 /*
 407  * cavet emptor: 82577/78 have been entered speculatitively.
 408  * awating datasheet from intel.
 409  */
 410 enum {
 411         i82563,
 412         i82566,
 413         i82567,
 414         i82567m,
 415         i82571,
 416         i82572,
 417         i82573,
 418         i82574,
 419         i82575,
 420         i82576,
 421         i82577,
 422         i82577m,
 423         i82578,
 424         i82578m,
 425         i82579,
 426         i82580,
 427         i82583,
 428         Nctlrtype,
 429 };
 430
 431 enum {
 432         Fload   = 1<<0,
 433         Fert    = 1<<1,
 434         F75     = 1<<2,
 435         Fpba    = 1<<3,
 436         Fflashea        = 1<<4,
 437 };
 438
 439 typedef struct Ctlrtype Ctlrtype;
 440 struct Ctlrtype {
 441         int     type;
 442         int     mtu;
 443         int     flag;
 444         char    *name;
 445 };
 446
 447 static Ctlrtype cttab[Nctlrtype] = {
 448         i82563,         9014,   Fpba,           "i82563",
 449         i82566,         1514,   Fload,          "i82566",
 450         i82567,         9234,   Fload,          "i82567",
 451         i82567m,                1514,   0,              "i82567m",
 452         i82571,         9234,   Fpba,           "i82571",
 453         i82572,         9234,   Fpba,           "i82572",
 454         i82573,         8192,   Fert,           "i82573",               /* terrible perf above 8k */
 455         i82574,         9018,   0,              "i82574",
 456         i82575,         9728,   F75|Fflashea,   "i82575",
 457         i82576,         9728,   F75,            "i82576",
 458         i82577,         4096,   Fload|Fert,     "i82577",
 459         i82577m,                1514,   Fload|Fert,     "i82577",
 460         i82578,         4096,   Fload|Fert,     "i82578",
 461         i82578m,                1514,   Fload|Fert,     "i82578",
 462         i82579,         9018,   Fload|Fert,     "i82579",
 463         i82580,         9728,   F75,            "i82580",
 464         i82583,         1514,   0,              "i82583",
 465 };
 466
 467 typedef void (*Freefn)(Block*);
 468
 469 typedef struct Ctlr Ctlr;
 470 struct Ctlr {
 471         ulong   port;
 472         Pcidev  *pcidev;
 473         Ctlr    *next;
 474         int     active;
 475         int     type;
 476         int     pool;
 477         ushort  eeprom[0x40];
 478
 479         QLock   alock;                  /* attach */
 480         void    *alloc;                 /* receive/transmit descriptors */
 481         int     nrd;
 482         int     ntd;
 483         uint    rbsz;
 484
 485         int     *nic;
 486         Lock    imlock;
 487         int     im;                     /* interrupt mask */
 488
 489         Rendez  lrendez;
 490         int     lim;
 491
 492         QLock   slock;
 493         uint    statistics[Nstatistics];
 494         uint    lsleep;
 495         uint    lintr;
 496         uint    rsleep;
 497         uint    rintr;
 498         uint    txdw;
 499         uint    tintr;
 500         uint    ixsm;
 501         uint    ipcs;
 502         uint    tcpcs;
 503         uint    speeds[4];
 504         uint    phyerrata;
 505
 506         uchar   ra[Eaddrlen];           /* receive address */
 507         ulong   mta[128];               /* multicast table array */
 508
 509         Rendez  rrendez;
 510         int     rim;
 511         int     rdfree;
 512         Rd      *rdba;                  /* receive descriptor base address */
 513         Block   **rb;                   /* receive buffers */
 514         uint    rdh;                    /* receive descriptor head */
 515         uint    rdt;                    /* receive descriptor tail */
 516         int     rdtr;                   /* receive delay timer ring value */
 517         int     radv;                   /* receive interrupt absolute delay timer */
 518
 519         Rendez  trendez;
 520         QLock   tlock;
 521         int     tbusy;
 522         Td      *tdba;                  /* transmit descriptor base address */
 523         Block   **tb;                   /* transmit buffers */
 524         int     tdh;                    /* transmit descriptor head */
 525         int     tdt;                    /* transmit descriptor tail */
 526
 527         int     fcrtl;
 528         int     fcrth;
 529
 530         uint    pba;                    /* packet buffer allocation */
 531 };
 532
 533 typedef struct Rbpool Rbpool;
 534 struct Rbpool {
 535         union {
 536                 struct {
 537                         Lock;
 538                         Block   *b;
 539                         uint    nstarve;
 540                         uint    nwakey;
 541                         uint    starve;
 542                         Rendez;
 543                 };
 544                 uchar pad[128];         /* cacheline */
 545         };
 546
 547         Block   *x;
 548         uint    nfast;
 549         uint    nslow;
 550 };
 551
 552 #define csr32r(c, r)    (*((c)->nic+((r)/4)))
 553 #define csr32w(c, r, v) (*((c)->nic+((r)/4)) = (v))
 554
 555 static  Ctlr    *i82563ctlrhead;
 556 static  Ctlr    *i82563ctlrtail;
 557 static  Rbpool  rbtab[Npool];
 558
 559 static char *statistics[Nstatistics] = {
 560         "CRC Error",
 561         "Alignment Error",
 562         "Symbol Error",
 563         "RX Error",
 564         "Missed Packets",
 565         "Single Collision",
 566         "Excessive Collisions",
 567         "Multiple Collision",
 568         "Late Collisions",
 569         nil,
 570         "Collision",
 571         "Transmit Underrun",
 572         "Defer",
 573         "Transmit - No CRS",
 574         "Sequence Error",
 575         "Carrier Extension Error",
 576         "Receive Error Length",
 577         nil,
 578         "XON Received",
 579         "XON Transmitted",
 580         "XOFF Received",
 581         "XOFF Transmitted",
 582         "FC Received Unsupported",
 583         "Packets Received (64 Bytes)",
 584         "Packets Received (65-127 Bytes)",
 585         "Packets Received (128-255 Bytes)",
 586         "Packets Received (256-511 Bytes)",
 587         "Packets Received (512-1023 Bytes)",
 588         "Packets Received (1024-mtu Bytes)",
 589         "Good Packets Received",
 590         "Broadcast Packets Received",
 591         "Multicast Packets Received",
 592         "Good Packets Transmitted",
 593         nil,
 594         "Good Octets Received",
 595         nil,
 596         "Good Octets Transmitted",
 597         nil,
 598         nil,
 599         nil,
 600         "Receive No Buffers",
 601         "Receive Undersize",
 602         "Receive Fragment",
 603         "Receive Oversize",
 604         "Receive Jabber",
 605         "Management Packets Rx",
 606         "Management Packets Drop",
 607         "Management Packets Tx",
 608         "Total Octets Received",
 609         nil,
 610         "Total Octets Transmitted",
 611         nil,
 612         "Total Packets Received",
 613         "Total Packets Transmitted",
 614         "Packets Transmitted (64 Bytes)",
 615         "Packets Transmitted (65-127 Bytes)",
 616         "Packets Transmitted (128-255 Bytes)",
 617         "Packets Transmitted (256-511 Bytes)",
 618         "Packets Transmitted (512-1023 Bytes)",
 619         "Packets Transmitted (1024-mtu Bytes)",
 620         "Multicast Packets Transmitted",
 621         "Broadcast Packets Transmitted",
 622         "TCP Segmentation Context Transmitted",
 623         "TCP Segmentation Context Fail",
 624         "Interrupt Assertion",
 625         "Interrupt Rx Pkt Timer",
 626         "Interrupt Rx Abs Timer",
 627         "Interrupt Tx Pkt Timer",
 628         "Interrupt Tx Abs Timer",
 629         "Interrupt Tx Queue Empty",
 630         "Interrupt Tx Desc Low",
 631         "Interrupt Rx Min",
 632         "Interrupt Rx Overrun",
 633 };
 634
 635 static char*
 636 cname(Ctlr *c)
 637 {
 638         return cttab[c->type].name;
 639 }
 640
 641 static long
 642 i82563ifstat(Ether *edev, void *a, long n, ulong offset)
 643 {
 644         char *s, *p, *e, *stat;
 645         int i, r;
 646         uvlong tuvl, ruvl;
 647         Ctlr *ctlr;
 648         Rbpool *b;
 649
 650         p = s = smalloc(READSTR);
 651         e = p + READSTR;
 652
 653         ctlr = edev->ctlr;
 654         qlock(&ctlr->slock);
 655
 656         for(i = 0; i < Nstatistics; i++){
 657                 r = csr32r(ctlr, Statistics + i*4);
 658                 if((stat = statistics[i]) == nil)
 659                         continue;
 660                 switch(i){
 661                 case Gorcl:
 662                 case Gotcl:
 663                 case Torl:
 664                 case Totl:
 665                         ruvl = r;
 666                         ruvl += (uvlong)csr32r(ctlr, Statistics+(i+1)*4) << 32;
 667                         tuvl = ruvl;
 668                         tuvl += ctlr->statistics[i];
 669                         tuvl += (uvlong)ctlr->statistics[i+1] << 32;
 670                         if(tuvl == 0)
 671                                 continue;
 672                         ctlr->statistics[i] = tuvl;
 673                         ctlr->statistics[i+1] = tuvl >> 32;
 674                         p = seprint(p, e, "%s: %llud %llud\n", stat, tuvl, ruvl);
 675                         i++;
 676                         break;
 677
 678                 default:
 679                         ctlr->statistics[i] += r;
 680                         if(ctlr->statistics[i] == 0)
 681                                 continue;
 682                         p = seprint(p, e, "%s: %ud %ud\n", stat,
 683                                 ctlr->statistics[i], r);
 684                         break;
 685                 }
 686         }
 687
 688         p = seprint(p, e, "lintr: %ud %ud\n", ctlr->lintr, ctlr->lsleep);
 689         p = seprint(p, e, "rintr: %ud %ud\n", ctlr->rintr, ctlr->rsleep);
 690         p = seprint(p, e, "tintr: %ud %ud\n", ctlr->tintr, ctlr->txdw);
 691         p = seprint(p, e, "ixcs: %ud %ud %ud\n", ctlr->ixsm, ctlr->ipcs, ctlr->tcpcs);
 692         p = seprint(p, e, "rdtr: %ud\n", ctlr->rdtr);
 693         p = seprint(p, e, "radv: %ud\n", ctlr->radv);
 694         p = seprint(p, e, "ctrl: %.8ux\n", csr32r(ctlr, Ctrl));
 695         p = seprint(p, e, "ctrlext: %.8ux\n", csr32r(ctlr, Ctrlext));
 696         p = seprint(p, e, "status: %.8ux\n", csr32r(ctlr, Status));
 697         p = seprint(p, e, "txcw: %.8ux\n", csr32r(ctlr, Txcw));
 698         p = seprint(p, e, "txdctl: %.8ux\n", csr32r(ctlr, Txdctl));
 699         p = seprint(p, e, "pba: %.8ux\n", ctlr->pba);
 700
 701         b = rbtab + ctlr->pool;
 702         p = seprint(p, e, "pool: fast %ud slow %ud nstarve %ud nwakey %ud starve %ud\n",
 703                 b->nfast, b->nslow, b->nstarve, b->nwakey, b->starve);
 704         p = seprint(p, e, "speeds: 10:%ud 100:%ud 1000:%ud ?:%ud\n",
 705                 ctlr->speeds[0], ctlr->speeds[1], ctlr->speeds[2], ctlr->speeds[3]);
 706         p = seprint(p, e, "type: %s\n", cname(ctlr));
 707
 708 //      p = seprint(p, e, "eeprom:");
 709 //      for(i = 0; i < 0x40; i++){
 710 //              if(i && ((i & 7) == 0))
 711 //                      p = seprint(p, e, "\n       ");
 712 //              p = seprint(p, e, " %4.4ux", ctlr->eeprom[i]);
 713 //      }
 714 //      p = seprint(p, e, "\n");
 715
 716         USED(p);
 717         n = readstr(offset, a, n, s);
 718         free(s);
 719         qunlock(&ctlr->slock);
 720
 721         return n;
 722 }
 723
 724 static void
 725 i82563promiscuous(void *arg, int on)
 726 {
 727         int rctl;
 728         Ctlr *ctlr;
 729         Ether *edev;
 730
 731         edev = arg;
 732         ctlr = edev->ctlr;
 733
 734         rctl = csr32r(ctlr, Rctl);
 735         rctl &= ~MoMASK;
 736         if(on)
 737                 rctl |= Upe|Mpe;
 738         else
 739                 rctl &= ~(Upe|Mpe);
 740         csr32w(ctlr, Rctl, rctl);
 741 }
 742
 743 static void
 744 i82563multicast(void *arg, uchar *addr, int on)
 745 {
 746         int bit, x;
 747         Ctlr *ctlr;
 748         Ether *edev;
 749
 750         edev = arg;
 751         ctlr = edev->ctlr;
 752
 753         x = addr[5]>>1;
 754         if(ctlr->type == i82566)
 755                 x &= 31;
 756         bit = ((addr[5] & 1)<<4)|(addr[4]>>4);
 757         /*
 758          * multiple ether addresses can hash to the same filter bit,
 759          * so it's never safe to clear a filter bit.
 760          * if we want to clear filter bits, we need to keep track of
 761          * all the multicast addresses in use, clear all the filter bits,
 762          * then set the ones corresponding to in-use addresses.
 763          */
 764         if(on)
 765                 ctlr->mta[x] |= 1<<bit;
 766 //      else
 767 //              ctlr->mta[x] &= ~(1<<bit);
 768
 769         csr32w(ctlr, Mta+x*4, ctlr->mta[x]);
 770 }
 771
 772 static int
 773 icansleep(void *v)
 774 {
 775         Rbpool *p;
 776         int r;
 777
 778         p = v;
 779         ilock(p);
 780         r = p->starve == 0;
 781         iunlock(p);
 782
 783         return r;
 784 }
 785
 786 static Block*
 787 i82563rballoc(Rbpool *p)
 788 {
 789         Block *b;
 790
 791         for(;;){
 792                 if((b = p->x) != nil){
 793                         p->nfast++;
 794                         p->x = b->next;
 795                         b->next = nil;
 796                         _xinc(&b->ref);
 797                         return b;
 798                 }
 799
 800                 ilock(p);
 801                 b = p->b;
 802                 p->b = nil;
 803                 if(b == nil){
 804                         p->nstarve++;
 805                         iunlock(p);
 806                         return nil;
 807                 }
 808                 p->nslow++;
 809                 iunlock(p);
 810                 p->x = b;
 811         }
 812 }
 813
 814 static void
 815 rbfree(Block *b, int t)
 816 {
 817         Rbpool *p;
 818
 819         p = rbtab + t;
 820         b->rp = b->wp = (uchar*)ROUND((uintptr)b->base, Rbalign);
 821         b->flag &= ~(Bipck | Budpck | Btcpck | Bpktck);
 822
 823         ilock(p);
 824         b->next = p->b;
 825         p->b = b;
 826         if(p->starve){
 827                 if(0)
 828                         iprint("wakey %d; %d %d\n", t, p->nstarve, p->nwakey);
 829                 p->nwakey++;
 830                 p->starve = 0;
 831                 wakeup(p);
 832         }
 833         iunlock(p);
 834 }
 835
 836 static void
 837 rbfree0(Block *b)
 838 {
 839         rbfree(b, 0);
 840 }
 841
 842 static void
 843 rbfree1(Block *b)
 844 {
 845         rbfree(b, 1);
 846 }
 847
 848 static void
 849 rbfree2(Block *b)
 850 {
 851         rbfree(b, 2);
 852 }
 853
 854 static void
 855 rbfree3(Block *b)
 856 {
 857         rbfree(b, 3);
 858 }
 859
 860 static void
 861 rbfree4(Block *b)
 862 {
 863         rbfree(b, 4);
 864 }
 865
 866 static void
 867 rbfree5(Block *b)
 868 {
 869         rbfree(b, 5);
 870 }
 871
 872 static void
 873 rbfree6(Block *b)
 874 {
 875         rbfree(b, 6);
 876 }
 877
 878 static void
 879 rbfree7(Block *b)
 880 {
 881         rbfree(b, 7);
 882 }
 883
 884 static Freefn freetab[Npool] = {
 885         rbfree0,
 886         rbfree1,
 887         rbfree2,
 888         rbfree3,
 889         rbfree4,
 890         rbfree5,
 891         rbfree6,
 892         rbfree7,
 893 };
 894
 895 static int
 896 newpool(void)
 897 {
 898         static int seq;
 899
 900         if(seq == nelem(freetab))
 901                 return -1;
 902         if(freetab[seq] == nil){
 903                 print("82563: bad freetab\n");
 904                 return -1;
 905         }
 906         return seq++;
 907 }
 908
 909 static void
 910 i82563im(Ctlr *ctlr, int im)
 911 {
 912         ilock(&ctlr->imlock);
 913         ctlr->im |= im;
 914         csr32w(ctlr, Ims, ctlr->im);
 915         iunlock(&ctlr->imlock);
 916 }
 917
 918 static void
 919 i82563txinit(Ctlr *ctlr)
 920 {
 921         int i, r;
 922         Block *b;
 923
 924         if(cttab[ctlr->type].flag & F75)
 925                 csr32w(ctlr, Tctl, 0x0F<<CtSHIFT | Psp);
 926         else
 927                 csr32w(ctlr, Tctl, 0x0F<<CtSHIFT | Psp | 66<<ColdSHIFT | Mulr);
 928         csr32w(ctlr, Tipg, 6<<20 | 8<<10 | 8);          /* yb sez: 0x702008 */
 929         csr32w(ctlr, Tdbal, PCIWADDR(ctlr->tdba));
 930 //      csr32w(ctlr, Tdbah, Pciwaddrh(ctlr->tdba));
 931         csr32w(ctlr, Tdbah, 0);
 932         csr32w(ctlr, Tdlen, ctlr->ntd * sizeof(Td));
 933         ctlr->tdh = PREV(0, ctlr->ntd);
 934         csr32w(ctlr, Tdh, 0);
 935         ctlr->tdt = 0;
 936         csr32w(ctlr, Tdt, 0);
 937         for(i = 0; i < ctlr->ntd; i++){
 938                 if((b = ctlr->tb[i]) != nil){
 939                         ctlr->tb[i] = nil;
 940                         freeb(b);
 941                 }
 942                 memset(&ctlr->tdba[i], 0, sizeof(Td));
 943         }
 944         csr32w(ctlr, Tidv, 128);
 945         csr32w(ctlr, Tadv, 64);
 946         csr32w(ctlr, Tctl, csr32r(ctlr, Tctl) | Ten);
 947         r = csr32r(ctlr, Txdctl) & ~WthreshMASK;
 948         r |= 4<<WthreshSHIFT | 4<<PthreshSHIFT;
 949         if(cttab[ctlr->type].flag & F75)
 950                 r |= Enable;
 951         csr32w(ctlr, Txdctl, r);
 952 }
 953
 954 #define Next(x, m)      (((x)+1) & (m))
 955
 956 static int
 957 i82563cleanup(Ctlr *c)
 958 {
 959         Block *b;
 960         int tdh, m, n;
 961
 962         tdh = c->tdh;
 963         m = c->ntd-1;
 964         while(c->tdba[n = Next(tdh, m)].status & Tdd){
 965                 tdh = n;
 966                 if((b = c->tb[tdh]) != nil){
 967                         c->tb[tdh] = nil;
 968                         freeb(b);
 969                 }else
 970                         iprint("82563 tx underrun!\n");
 971                 c->tdba[tdh].status = 0;
 972         }
 973
 974         return c->tdh = tdh;
 975 }
 976
 977 static int
 978 notrim(void *v)
 979 {
 980         Ctlr *c;
 981
 982         c = v;
 983         return (c->im & Txdw) == 0;
 984 }
 985
 986 static void
 987 i82563tproc(void *v)
 988 {
 989         Td *td;
 990         Block *bp;
 991         Ether *edev;
 992         Ctlr *ctlr;
 993         int tdh, tdt, m;
 994
 995         edev = v;
 996         ctlr = edev->ctlr;
 997         tdt = ctlr->tdt;
 998         m = ctlr->ntd-1;
 999
1000         i82563txinit(ctlr);
1001
1002         for(;;){
1003                 tdh = i82563cleanup(ctlr);
1004
1005                 if(Next(tdt, m) == tdh){
1006                         ctlr->txdw++;
1007                         i82563im(ctlr, Txdw);
1008                         sleep(&ctlr->trendez, notrim, ctlr);
1009                 }
1010                 bp = qbread(edev->oq, 100000);
1011                 td = &ctlr->tdba[tdt];
1012                 td->addr[0] = PCIWADDR(bp->rp);
1013 //              td->addr[1] = Pciwaddrh(bp->rp);
1014                 td->control = Ide|Rs|Ifcs|Teop|BLEN(bp);
1015                 ctlr->tb[tdt] = bp;
1016                 tdt = Next(tdt, m);
1017                 csr32w(ctlr, Tdt, tdt);
1018         }
1019 }
1020
1021 static int
1022 i82563replenish(Ctlr *ctlr, int maysleep)
1023 {
1024         uint rdt, m, i;
1025         Block *bp;
1026         Rbpool *p;
1027         Rd *rd;
1028
1029         rdt = ctlr->rdt;
1030         m = ctlr->nrd-1;
1031         p = rbtab + ctlr->pool;
1032         i = 0;
1033         for(; Next(rdt, m) != ctlr->rdh; rdt = Next(rdt, m)){
1034                 rd = &ctlr->rdba[rdt];
1035                 if(ctlr->rb[rdt] != nil){
1036                         iprint("82563: tx overrun\n");
1037                         break;
1038                 }
1039         redux:
1040                 bp = i82563rballoc(p);
1041                 if(bp == nil){
1042                         if(rdt - ctlr->rdh >= 16)
1043                                 break;
1044                         print("i82563%d: no rx buffers\n", ctlr->pool);
1045                         if(maysleep == 0)
1046                                 return -1;
1047                         ilock(p);
1048                         p->starve = 1;
1049                         iunlock(p);
1050                         sleep(p, icansleep, p);
1051                         goto redux;
1052                 }
1053                 i++;
1054                 ctlr->rb[rdt] = bp;
1055                 rd->addr[0] = PCIWADDR(bp->rp);
1056         //      rd->addr[1] = Pciwaddrh(bp->rp);
1057                 rd->status = 0;
1058                 ctlr->rdfree++;
1059         }
1060         if(i != 0){
1061                 ctlr->rdt = rdt;
1062                 csr32w(ctlr, Rdt, rdt);
1063         }
1064         return 0;
1065 }
1066
1067 static void
1068 i82563rxinit(Ctlr *ctlr)
1069 {
1070         int i;
1071         Block *bp;
1072
1073         if(ctlr->rbsz <= 2048)
1074                 csr32w(ctlr, Rctl, Dpf|Bsize2048|Bam|RdtmsHALF);
1075         else{
1076                 i = ctlr->rbsz / 1024;
1077                 if(ctlr->rbsz % 1024)
1078                         i++;
1079                 if(cttab[ctlr->type].flag & F75){
1080                         csr32w(ctlr, Rctl, Lpe|Dpf|Bsize2048|Bam|RdtmsHALF|Secrc);
1081                         if(ctlr->type != i82575)
1082                                 i |= (ctlr->nrd/2>>4)<<20;              /* RdmsHalf */
1083                         csr32w(ctlr, Srrctl, i | Dropen);
1084                         csr32w(ctlr, Rmpl, ctlr->rbsz);
1085 //                      csr32w(ctlr, Drxmxod, 0x7ff);
1086                 }else
1087                         csr32w(ctlr, Rctl, Lpe|Dpf|BsizeFlex*i|Bam|RdtmsHALF|Secrc);
1088         }
1089
1090         if(cttab[ctlr->type].flag & Fert)
1091                 csr32w(ctlr, Ert, 1024/8);
1092
1093         if(ctlr->type == i82566)
1094                 csr32w(ctlr, Pbs, 16);
1095
1096         csr32w(ctlr, Rdbal, PCIWADDR(ctlr->rdba));
1097 //      csr32w(ctlr, Rdbah, Pciwaddrh(ctlr->rdba));
1098         csr32w(ctlr, Rdbah, 0);
1099         csr32w(ctlr, Rdlen, ctlr->nrd * sizeof(Rd));
1100         ctlr->rdh = 0;
1101         csr32w(ctlr, Rdh, 0);
1102         ctlr->rdt = 0;
1103         csr32w(ctlr, Rdt, 0);
1104         ctlr->rdtr = 25;
1105         ctlr->radv = 500;
1106         csr32w(ctlr, Rdtr, ctlr->rdtr);
1107         csr32w(ctlr, Radv, ctlr->radv);
1108
1109         for(i = 0; i < ctlr->nrd; i++)
1110                 if((bp = ctlr->rb[i]) != nil){
1111                         ctlr->rb[i] = nil;
1112                         freeb(bp);
1113                 }
1114         if(cttab[ctlr->type].flag & F75)
1115                 csr32w(ctlr, Rxdctl, 1<<WthreshSHIFT | 8<<PthreshSHIFT | 1<<HthreshSHIFT | Enable);
1116         else
1117                 csr32w(ctlr, Rxdctl, 2<<WthreshSHIFT | 2<<PthreshSHIFT);
1118
1119         /*
1120          * Enable checksum offload.
1121          */
1122         csr32w(ctlr, Rxcsum, Tuofl | Ipofl | ETHERHDRSIZE);
1123 }
1124
1125 static int
1126 i82563rim(void *v)
1127 {
1128         return ((Ctlr*)v)->rim != 0;
1129 }
1130
1131 static void
1132 i82563rproc(void *arg)
1133 {
1134         uint m, rdh, rim, im;
1135         Block *bp;
1136         Ctlr *ctlr;
1137         Ether *edev;
1138         Rd *rd;
1139
1140         edev = arg;
1141         ctlr = edev->ctlr;
1142
1143         i82563rxinit(ctlr);
1144         csr32w(ctlr, Rctl, csr32r(ctlr, Rctl) | Ren);
1145         if(cttab[ctlr->type].flag & F75){
1146                 csr32w(ctlr, Rxdctl, csr32r(ctlr, Rxdctl) | Enable);
1147                 im = Rxt0|Rxo|Rxdmt0|Rxseq|Ack;
1148         }else
1149                 im = Rxt0|Rxo|Rxdmt0|Rxseq|Ack;
1150         m = ctlr->nrd-1;
1151
1152         for(;;){
1153                 i82563im(ctlr, im);
1154                 ctlr->rsleep++;
1155                 i82563replenish(ctlr, 1);
1156                 sleep(&ctlr->rrendez, i82563rim, ctlr);
1157
1158                 rdh = ctlr->rdh;
1159                 for(;;){
1160                         rd = &ctlr->rdba[rdh];
1161                         rim = ctlr->rim;
1162                         ctlr->rim = 0;
1163                         if(!(rd->status & Rdd))
1164                                 break;
1165
1166                         /*
1167                          * Accept eop packets with no errors.
1168                          * With no errors and the Ixsm bit set,
1169                          * the descriptor status Tpcs and Ipcs bits give
1170                          * an indication of whether the checksums were
1171                          * calculated and valid.
1172                          */
1173                         bp = ctlr->rb[rdh];
1174                         if((rd->status & Reop) && rd->errors == 0){
1175                                 bp->wp += rd->length;
1176                                 bp->lim = bp->wp;       /* lie like a dog.  avoid packblock. */
1177                                 if(!(rd->status & Ixsm)){
1178                                         ctlr->ixsm++;
1179                                         if(rd->status & Ipcs){
1180                                                 /*
1181                                                  * IP checksum calculated
1182                                                  * (and valid as errors == 0).
1183                                                  */
1184                                                 ctlr->ipcs++;
1185                                                 bp->flag |= Bipck;
1186                                         }
1187                                         if(rd->status & Tcpcs){
1188                                                 /*
1189                                                  * TCP/UDP checksum calculated
1190                                                  * (and valid as errors == 0).
1191                                                  */
1192                                                 ctlr->tcpcs++;
1193                                                 bp->flag |= Btcpck|Budpck;
1194                                         }
1195                                         bp->checksum = rd->checksum;
1196                                         bp->flag |= Bpktck;
1197                                 }
1198                                 etheriq(edev, bp, 1);
1199                         } else
1200                                 freeb(bp);
1201                         ctlr->rb[rdh] = nil;
1202                         rd->status = 0;
1203                         ctlr->rdfree--;
1204                         ctlr->rdh = rdh = Next(rdh, m);
1205                         if(ctlr->nrd-ctlr->rdfree >= 32 || (rim & Rxdmt0))
1206                                 if(i82563replenish(ctlr, 0) == -1)
1207                                         break;
1208                 }
1209         }
1210 }
1211
1212 static int
1213 i82563lim(void *v)
1214 {
1215         return ((Ctlr*)v)->lim != 0;
1216 }
1217
1218 static int speedtab[] = {
1219         10, 100, 1000, 0
1220 };
1221
1222 static uint
1223 phyread(Ctlr *c, int phyno, int reg)
1224 {
1225         uint phy, i;
1226
1227         csr32w(c, Mdic, MDIrop | phyno<<MDIpSHIFT | reg<<MDIrSHIFT);
1228         phy = 0;
1229         for(i = 0; i < 64; i++){
1230                 phy = csr32r(c, Mdic);
1231                 if(phy & (MDIe|MDIready))
1232                         break;
1233                 microdelay(1);
1234         }
1235         if((phy & (MDIe|MDIready)) != MDIready){
1236                 print("%s: phy %d wedged %.8ux\n", cttab[c->type].name, phyno, phy);
1237                 return ~0;
1238         }
1239         return phy & 0xffff;
1240 }
1241
1242 static uint
1243 phywrite0(Ctlr *c, int phyno, int reg, ushort val)
1244 {
1245         uint phy, i;
1246
1247         csr32w(c, Mdic, MDIwop | phyno<<MDIpSHIFT | reg<<MDIrSHIFT | val);
1248         phy = 0;
1249         for(i = 0; i < 64; i++){
1250                 phy = csr32r(c, Mdic);
1251                 if(phy & (MDIe|MDIready))
1252                         break;
1253                 microdelay(1);
1254         }
1255         if((phy & (MDIe|MDIready)) != MDIready)
1256                 return ~0;
1257         return 0;
1258 }
1259
1260 static uint
1261 setpage(Ctlr *c, uint phyno, uint p, uint r)
1262 {
1263         uint pr;
1264
1265         if(c->type == i82563){
1266                 if(r >= 16 && r <= 28 && r != 22)
1267                         pr = Phypage;
1268                 else if(r == 30 || r == 31)
1269                         pr = Phyapage;
1270                 else
1271                         return 0;
1272                 return phywrite0(c, phyno, pr, p);
1273         }else if(p == 0)
1274                 return 0;
1275         return ~0;
1276 }
1277
1278 static uint
1279 phywrite(Ctlr *c, uint phyno, uint reg, ushort v)
1280 {
1281         if(setpage(c, phyno, reg>>8, reg & 0xff) == ~0)
1282                 panic("%s: bad phy reg %.4ux", cname(c), reg);
1283         return phywrite0(c, phyno, reg & 0xff, v);
1284 }
1285
1286 static void
1287 phyerrata(Ether *e, Ctlr *c)
1288 {
1289         if(e->mbps == 0)
1290                 if(c->phyerrata == 0){
1291                         c->phyerrata++;
1292                         phywrite(c, 1, Phyprst, Prst);  /* try a port reset */
1293                         print("%s: phy port reset\n", cname(c));
1294                 }
1295         else
1296                 c->phyerrata = 0;
1297 }
1298
1299 static void
1300 phyl79proc(void *v)
1301 {
1302         uint a, i, r, phy, phyno;
1303         Ctlr *c;
1304         Ether *e;
1305
1306         e = v;
1307         c = e->ctlr;
1308
1309         phyno = 1;
1310         if(c->type == i82579)
1311                 phyno = 2;
1312
1313         for(;;){
1314                 phy = phyread(c, phyno, Phystat);
1315                 if(phy == ~0)
1316                         goto next;
1317                 i = (phy>>8) & 3;
1318                 a = phy & Ans;
1319                 if(a){
1320                         r = phyread(c, phyno, Phyctl);
1321                         phywrite(c, phyno, Phyctl, r | Ran | Ean);
1322                 }
1323                 e->link = (phy & Link) != 0;
1324                 if(e->link == 0)
1325                         i = 3;
1326                 c->speeds[i]++;
1327                 e->mbps = speedtab[i];
1328 next:
1329                 c->lim = 0;
1330                 i82563im(c, Lsc);
1331                 c->lsleep++;
1332                 sleep(&c->lrendez, i82563lim, c);
1333         }
1334 }
1335
1336 static void
1337 phylproc(void *v)
1338 {
1339         uint a, i, phy;
1340         Ctlr *c;
1341         Ether *e;
1342
1343         e = v;
1344         c = e->ctlr;
1345
1346         if(c->type == i82573 && (phy = phyread(c, 1, Phyier)) != ~0)
1347                 phywrite(c, 1, Phyier, phy | Lscie | Ancie | Spdie | Panie);
1348         for(;;){
1349                 phy = phyread(c, 1, Physsr);
1350                 if(phy == ~0)
1351                         goto next;
1352                 i = (phy>>14) & 3;
1353                 switch(c->type){
1354                 default:
1355                         a = 0;
1356                         break;
1357                 case i82563:
1358                 case i82578:
1359                 case i82578m:
1360                 case i82583:
1361                         a = phyread(c, 1, Phyisr) & Ane;
1362                         break;
1363                 case i82571:
1364                 case i82572:
1365                 case i82575:
1366                 case i82576:
1367                         a = phyread(c, 1, Phylhr) & Anf;
1368                         i = (i-1) & 3;
1369                         break;
1370                 }
1371                 if(a)
1372                         phywrite(c, 1, Phyctl, phyread(c, 1, Phyctl) | Ran | Ean);
1373                 e->link = (phy & Rtlink) != 0;
1374                 if(e->link == 0)
1375                         i = 3;
1376                 c->speeds[i]++;
1377                 e->mbps = speedtab[i];
1378                 if(c->type == i82563)
1379                         phyerrata(e, c);
1380 next:
1381                 c->lim = 0;
1382                 i82563im(c, Lsc);
1383                 c->lsleep++;
1384                 sleep(&c->lrendez, i82563lim, c);
1385         }
1386 }
1387
1388 static void
1389 pcslproc(void *v)
1390 {
1391         uint i, phy;
1392         Ctlr *c;
1393         Ether *e;
1394
1395         e = v;
1396         c = e->ctlr;
1397
1398         for(;;){
1399                 phy = csr32r(c, Pcsstat);
1400                 e->link = phy & Linkok;
1401                 i = 3;
1402                 if(e->link)
1403                         i = (phy & 6) >> 1;
1404                 else if(phy & Anbad)
1405                         csr32w(c, Pcsctl, csr32r(c, Pcsctl) | Pan | Prestart);
1406                 c->speeds[i]++;
1407                 e->mbps = speedtab[i];
1408                 c->lim = 0;
1409                 i82563im(c, Lsc);
1410                 c->lsleep++;
1411                 sleep(&c->lrendez, i82563lim, c);
1412         }
1413 }
1414
1415 static void
1416 serdeslproc(void *v)
1417 {
1418         uint i, tx, rx;
1419         Ctlr *c;
1420         Ether *e;
1421
1422         e = v;
1423         c = e->ctlr;
1424
1425         for(;;){
1426                 rx = csr32r(c, Rxcw);
1427                 tx = csr32r(c, Txcw);
1428                 USED(tx);
1429                 e->link = (rx & 1<<31) != 0;
1430 //              e->link = (csr32r(c, Status) & Lu) != 0;
1431                 i = 3;
1432                 if(e->link)
1433                         i = 2;
1434                 c->speeds[i]++;
1435                 e->mbps = speedtab[i];
1436                 c->lim = 0;
1437                 i82563im(c, Lsc);
1438                 c->lsleep++;
1439                 sleep(&c->lrendez, i82563lim, c);
1440         }
1441 }
1442
1443 static void
1444 i82563attach(Ether *edev)
1445 {
1446         char name[KNAMELEN];
1447         int i;
1448         Block *bp;
1449         Ctlr *ctlr;
1450
1451         ctlr = edev->ctlr;
1452         qlock(&ctlr->alock);
1453         if(ctlr->alloc != nil){
1454                 qunlock(&ctlr->alock);
1455                 return;
1456         }
1457
1458         ctlr->nrd = Nrd;
1459         ctlr->ntd = Ntd;
1460         ctlr->alloc = malloc(ctlr->nrd*sizeof(Rd)+ctlr->ntd*sizeof(Td) + 255);
1461         if(ctlr->alloc == nil){
1462                 qunlock(&ctlr->alock);
1463                 error(Enomem);
1464         }
1465         ctlr->rdba = (Rd*)ROUNDUP((uintptr)ctlr->alloc, 256);
1466         ctlr->tdba = (Td*)(ctlr->rdba + ctlr->nrd);
1467
1468         ctlr->rb = malloc(ctlr->nrd * sizeof(Block*));
1469         ctlr->tb = malloc(ctlr->ntd * sizeof(Block*));
1470
1471         if(waserror()){
1472                 while(bp = i82563rballoc(rbtab + ctlr->pool)){
1473                         bp->free = nil;
1474                         freeb(bp);
1475                 }
1476                 free(ctlr->tb);
1477                 ctlr->tb = nil;
1478                 free(ctlr->rb);
1479                 ctlr->rb = nil;
1480                 free(ctlr->alloc);
1481                 ctlr->alloc = nil;
1482                 qunlock(&ctlr->alock);
1483                 nexterror();
1484         }
1485
1486         for(i = 0; i < Nrb; i++){
1487                 bp = allocb(ctlr->rbsz + Rbalign);
1488                 bp->free = freetab[ctlr->pool];
1489                 freeb(bp);
1490         }
1491
1492         snprint(name, sizeof name, "#l%dl", edev->ctlrno);
1493         if((csr32r(ctlr, Ctrlext) & Linkmode) == Serdes)
1494                 kproc(name, pcslproc, edev);            /* phy based serdes */
1495         else if(csr32r(ctlr, Status) & Tbimode)
1496                 kproc(name, serdeslproc, edev);         /* mac based serdes */
1497         else if(ctlr->type == i82579 || ctlr->type == i82580)
1498                 kproc(name, phyl79proc, edev);
1499         else
1500                 kproc(name, phylproc, edev);
1501
1502         snprint(name, sizeof name, "#l%dr", edev->ctlrno);
1503         kproc(name, i82563rproc, edev);
1504
1505         snprint(name, sizeof name, "#l%dt", edev->ctlrno);
1506         kproc(name, i82563tproc, edev);
1507
1508         qunlock(&ctlr->alock);
1509         poperror();
1510 }
1511
1512 static void
1513 i82563interrupt(Ureg*, void *arg)
1514 {
1515         Ctlr *ctlr;
1516         Ether *edev;
1517         int icr, im;
1518
1519         edev = arg;
1520         ctlr = edev->ctlr;
1521
1522         ilock(&ctlr->imlock);
1523         csr32w(ctlr, Imc, ~0);
1524         im = ctlr->im;
1525
1526         while(icr = csr32r(ctlr, Icr) & ctlr->im){
1527                 if(icr & Lsc){
1528                         im &= ~Lsc;
1529                         ctlr->lim = icr & Lsc;
1530                         wakeup(&ctlr->lrendez);
1531                         ctlr->lintr++;
1532                 }
1533                 if(icr & (Rxt0|Rxo|Rxdmt0|Rxseq|Ack)){
1534                         ctlr->rim = icr & (Rxt0|Rxo|Rxdmt0|Rxseq|Ack);
1535                         im &= ~(Rxt0|Rxo|Rxdmt0|Rxseq|Ack);
1536                         wakeup(&ctlr->rrendez);
1537                         ctlr->rintr++;
1538                 }
1539                 if(icr & Txdw){
1540                         im &= ~Txdw;
1541                         ctlr->tintr++;
1542                         wakeup(&ctlr->trendez);
1543                 }
1544         }
1545
1546         ctlr->im = im;
1547         csr32w(ctlr, Ims, im);
1548         iunlock(&ctlr->imlock);
1549 }
1550
1551 static int
1552 i82563detach(Ctlr *ctlr)
1553 {
1554         int r, timeo;
1555
1556         /* balance rx/tx packet buffer; survives reset */
1557         if(ctlr->rbsz > 8192 && cttab[ctlr->type].flag & Fpba){
1558                 ctlr->pba = csr32r(ctlr, Pba);
1559                 r = ctlr->pba >> 16;
1560                 r += ctlr->pba & 0xffff;
1561                 r >>= 1;
1562                 csr32w(ctlr, Pba, r);
1563         }else if(ctlr->type == i82573 && ctlr->rbsz > 1514)
1564                 csr32w(ctlr, Pba, 14);
1565         ctlr->pba = csr32r(ctlr, Pba);
1566
1567         /*
1568          * Perform a device reset to get the chip back to the
1569          * power-on state, followed by an EEPROM reset to read
1570          * the defaults for some internal registers.
1571          */
1572         csr32w(ctlr, Imc, ~0);
1573         csr32w(ctlr, Rctl, 0);
1574         csr32w(ctlr, Tctl, csr32r(ctlr, Tctl) & ~Ten);
1575
1576         delay(10);
1577
1578         r = csr32r(ctlr, Ctrl);
1579         if(ctlr->type == i82566 || ctlr->type == i82579)
1580                 r |= Phyrst;
1581         csr32w(ctlr, Ctrl, Devrst | r);
1582         delay(1);
1583         for(timeo = 0;; timeo++){
1584                 if((csr32r(ctlr, Ctrl) & (Devrst|Phyrst)) == 0)
1585                         break;
1586                 if(timeo >= 1000)
1587                         return -1;
1588                 delay(1);
1589         }
1590
1591         r = csr32r(ctlr, Ctrl);
1592         csr32w(ctlr, Ctrl, Slu|r);
1593
1594         r = csr32r(ctlr, Ctrlext);
1595         csr32w(ctlr, Ctrlext, r|Eerst);
1596         delay(1);
1597         for(timeo = 0; timeo < 1000; timeo++){
1598                 if(!(csr32r(ctlr, Ctrlext) & Eerst))
1599                         break;
1600                 delay(1);
1601         }
1602         if(csr32r(ctlr, Ctrlext) & Eerst)
1603                 return -1;
1604
1605         csr32w(ctlr, Imc, ~0);
1606         delay(1);
1607         for(timeo = 0; timeo < 1000; timeo++){
1608                 if((csr32r(ctlr, Icr) & ~Rxcfg) == 0)
1609                         break;
1610                 delay(1);
1611         }
1612         if(csr32r(ctlr, Icr) & ~Rxcfg)
1613                 return -1;
1614
1615         return 0;
1616 }
1617
1618 static void
1619 i82563shutdown(Ether *edev)
1620 {
1621         i82563detach(edev->ctlr);
1622 }
1623
1624 static ushort
1625 eeread(Ctlr *ctlr, int adr)
1626 {
1627         csr32w(ctlr, Eerd, EEstart | adr << 2);
1628         while ((csr32r(ctlr, Eerd) & EEdone) == 0)
1629                 ;
1630         return csr32r(ctlr, Eerd) >> 16;
1631 }
1632
1633 static int
1634 eeload(Ctlr *ctlr)
1635 {
1636         ushort sum;
1637         int data, adr;
1638
1639         sum = 0;
1640         for (adr = 0; adr < 0x40; adr++) {
1641                 data = eeread(ctlr, adr);
1642                 ctlr->eeprom[adr] = data;
1643                 sum += data;
1644         }
1645         return sum;
1646 }
1647
1648 static int
1649 fcycle(Ctlr *, Flash *f)
1650 {
1651         ushort s, i;
1652
1653         s = f->reg[Fsts];
1654         if((s&Fvalid) == 0)
1655                 return -1;
1656         f->reg[Fsts] |= Fcerr | Ael;
1657         for(i = 0; i < 10; i++){
1658                 if((s&Scip) == 0)
1659                         return 0;
1660                 delay(1);
1661                 s = f->reg[Fsts];
1662         }
1663         return -1;
1664 }
1665
1666 static int
1667 fread(Ctlr *c, Flash *f, int ladr)
1668 {
1669         ushort s;
1670
1671         delay(1);
1672         if(fcycle(c, f) == -1)
1673                 return -1;
1674         f->reg[Fsts] |= Fdone;
1675         f->reg32[Faddr] = ladr;
1676
1677         /* setup flash control register */
1678         s = f->reg[Fctl] & ~0x3ff;
1679         f->reg[Fctl] = s | 1<<8 | Fgo;  /* 2 byte read */
1680
1681         while((f->reg[Fsts] & Fdone) == 0)
1682                 ;
1683         if(f->reg[Fsts] & (Fcerr|Ael))
1684                 return -1;
1685         return f->reg32[Fdata] & 0xffff;
1686 }
1687
1688 static int
1689 fload(Ctlr *c)
1690 {
1691         ulong data, io, r, adr;
1692         ushort sum;
1693         Flash f;
1694
1695         io = c->pcidev->mem[1].bar & ~0x0f;
1696         f.reg = vmap(io, c->pcidev->mem[1].size);
1697         if(f.reg == nil)
1698                 return -1;
1699         f.reg32 = (ulong*)f.reg;
1700         f.sz = f.reg32[Bfpr];
1701         if(csr32r(c, Eec) & 1<<22){
1702                 if(c->type == i82579)
1703                         f.sz  += 16;            /* sector size: 64k */
1704                 else
1705                         f.sz  += 1;             /* sector size: 4k */
1706         }
1707         r = (f.sz & 0x1fff) << 12;
1708         sum = 0;
1709         for(adr = 0; adr < 0x40; adr++) {
1710                 data = fread(c, &f, r + adr*2);
1711                 if(data == -1)
1712                         return -1;
1713                 c->eeprom[adr] = data;
1714                 sum += data;
1715         }
1716         vunmap(f.reg, c->pcidev->mem[1].size);
1717         return sum;
1718 }
1719
1720 static void
1721 defaultea(Ctlr *ctlr, uchar *ra)
1722 {
1723         uint i, r;
1724         uvlong u;
1725         static uchar nilea[Eaddrlen];
1726
1727         if(memcmp(ra, nilea, Eaddrlen) != 0)
1728                 return;
1729         if(cttab[ctlr->type].flag & Fflashea){
1730                 /* intel mb bug */
1731                 u = (uvlong)csr32r(ctlr, Rah)<<32u | (ulong)csr32r(ctlr, Ral);
1732                 for(i = 0; i < Eaddrlen; i++)
1733                         ra[i] = u >> 8*i;
1734         }
1735         if(memcmp(ra, nilea, Eaddrlen) != 0)
1736                 return;
1737         for(i = 0; i < Eaddrlen/2; i++){
1738                 ra[2*i] = ctlr->eeprom[Ea+i];
1739                 ra[2*i+1] = ctlr->eeprom[Ea+i] >> 8;
1740         }
1741         r = (csr32r(ctlr, Status) & Lanid) >> 2;
1742         ra[5] += r;                             /* ea ctlr[n] = ea ctlr[0]+n */
1743 }
1744
1745 static int
1746 i82563reset(Ctlr *ctlr)
1747 {
1748         uchar *ra;
1749         int i, r;
1750
1751         if(i82563detach(ctlr))
1752                 return -1;
1753         if(cttab[ctlr->type].flag & Fload)
1754                 r = fload(ctlr);
1755         else
1756                 r = eeload(ctlr);
1757         if(r != 0 && r != 0xbaba){
1758                 print("%s: bad eeprom checksum - %#.4ux\n",
1759                         cname(ctlr), r);
1760                 return -1;
1761         }
1762
1763         ra = ctlr->ra;
1764         defaultea(ctlr, ra);
1765         csr32w(ctlr, Ral, ra[3]<<24 | ra[2]<<16 | ra[1]<<8 | ra[0]);
1766         csr32w(ctlr, Rah, 1<<31 | ra[5]<<8 | ra[4]);
1767         for(i = 1; i < 16; i++){
1768                 csr32w(ctlr, Ral+i*8, 0);
1769                 csr32w(ctlr, Rah+i*8, 0);
1770         }
1771         memset(ctlr->mta, 0, sizeof(ctlr->mta));
1772         for(i = 0; i < 128; i++)
1773                 csr32w(ctlr, Mta + i*4, 0);
1774         csr32w(ctlr, Fcal, 0x00C28001);
1775         csr32w(ctlr, Fcah, 0x0100);
1776         if(ctlr->type != i82579)
1777                 csr32w(ctlr, Fct, 0x8808);
1778         csr32w(ctlr, Fcttv, 0x0100);
1779         csr32w(ctlr, Fcrtl, ctlr->fcrtl);
1780         csr32w(ctlr, Fcrth, ctlr->fcrth);
1781         if(cttab[ctlr->type].flag & F75)
1782                 csr32w(ctlr, Eitr, 128<<2);             /* 128 ¼ microsecond intervals */
1783         return 0;
1784 }
1785
1786 enum {
1787         CMrdtr,
1788         CMradv,
1789         CMpause,
1790         CMan,
1791 };
1792
1793 static Cmdtab i82563ctlmsg[] = {
1794         CMrdtr, "rdtr", 2,
1795         CMradv, "radv", 2,
1796         CMpause, "pause", 1,
1797         CMan,   "an",   1,
1798 };
1799
1800 static long
1801 i82563ctl(Ether *edev, void *buf, long n)
1802 {
1803         char *p;
1804         ulong v;
1805         Ctlr *ctlr;
1806         Cmdbuf *cb;
1807         Cmdtab *ct;
1808
1809         if((ctlr = edev->ctlr) == nil)
1810                 error(Enonexist);
1811
1812         cb = parsecmd(buf, n);
1813         if(waserror()){
1814                 free(cb);
1815                 nexterror();
1816         }
1817
1818         ct = lookupcmd(cb, i82563ctlmsg, nelem(i82563ctlmsg));
1819         switch(ct->index){
1820         case CMrdtr:
1821                 v = strtoul(cb->f[1], &p, 0);
1822                 if(*p || v > 0xffff)
1823                         error(Ebadarg);
1824                 ctlr->rdtr = v;
1825                 csr32w(ctlr, Rdtr, v);
1826                 break;
1827         case CMradv:
1828                 v = strtoul(cb->f[1], &p, 0);
1829                 if(*p || v > 0xffff)
1830                         error(Ebadarg);
1831                 ctlr->radv = v;
1832                 csr32w(ctlr, Radv, v);
1833                 break;
1834         case CMpause:
1835                 csr32w(ctlr, Ctrl, csr32r(ctlr, Ctrl) ^ (1<<27 | 1<<28));
1836                 break;
1837         case CMan:
1838                 csr32w(ctlr, Ctrl, csr32r(ctlr, Ctrl) | Lrst | Phyrst);
1839                 break;
1840         }
1841         free(cb);
1842         poperror();
1843
1844         return n;
1845 }
1846
1847 static int
1848 didtype(int d)
1849 {
1850         switch(d){
1851         case 0x1096:
1852         case 0x10ba:            /* “gilgal” */
1853         // case 0x1098:         /* serdes; not seen */
1854         // case 0x10bb:         /* serdes */
1855                 return i82563;
1856         case 0x1049:            /* mm */
1857         case 0x104a:            /* dm */
1858         case 0x104b:            /* dc */
1859         case 0x104d:            /* v “ninevah” */
1860         case 0x10bd:            /* dm-2 */
1861         case 0x294c:            /* ich 9 */
1862                 return i82566;
1863         case 0x10de:            /* lm ich10d */
1864         case 0x10df:            /* lf ich10 */
1865         case 0x10e5:            /* lm ich9 */
1866         case 0x10f5:            /* lm ich9m; “boazman” */
1867                 return i82567;
1868         case 0x10bf:            /* lf ich9m */
1869         case 0x10cb:            /* v ich9m */
1870         case 0x10cd:            /* lf ich10 */
1871         case 0x10ce:            /* v ich10 */
1872         case 0x10cc:            /* lm ich10 */
1873                 return i82567m;
1874         case 0x105e:            /* eb */
1875         case 0x105f:            /* eb */
1876         case 0x1060:            /* eb */
1877         case 0x10a4:            /* eb */
1878         case 0x10a5:            /* eb  fiber */
1879         case 0x10bc:            /* eb */
1880         case 0x10d9:            /* eb serdes */
1881         case 0x10da:            /* eb serdes “ophir” */
1882                 return i82571;
1883         case 0x107d:            /* eb copper */
1884         case 0x107e:            /* ei fiber */
1885         case 0x107f:            /* ei */
1886         case 0x10b9:            /* ei “rimon” */
1887                 return i82572;
1888         case 0x108b:            /*  e “vidalia” */
1889         case 0x108c:            /*  e (iamt) */
1890         case 0x109a:            /*  l “tekoa” */
1891                 return i82573;
1892         case 0x10d3:            /* l or it; “hartwell” */
1893                 return i82574;
1894         case 0x10a7:
1895         case 0x10a9:            /* fiber/serdes */
1896                 return i82575;
1897         case 0x10c9:            /* copper */
1898         case 0x10e6:            /* fiber */
1899         case 0x10e7:            /* serdes; “kawela” */
1900                 return i82576;
1901         case 0x10ea:            /* lc “calpella”; aka pch lan */
1902                 return i82577;
1903         case 0x10eb:            /* lm “calpella” */
1904                 return i82577m;
1905         case 0x10ef:            /* dc “piketon” */
1906                 return i82578;
1907         case 0x1502:            /* lm */
1908         case 0x1503:            /* v */
1909                 return i82579;
1910         case 0x10f0:            /* dm “king's creek” */
1911                 return i82578m;
1912         case 0x150e:            /* “barton hills” */
1913         case 0x150f:            /* fiber */
1914         case 0x1510:            /* backplane */
1915         case 0x1511:            /* sfp */
1916         case 0x1516:
1917                 return i82580;
1918         case 0x1506:            /* v */
1919                 return i82583;
1920         }
1921         return -1;
1922 }
1923
1924 static void
1925 hbafixup(Pcidev *p)
1926 {
1927         uint i;
1928
1929         i = pcicfgr32(p, PciSVID);
1930         if((i & 0xffff) == 0x1b52 && p->did == 1)
1931                 p->did = i>>16;
1932 }
1933
1934 static void
1935 i82563pci(void)
1936 {
1937         int type;
1938         Ctlr *ctlr;
1939         Pcidev *p;
1940
1941         for(p = nil; p = pcimatch(p, 0x8086, 0);){
1942                 hbafixup(p);
1943                 if((type = didtype(p->did)) == -1)
1944                         continue;
1945                 ctlr = malloc(sizeof(Ctlr));
1946                 ctlr->type = type;
1947                 ctlr->pcidev = p;
1948                 ctlr->rbsz = cttab[type].mtu;
1949                 ctlr->port = p->mem[0].bar & ~0x0F;
1950                 if(i82563ctlrhead != nil)
1951                         i82563ctlrtail->next = ctlr;
1952                 else
1953                         i82563ctlrhead = ctlr;
1954                 i82563ctlrtail = ctlr;
1955         }
1956 }
1957
1958 static int
1959 setup(Ctlr *ctlr)
1960 {
1961         Pcidev *p;
1962
1963         if((ctlr->pool = newpool()) == -1){
1964                 print("%s: no pool\n", cname(ctlr));
1965                 return -1;
1966         }
1967         p = ctlr->pcidev;
1968         ctlr->nic = vmap(ctlr->port, p->mem[0].size);
1969         if(ctlr->nic == nil){
1970                 print("%s: can't map %#p\n", cname(ctlr), ctlr->port);
1971                 return -1;
1972         }
1973         if(i82563reset(ctlr)){
1974                 vunmap(ctlr->nic, p->mem[0].size);
1975                 return -1;
1976         }
1977         pcisetbme(ctlr->pcidev);
1978         return 0;
1979 }
1980
1981 static int
1982 pnp(Ether *edev, int type)
1983 {
1984         Ctlr *ctlr;
1985         static int done;
1986
1987         if(!done) {
1988                 i82563pci();
1989                 done = 1;
1990         }
1991
1992         /*
1993          * Any adapter matches if no edev->port is supplied,
1994          * otherwise the ports must match.
1995          */
1996         for(ctlr = i82563ctlrhead; ; ctlr = ctlr->next){
1997                 if(ctlr == nil)
1998                         return -1;
1999                 if(ctlr->active)
2000                         continue;
2001                 if(type != -1 && ctlr->type != type)
2002                         continue;
2003                 if(edev->port == 0 || edev->port == ctlr->port){
2004                         ctlr->active = 1;
2005                         memmove(ctlr->ra, edev->ea, Eaddrlen);
2006                         if(setup(ctlr) == 0)
2007                                 break;
2008                 }
2009         }
2010
2011         edev->ctlr = ctlr;
2012         edev->port = ctlr->port;
2013         edev->irq = ctlr->pcidev->intl;
2014         edev->tbdf = ctlr->pcidev->tbdf;
2015         edev->mbps = 1000;
2016         edev->maxmtu = ctlr->rbsz;
2017         memmove(edev->ea, ctlr->ra, Eaddrlen);
2018
2019         /*
2020          * Linkage to the generic ethernet driver.
2021          */
2022         edev->attach = i82563attach;
2023 //      edev->transmit = i82563transmit;
2024         edev->interrupt = i82563interrupt;
2025         edev->ifstat = i82563ifstat;
2026         edev->ctl = i82563ctl;
2027
2028         edev->arg = edev;
2029         edev->promiscuous = i82563promiscuous;
2030         edev->shutdown = i82563shutdown;
2031         edev->multicast = i82563multicast;
2032
2033         return 0;
2034 }
2035
2036 static int
2037 anypnp(Ether *e)
2038 {
2039         return pnp(e, -1);
2040 }
2041
2042 static int
2043 i82563pnp(Ether *e)
2044 {
2045         return pnp(e, i82563);
2046 }
2047
2048 static int
2049 i82566pnp(Ether *e)
2050 {
2051         return pnp(e, i82566);
2052 }
2053
2054 static int
2055 i82567pnp(Ether *e)
2056 {
2057         return pnp(e, i82567m) & pnp(e, i82567);
2058 }
2059
2060 static int
2061 i82571pnp(Ether *e)
2062 {
2063         return pnp(e, i82571);
2064 }
2065
2066 static int
2067 i82572pnp(Ether *e)
2068 {
2069         return pnp(e, i82572);
2070 }
2071
2072 static int
2073 i82573pnp(Ether *e)
2074 {
2075         return pnp(e, i82573);
2076 }
2077
2078 static int
2079 i82574pnp(Ether *e)
2080 {
2081         return pnp(e, i82574);
2082 }
2083
2084 static int
2085 i82575pnp(Ether *e)
2086 {
2087         return pnp(e, i82575);
2088 }
2089
2090 static int
2091 i82576pnp(Ether *e)
2092 {
2093         return pnp(e, i82576);
2094 }
2095
2096 static int
2097 i82577pnp(Ether *e)
2098 {
2099         return pnp(e, i82577m) & pnp(e, i82577);
2100 }
2101
2102 static int
2103 i82578pnp(Ether *e)
2104 {
2105         return pnp(e, i82578m) & pnp(e, i82578);
2106 }
2107
2108 static int
2109 i82579pnp(Ether *e)
2110 {
2111         return pnp(e, i82579);
2112 }
2113
2114 static int
2115 i82580pnp(Ether *e)
2116 {
2117         return pnp(e, i82580);
2118 }
2119
2120 static int
2121 i82583pnp(Ether *e)
2122 {
2123         return pnp(e, i82583);
2124 }
2125
2126 void
2127 ether82563link(void)
2128 {
2129         /*
2130          * recognise lots of model numbers for debugging
2131          * also good for forcing onboard nic(s) as ether0
2132          * try to make that unnecessary by listing lom first.
2133          */
2134         addethercard("i82563", i82563pnp);
2135         addethercard("i82566", i82566pnp);
2136         addethercard("i82574", i82574pnp);
2137         addethercard("i82576", i82576pnp);
2138         addethercard("i82567", i82567pnp);
2139         addethercard("i82573", i82573pnp);
2140
2141         addethercard("i82571", i82571pnp);
2142         addethercard("i82572", i82572pnp);
2143         addethercard("i82575", i82575pnp);
2144         addethercard("i82577", i82577pnp);
2145         addethercard("i82578", i82578pnp);
2146         addethercard("i82579", i82579pnp);
2147         addethercard("i82580", i82580pnp);
2148         addethercard("i82583", i82583pnp);
2149         addethercard("igbepcie", anypnp);
2150 }