sys/src/9/pc/ether82563.c

   1 /*
   2  * Intel 8256[367], 8257[1-9], 8258[03]
   3  *      Gigabit Ethernet PCI-Express Controllers
   4  * Coraid EtherDrive® hba
   5  */
   6 #include "u.h"
   7 #include "../port/lib.h"
   8 #include "mem.h"
   9 #include "dat.h"
  10 #include "fns.h"
  11 #include "io.h"
  12 #include "../port/error.h"
  13 #include "../port/netif.h"
  14
  15 #include "etherif.h"
  16
  17 /*
  18  * note: the 82575, 82576 and 82580 are operated using registers aliased
  19  * to the 82563-style architecture.  many features seen in the 82598
  20  * are also seen in the 82575 part.
  21  */
  22
  23 enum {
  24         /* General */
  25
  26         Ctrl            = 0x0000,       /* Device Control */
  27         Status          = 0x0008,       /* Device Status */
  28         Eec             = 0x0010,       /* EEPROM/Flash Control/Data */
  29         Eerd            = 0x0014,       /* EEPROM Read */
  30         Ctrlext         = 0x0018,       /* Extended Device Control */
  31         Fla             = 0x001c,       /* Flash Access */
  32         Mdic            = 0x0020,       /* MDI Control */
  33         Fcal            = 0x0028,       /* Flow Control Address Low */
  34         Fcah            = 0x002C,       /* Flow Control Address High */
  35         Fct             = 0x0030,       /* Flow Control Type */
  36         Kumctrlsta      = 0x0034,       /* Kumeran Control and Status Register */
  37         Vet             = 0x0038,       /* VLAN EtherType */
  38         Fcttv           = 0x0170,       /* Flow Control Transmit Timer Value */
  39         Txcw            = 0x0178,       /* Transmit Configuration Word */
  40         Rxcw            = 0x0180,       /* Receive Configuration Word */
  41         Ledctl          = 0x0E00,       /* LED control */
  42         Pba             = 0x1000,       /* Packet Buffer Allocation */
  43         Pbs             = 0x1008,       /* Packet Buffer Size */
  44
  45         /* Interrupt */
  46
  47         Icr             = 0x00C0,       /* Interrupt Cause Read */
  48         Itr             = 0x00c4,       /* Interrupt Throttling Rate */
  49         Ics             = 0x00C8,       /* Interrupt Cause Set */
  50         Ims             = 0x00D0,       /* Interrupt Mask Set/Read */
  51         Imc             = 0x00D8,       /* Interrupt mask Clear */
  52         Iam             = 0x00E0,       /* Interrupt acknowledge Auto Mask */
  53         Eitr            = 0x1680,       /* Extended itr; 82575/6 80 only */
  54
  55         /* Receive */
  56
  57         Rctl            = 0x0100,       /* Control */
  58         Ert             = 0x2008,       /* Early Receive Threshold (573[EVL], 82578 only) */
  59         Fcrtl           = 0x2160,       /* Flow Control RX Threshold Low */
  60         Fcrth           = 0x2168,       /* Flow Control Rx Threshold High */
  61         Psrctl          = 0x2170,       /* Packet Split Receive Control */
  62         Drxmxod         = 0x2540,       /* dma max outstanding bytes (82575) */
  63         Rdbal           = 0x2800,       /* Rdesc Base Address Low Queue 0 */
  64         Rdbah           = 0x2804,       /* Rdesc Base Address High Queue 0 */
  65         Rdlen           = 0x2808,       /* Descriptor Length Queue 0 */
  66         Srrctl          = 0x280c,       /* split and replication rx control (82575) */
  67         Rdh             = 0x2810,       /* Descriptor Head Queue 0 */
  68         Rdt             = 0x2818,       /* Descriptor Tail Queue 0 */
  69         Rdtr            = 0x2820,       /* Descriptor Timer Ring */
  70         Rxdctl          = 0x2828,       /* Descriptor Control */
  71         Radv            = 0x282C,       /* Interrupt Absolute Delay Timer */
  72         Rsrpd           = 0x2c00,       /* Small Packet Detect */
  73         Raid            = 0x2c08,       /* ACK interrupt delay */
  74         Cpuvec          = 0x2c10,       /* CPU Vector */
  75         Rxcsum          = 0x5000,       /* Checksum Control */
  76         Rmpl            = 0x5004,       /* rx maximum packet length (82575) */
  77         Rfctl           = 0x5008,       /* Filter Control */
  78         Mta             = 0x5200,       /* Multicast Table Array */
  79         Ral             = 0x5400,       /* Receive Address Low */
  80         Rah             = 0x5404,       /* Receive Address High */
  81         Vfta            = 0x5600,       /* VLAN Filter Table Array */
  82         Mrqc            = 0x5818,       /* Multiple Receive Queues Command */
  83
  84         /* Transmit */
  85
  86         Tctl            = 0x0400,       /* Transmit Control */
  87         Tipg            = 0x0410,       /* Transmit IPG */
  88         Tkabgtxd        = 0x3004,       /* glci afe band gap transmit ref data, or something */
  89         Tdbal           = 0x3800,       /* Tdesc Base Address Low */
  90         Tdbah           = 0x3804,       /* Tdesc Base Address High */
  91         Tdlen           = 0x3808,       /* Descriptor Length */
  92         Tdh             = 0x3810,       /* Descriptor Head */
  93         Tdt             = 0x3818,       /* Descriptor Tail */
  94         Tidv            = 0x3820,       /* Interrupt Delay Value */
  95         Txdctl          = 0x3828,       /* Descriptor Control */
  96         Tadv            = 0x382C,       /* Interrupt Absolute Delay Timer */
  97         Tarc0           = 0x3840,       /* Arbitration Counter Queue 0 */
  98
  99         /* Statistics */
 100
 101         Statistics      = 0x4000,       /* Start of Statistics Area */
 102         Gorcl           = 0x88/4,       /* Good Octets Received Count */
 103         Gotcl           = 0x90/4,       /* Good Octets Transmitted Count */
 104         Torl            = 0xC0/4,       /* Total Octets Received */
 105         Totl            = 0xC8/4,       /* Total Octets Transmitted */
 106         Nstatistics     = 0x124/4,
 107 };
 108
 109 enum {                                  /* Ctrl */
 110         Lrst            = 1<<3,         /* link reset */
 111         Slu             = 1<<6,         /* Set Link Up */
 112         Devrst          = 1<<26,        /* Device Reset */
 113         Rfce            = 1<<27,        /* Receive Flow Control Enable */
 114         Tfce            = 1<<28,        /* Transmit Flow Control Enable */
 115         Phyrst          = 1<<31,        /* Phy Reset */
 116 };
 117
 118 enum {                                  /* Status */
 119         Lu              = 1<<1,         /* Link Up */
 120         Lanid           = 3<<2,         /* mask for Lan ID. */
 121         Txoff           = 1<<4,         /* Transmission Paused */
 122         Tbimode         = 1<<5,         /* TBI Mode Indication */
 123         Phyra           = 1<<10,        /* PHY Reset Asserted */
 124         GIOme           = 1<<19,        /* GIO Master Enable Status */
 125 };
 126
 127 enum {                                  /* Eerd */
 128         EEstart         = 1<<0,         /* Start Read */
 129         EEdone          = 1<<1,         /* Read done */
 130 };
 131
 132 enum {                                  /* Ctrlext */
 133         Eerst           = 1<<13,        /* EEPROM Reset */
 134         Linkmode        = 3<<23,        /* linkmode */
 135         Serdes          = 3<<23,        /* " serdes */
 136 };
 137
 138 enum {                                  /* EEPROM content offsets */
 139         Ea              = 0x00,         /* Ethernet Address */
 140 };
 141
 142 enum {                                  /* Mdic */
 143         MDIdMASK        = 0x0000FFFF,   /* Data */
 144         MDIdSHIFT       = 0,
 145         MDIrMASK        = 0x001F0000,   /* PHY Register Address */
 146         MDIrSHIFT       = 16,
 147         MDIpMASK        = 0x03E00000,   /* PHY Address */
 148         MDIpSHIFT       = 21,
 149         MDIwop          = 0x04000000,   /* Write Operation */
 150         MDIrop          = 0x08000000,   /* Read Operation */
 151         MDIready        = 0x10000000,   /* End of Transaction */
 152         MDIie           = 0x20000000,   /* Interrupt Enable */
 153         MDIe            = 0x40000000,   /* Error */
 154 };
 155
 156 enum {                                  /* phy interface */
 157         Phyctl          = 0,            /* phy ctl register */
 158         Phyisr          = 19,           /* 82563 phy interrupt status register */
 159         Phylhr          = 19,           /* 8257[12] link health register */
 160         Physsr          = 17,           /* phy secondary status register */
 161         Phyprst         = 193<<8 | 17,  /* 8256[34] phy port reset */
 162         Phyier          = 18,           /* 82573 phy interrupt enable register */
 163         Phypage         = 22,           /* 8256[34] page register */
 164         Phystat         = 26,           /* 82580 phy status */
 165         Phyapage        = 29,
 166         Rtlink          = 1<<10,        /* realtime link status */
 167         Phyan           = 1<<11,        /* phy has autonegotiated */
 168
 169         /* Phyctl bits */
 170         Ran             = 1<<9, /* restart auto negotiation */
 171         Ean             = 1<<12,        /* enable auto negotiation */
 172
 173         /* Phyprst bits */
 174         Prst            = 1<<0, /* reset the port */
 175
 176         /* 82573 Phyier bits */
 177         Lscie           = 1<<10,        /* link status changed ie */
 178         Ancie           = 1<<11,        /* auto negotiation complete ie */
 179         Spdie           = 1<<14,        /* speed changed ie */
 180         Panie           = 1<<15,        /* phy auto negotiation error ie */
 181
 182         /* Phylhr/Phyisr bits */
 183         Anf             = 1<<6, /* lhr: auto negotiation fault */
 184         Ane             = 1<<15,        /* isr: auto negotiation error */
 185
 186         /* 82580 Phystat bits */
 187         Ans     = 1<<14 | 1<<15,        /* 82580 autoneg. status */
 188         Link    = 1<<6,         /* 82580 Link */
 189
 190         /* Rxcw builtin serdes */
 191         Anc             = 1<<31,
 192         Rxsynch         = 1<<30,
 193         Rxcfg           = 1<<29,
 194         Rxcfgch         = 1<<28,
 195         Rxcfgbad        = 1<<27,
 196         Rxnc            = 1<<26,
 197
 198         /* Txcw */
 199         Txane           = 1<<31,
 200         Txcfg           = 1<<30,
 201 };
 202
 203 enum {                                  /* fiber (pcs) interface */
 204         Pcsctl  = 0x4208,               /* pcs control */
 205         Pcsstat = 0x420c,               /* pcs status */
 206
 207         /* Pcsctl bits */
 208         Pan     = 1<<16,                /* autoegotiate */
 209         Prestart        = 1<<17,                /* restart an (self clearing) */
 210
 211         /* Pcsstat bits */
 212         Linkok  = 1<<0,         /* link is okay */
 213         Andone  = 1<<16,                /* an phase is done see below for success */
 214         Anbad   = 1<<19 | 1<<20,        /* Anerror | Anremfault */
 215 };
 216
 217 enum {                                  /* Icr, Ics, Ims, Imc */
 218         Txdw            = 0x00000001,   /* Transmit Descriptor Written Back */
 219         Txqe            = 0x00000002,   /* Transmit Queue Empty */
 220         Lsc             = 0x00000004,   /* Link Status Change */
 221         Rxseq           = 0x00000008,   /* Receive Sequence Error */
 222         Rxdmt0          = 0x00000010,   /* Rdesc Minimum Threshold Reached */
 223         Rxo             = 0x00000040,   /* Receiver Overrun */
 224         Rxt0            = 0x00000080,   /* Receiver Timer Interrupt; !82575/6/80 only */
 225         Rxdw            = 0x00000080,   /* Rdesc write back; 82575/6/80 only */
 226         Mdac            = 0x00000200,   /* MDIO Access Completed */
 227         Rxcfgset                = 0x00000400,   /* Receiving /C/ ordered sets */
 228         Ack             = 0x00020000,   /* Receive ACK frame */
 229 };
 230
 231 enum {                                  /* Txcw */
 232         TxcwFd          = 0x00000020,   /* Full Duplex */
 233         TxcwHd          = 0x00000040,   /* Half Duplex */
 234         TxcwPauseMASK   = 0x00000180,   /* Pause */
 235         TxcwPauseSHIFT  = 7,
 236         TxcwPs          = 1<<TxcwPauseSHIFT,    /* Pause Supported */
 237         TxcwAs          = 2<<TxcwPauseSHIFT,    /* Asymmetric FC desired */
 238         TxcwRfiMASK     = 0x00003000,   /* Remote Fault Indication */
 239         TxcwRfiSHIFT    = 12,
 240         TxcwNpr         = 0x00008000,   /* Next Page Request */
 241         TxcwConfig      = 0x40000000,   /* Transmit COnfig Control */
 242         TxcwAne         = 0x80000000,   /* Auto-Negotiation Enable */
 243 };
 244
 245 enum {                                  /* Rctl */
 246         Rrst            = 0x00000001,   /* Receiver Software Reset */
 247         Ren             = 0x00000002,   /* Receiver Enable */
 248         Sbp             = 0x00000004,   /* Store Bad Packets */
 249         Upe             = 0x00000008,   /* Unicast Promiscuous Enable */
 250         Mpe             = 0x00000010,   /* Multicast Promiscuous Enable */
 251         Lpe             = 0x00000020,   /* Long Packet Reception Enable */
 252         RdtmsMASK       = 0x00000300,   /* Rdesc Minimum Threshold Size */
 253         RdtmsHALF       = 0x00000000,   /* Threshold is 1/2 Rdlen */
 254         RdtmsQUARTER    = 0x00000100,   /* Threshold is 1/4 Rdlen */
 255         RdtmsEIGHTH     = 0x00000200,   /* Threshold is 1/8 Rdlen */
 256         MoMASK          = 0x00003000,   /* Multicast Offset */
 257         Bam             = 0x00008000,   /* Broadcast Accept Mode */
 258         BsizeMASK       = 0x00030000,   /* Receive Buffer Size */
 259         Bsize16384      = 0x00010000,   /* Bsex = 1 */
 260         Bsize8192       = 0x00020000,   /* Bsex = 1 */
 261         Bsize2048       = 0x00000000,
 262         Bsize1024       = 0x00010000,
 263         Bsize512        = 0x00020000,
 264         Bsize256        = 0x00030000,
 265         BsizeFlex       = 0x08000000,   /* Flexable Bsize in 1kb increments */
 266         Vfe             = 0x00040000,   /* VLAN Filter Enable */
 267         Cfien           = 0x00080000,   /* Canonical Form Indicator Enable */
 268         Cfi             = 0x00100000,   /* Canonical Form Indicator value */
 269         Dpf             = 0x00400000,   /* Discard Pause Frames */
 270         Pmcf            = 0x00800000,   /* Pass MAC Control Frames */
 271         Bsex            = 0x02000000,   /* Buffer Size Extension */
 272         Secrc           = 0x04000000,   /* Strip CRC from incoming packet */
 273 };
 274
 275 enum {                                  /* Srrctl */
 276         Dropen          = 1<<31,
 277 };
 278
 279 enum {                                  /* Tctl */
 280         Trst            = 0x00000001,   /* Transmitter Software Reset */
 281         Ten             = 0x00000002,   /* Transmit Enable */
 282         Psp             = 0x00000008,   /* Pad Short Packets */
 283         Mulr            = 0x10000000,   /* Allow multiple concurrent requests */
 284         CtMASK          = 0x00000FF0,   /* Collision Threshold */
 285         CtSHIFT         = 4,
 286         ColdMASK        = 0x003FF000,   /* Collision Distance */
 287         ColdSHIFT       = 12,
 288         Swxoff          = 0x00400000,   /* Sofware XOFF Transmission */
 289         Pbe             = 0x00800000,   /* Packet Burst Enable */
 290         Rtlc            = 0x01000000,   /* Re-transmit on Late Collision */
 291         Nrtu            = 0x02000000,   /* No Re-transmit on Underrrun */
 292 };
 293
 294 enum {                                  /* [RT]xdctl */
 295         PthreshMASK     = 0x0000003F,   /* Prefetch Threshold */
 296         PthreshSHIFT    = 0,
 297         HthreshMASK     = 0x00003F00,   /* Host Threshold */
 298         HthreshSHIFT    = 8,
 299         WthreshMASK     = 0x003F0000,   /* Writeback Threshold */
 300         WthreshSHIFT    = 16,
 301         Gran            = 0x01000000,   /* Granularity; not 82575 */
 302         Enable          = 0x02000000,
 303 };
 304
 305 enum {                                  /* Rxcsum */
 306         Ipofl           = 0x0100,       /* IP Checksum Off-load Enable */
 307         Tuofl           = 0x0200,       /* TCP/UDP Checksum Off-load Enable */
 308 };
 309
 310 typedef struct Rd {                     /* Receive Descriptor */
 311         uint    addr[2];
 312         ushort  length;
 313         ushort  checksum;
 314         uchar   status;
 315         uchar   errors;
 316         ushort  special;
 317 } Rd;
 318
 319 enum {                                  /* Rd status */
 320         Rdd             = 0x01,         /* Descriptor Done */
 321         Reop            = 0x02,         /* End of Packet */
 322         Ixsm            = 0x04,         /* Ignore Checksum Indication */
 323         Vp              = 0x08,         /* Packet is 802.1Q (matched VET) */
 324         Tcpcs           = 0x20,         /* TCP Checksum Calculated on Packet */
 325         Ipcs            = 0x40,         /* IP Checksum Calculated on Packet */
 326         Pif             = 0x80,         /* Passed in-exact filter */
 327 };
 328
 329 enum {                                  /* Rd errors */
 330         Ce              = 0x01,         /* CRC Error or Alignment Error */
 331         Se              = 0x02,         /* Symbol Error */
 332         Seq             = 0x04,         /* Sequence Error */
 333         Cxe             = 0x10,         /* Carrier Extension Error */
 334         Tcpe            = 0x20,         /* TCP/UDP Checksum Error */
 335         Ipe             = 0x40,         /* IP Checksum Error */
 336         Rxe             = 0x80,         /* RX Data Error */
 337 };
 338
 339 typedef struct {                        /* Transmit Descriptor */
 340         uint    addr[2];                /* Data */
 341         uint    control;
 342         uint    status;
 343 } Td;
 344
 345 enum {                                  /* Tdesc control */
 346         LenMASK         = 0x000FFFFF,   /* Data/Packet Length Field */
 347         LenSHIFT        = 0,
 348         DtypeCD         = 0x00000000,   /* Data Type 'Context Descriptor' */
 349         DtypeDD         = 0x00100000,   /* Data Type 'Data Descriptor' */
 350         PtypeTCP        = 0x01000000,   /* TCP/UDP Packet Type (CD) */
 351         Teop            = 0x01000000,   /* End of Packet (DD) */
 352         PtypeIP         = 0x02000000,   /* IP Packet Type (CD) */
 353         Ifcs            = 0x02000000,   /* Insert FCS (DD) */
 354         Tse             = 0x04000000,   /* TCP Segmentation Enable */
 355         Rs              = 0x08000000,   /* Report Status */
 356         Rps             = 0x10000000,   /* Report Status Sent */
 357         Dext            = 0x20000000,   /* Descriptor Extension */
 358         Vle             = 0x40000000,   /* VLAN Packet Enable */
 359         Ide             = 0x80000000,   /* Interrupt Delay Enable */
 360 };
 361
 362 enum {                                  /* Tdesc status */
 363         Tdd             = 0x0001,       /* Descriptor Done */
 364         Ec              = 0x0002,       /* Excess Collisions */
 365         Lc              = 0x0004,       /* Late Collision */
 366         Tu              = 0x0008,       /* Transmit Underrun */
 367         CssMASK         = 0xFF00,       /* Checksum Start Field */
 368         CssSHIFT        = 8,
 369 };
 370
 371 typedef struct {
 372         ushort  *reg;
 373         ulong   *reg32;
 374         int     sz;
 375 } Flash;
 376
 377 enum {
 378         /* 16 and 32-bit flash registers for ich flash parts */
 379         Bfpr    = 0x00/4,               /* flash base 0:12; lim 16:28 */
 380         Fsts    = 0x04/2,               /* flash status; Hsfsts */
 381         Fctl    = 0x06/2,               /* flash control; Hsfctl */
 382         Faddr   = 0x08/4,               /* flash address to r/w */
 383         Fdata   = 0x10/4,               /* data @ address */
 384
 385         /* status register */
 386         Fdone   = 1<<0,                 /* flash cycle done */
 387         Fcerr   = 1<<1,                 /* cycle error; write 1 to clear */
 388         Ael     = 1<<2,                 /* direct access error log; 1 to clear */
 389         Scip    = 1<<5,                 /* spi cycle in progress */
 390         Fvalid  = 1<<14,                /* flash descriptor valid */
 391
 392         /* control register */
 393         Fgo     = 1<<0,                 /* start cycle */
 394         Flcycle = 1<<1,                 /* two bits: r=0; w=2 */
 395         Fdbc    = 1<<8,                 /* bytes to read; 5 bits */
 396 };
 397
 398 enum {
 399         Nrd             = 256,          /* power of two */
 400         Ntd             = 128,          /* power of two */
 401         Nrb             = 512+512,      /* private receive buffers per Ctlr */
 402         Rbalign         = BY2PG,        /* rx buffer alignment */
 403         Npool           = 8,
 404 };
 405
 406 /*
 407  * cavet emptor: 82577/78 have been entered speculatitively.
 408  * awating datasheet from intel.
 409  */
 410 enum {
 411         i82563,
 412         i82566,
 413         i82567,
 414         i82567m,
 415         i82571,
 416         i82572,
 417         i82573,
 418         i82574,
 419         i82575,
 420         i82576,
 421         i82577,
 422         i82577m,
 423         i82578,
 424         i82578m,
 425         i82579,
 426         i82580,
 427         i82583,
 428         Nctlrtype,
 429 };
 430
 431 enum {
 432         Fload   = 1<<0,
 433         Fert    = 1<<1,
 434         F75     = 1<<2,
 435         Fpba    = 1<<3,
 436         Fflashea        = 1<<4,
 437 };
 438
 439 typedef struct Ctlrtype Ctlrtype;
 440 struct Ctlrtype {
 441         int     type;
 442         int     mtu;
 443         int     flag;
 444         char    *name;
 445 };
 446
 447 static Ctlrtype cttab[Nctlrtype] = {
 448         i82563,         9014,   Fpba,           "i82563",
 449         i82566,         1514,   Fload,          "i82566",
 450         i82567,         9234,   Fload,          "i82567",
 451         i82567m,                1514,   0,              "i82567m",
 452         i82571,         9234,   Fpba,           "i82571",
 453         i82572,         9234,   Fpba,           "i82572",
 454         i82573,         8192,   Fert,           "i82573",               /* terrible perf above 8k */
 455         i82574,         9018,   0,              "i82574",
 456         i82575,         9728,   F75|Fflashea,   "i82575",
 457         i82576,         9728,   F75,            "i82576",
 458         i82577,         4096,   Fload|Fert,     "i82577",
 459         i82577m,                1514,   Fload|Fert,     "i82577",
 460         i82578,         4096,   Fload|Fert,     "i82578",
 461         i82578m,                1514,   Fload|Fert,     "i82578",
 462         i82579,         9018,   Fload|Fert,     "i82579",
 463         i82580,         9728,   F75,            "i82580",
 464         i82583,         1514,   0,              "i82583",
 465 };
 466
 467 typedef void (*Freefn)(Block*);
 468
 469 typedef struct Ctlr Ctlr;
 470 struct Ctlr {
 471         ulong   port;
 472         Pcidev  *pcidev;
 473         Ctlr    *next;
 474         int     active;
 475         int     type;
 476         int     pool;
 477         ushort  eeprom[0x40];
 478
 479         QLock   alock;                  /* attach */
 480         void    *alloc;                 /* receive/transmit descriptors */
 481         int     nrd;
 482         int     ntd;
 483         uint    rbsz;
 484
 485         int     *nic;
 486         Lock    imlock;
 487         int     im;                     /* interrupt mask */
 488
 489         Rendez  lrendez;
 490         int     lim;
 491
 492         QLock   slock;
 493         uint    statistics[Nstatistics];
 494         uint    lsleep;
 495         uint    lintr;
 496         uint    rsleep;
 497         uint    rintr;
 498         uint    txdw;
 499         uint    tintr;
 500         uint    ixsm;
 501         uint    ipcs;
 502         uint    tcpcs;
 503         uint    speeds[4];
 504         uint    phyerrata;
 505
 506         uchar   ra[Eaddrlen];           /* receive address */
 507         ulong   mta[128];               /* multicast table array */
 508
 509         Rendez  rrendez;
 510         int     rim;
 511         int     rdfree;
 512         Rd      *rdba;                  /* receive descriptor base address */
 513         Block   **rb;                   /* receive buffers */
 514         uint    rdh;                    /* receive descriptor head */
 515         uint    rdt;                    /* receive descriptor tail */
 516         int     rdtr;                   /* receive delay timer ring value */
 517         int     radv;                   /* receive interrupt absolute delay timer */
 518
 519         Rendez  trendez;
 520         QLock   tlock;
 521         int     tbusy;
 522         Td      *tdba;                  /* transmit descriptor base address */
 523         Block   **tb;                   /* transmit buffers */
 524         int     tdh;                    /* transmit descriptor head */
 525         int     tdt;                    /* transmit descriptor tail */
 526
 527         int     fcrtl;
 528         int     fcrth;
 529
 530         uint    pba;                    /* packet buffer allocation */
 531 };
 532
 533 typedef struct Rbpool Rbpool;
 534 struct Rbpool {
 535         union {
 536                 struct {
 537                         Lock;
 538                         Block   *b;
 539                         uint    nstarve;
 540                         uint    nwakey;
 541                         uint    starve;
 542                         Rendez;
 543                 };
 544                 uchar pad[128];         /* cacheline */
 545         };
 546
 547         Block   *x;
 548         uint    nfast;
 549         uint    nslow;
 550 };
 551
 552 #define csr32r(c, r)    (*((c)->nic+((r)/4)))
 553 #define csr32w(c, r, v) (*((c)->nic+((r)/4)) = (v))
 554
 555 static  Ctlr    *i82563ctlrhead;
 556 static  Ctlr    *i82563ctlrtail;
 557 static  Rbpool  rbtab[Npool];
 558
 559 static char *statistics[Nstatistics] = {
 560         "CRC Error",
 561         "Alignment Error",
 562         "Symbol Error",
 563         "RX Error",
 564         "Missed Packets",
 565         "Single Collision",
 566         "Excessive Collisions",
 567         "Multiple Collision",
 568         "Late Collisions",
 569         nil,
 570         "Collision",
 571         "Transmit Underrun",
 572         "Defer",
 573         "Transmit - No CRS",
 574         "Sequence Error",
 575         "Carrier Extension Error",
 576         "Receive Error Length",
 577         nil,
 578         "XON Received",
 579         "XON Transmitted",
 580         "XOFF Received",
 581         "XOFF Transmitted",
 582         "FC Received Unsupported",
 583         "Packets Received (64 Bytes)",
 584         "Packets Received (65-127 Bytes)",
 585         "Packets Received (128-255 Bytes)",
 586         "Packets Received (256-511 Bytes)",
 587         "Packets Received (512-1023 Bytes)",
 588         "Packets Received (1024-mtu Bytes)",
 589         "Good Packets Received",
 590         "Broadcast Packets Received",
 591         "Multicast Packets Received",
 592         "Good Packets Transmitted",
 593         nil,
 594         "Good Octets Received",
 595         nil,
 596         "Good Octets Transmitted",
 597         nil,
 598         nil,
 599         nil,
 600         "Receive No Buffers",
 601         "Receive Undersize",
 602         "Receive Fragment",
 603         "Receive Oversize",
 604         "Receive Jabber",
 605         "Management Packets Rx",
 606         "Management Packets Drop",
 607         "Management Packets Tx",
 608         "Total Octets Received",
 609         nil,
 610         "Total Octets Transmitted",
 611         nil,
 612         "Total Packets Received",
 613         "Total Packets Transmitted",
 614         "Packets Transmitted (64 Bytes)",
 615         "Packets Transmitted (65-127 Bytes)",
 616         "Packets Transmitted (128-255 Bytes)",
 617         "Packets Transmitted (256-511 Bytes)",
 618         "Packets Transmitted (512-1023 Bytes)",
 619         "Packets Transmitted (1024-mtu Bytes)",
 620         "Multicast Packets Transmitted",
 621         "Broadcast Packets Transmitted",
 622         "TCP Segmentation Context Transmitted",
 623         "TCP Segmentation Context Fail",
 624         "Interrupt Assertion",
 625         "Interrupt Rx Pkt Timer",
 626         "Interrupt Rx Abs Timer",
 627         "Interrupt Tx Pkt Timer",
 628         "Interrupt Tx Abs Timer",
 629         "Interrupt Tx Queue Empty",
 630         "Interrupt Tx Desc Low",
 631         "Interrupt Rx Min",
 632         "Interrupt Rx Overrun",
 633 };
 634
 635 static char*
 636 cname(Ctlr *c)
 637 {
 638         return cttab[c->type].name;
 639 }
 640
 641 static long
 642 i82563ifstat(Ether *edev, void *a, long n, ulong offset)
 643 {
 644         char *s, *p, *e, *stat;
 645         int i, r;
 646         uvlong tuvl, ruvl;
 647         Ctlr *ctlr;
 648         Rbpool *b;
 649
 650         ctlr = edev->ctlr;
 651         qlock(&ctlr->slock);
 652         p = s = malloc(READSTR);
 653         e = p + READSTR;
 654
 655         for(i = 0; i < Nstatistics; i++){
 656                 r = csr32r(ctlr, Statistics + i*4);
 657                 if((stat = statistics[i]) == nil)
 658                         continue;
 659                 switch(i){
 660                 case Gorcl:
 661                 case Gotcl:
 662                 case Torl:
 663                 case Totl:
 664                         ruvl = r;
 665                         ruvl += (uvlong)csr32r(ctlr, Statistics+(i+1)*4) << 32;
 666                         tuvl = ruvl;
 667                         tuvl += ctlr->statistics[i];
 668                         tuvl += (uvlong)ctlr->statistics[i+1] << 32;
 669                         if(tuvl == 0)
 670                                 continue;
 671                         ctlr->statistics[i] = tuvl;
 672                         ctlr->statistics[i+1] = tuvl >> 32;
 673                         p = seprint(p, e, "%s: %llud %llud\n", stat, tuvl, ruvl);
 674                         i++;
 675                         break;
 676
 677                 default:
 678                         ctlr->statistics[i] += r;
 679                         if(ctlr->statistics[i] == 0)
 680                                 continue;
 681                         p = seprint(p, e, "%s: %ud %ud\n", stat,
 682                                 ctlr->statistics[i], r);
 683                         break;
 684                 }
 685         }
 686
 687         p = seprint(p, e, "lintr: %ud %ud\n", ctlr->lintr, ctlr->lsleep);
 688         p = seprint(p, e, "rintr: %ud %ud\n", ctlr->rintr, ctlr->rsleep);
 689         p = seprint(p, e, "tintr: %ud %ud\n", ctlr->tintr, ctlr->txdw);
 690         p = seprint(p, e, "ixcs: %ud %ud %ud\n", ctlr->ixsm, ctlr->ipcs, ctlr->tcpcs);
 691         p = seprint(p, e, "rdtr: %ud\n", ctlr->rdtr);
 692         p = seprint(p, e, "radv: %ud\n", ctlr->radv);
 693         p = seprint(p, e, "ctrl: %.8ux\n", csr32r(ctlr, Ctrl));
 694         p = seprint(p, e, "ctrlext: %.8ux\n", csr32r(ctlr, Ctrlext));
 695         p = seprint(p, e, "status: %.8ux\n", csr32r(ctlr, Status));
 696         p = seprint(p, e, "txcw: %.8ux\n", csr32r(ctlr, Txcw));
 697         p = seprint(p, e, "txdctl: %.8ux\n", csr32r(ctlr, Txdctl));
 698         p = seprint(p, e, "pba: %.8ux\n", ctlr->pba);
 699
 700         b = rbtab + ctlr->pool;
 701         p = seprint(p, e, "pool: fast %ud slow %ud nstarve %ud nwakey %ud starve %ud\n",
 702                 b->nfast, b->nslow, b->nstarve, b->nwakey, b->starve);
 703         p = seprint(p, e, "speeds: 10:%ud 100:%ud 1000:%ud ?:%ud\n",
 704                 ctlr->speeds[0], ctlr->speeds[1], ctlr->speeds[2], ctlr->speeds[3]);
 705         p = seprint(p, e, "type: %s\n", cname(ctlr));
 706
 707 //      p = seprint(p, e, "eeprom:");
 708 //      for(i = 0; i < 0x40; i++){
 709 //              if(i && ((i & 7) == 0))
 710 //                      p = seprint(p, e, "\n       ");
 711 //              p = seprint(p, e, " %4.4ux", ctlr->eeprom[i]);
 712 //      }
 713 //      p = seprint(p, e, "\n");
 714
 715         USED(p);
 716         n = readstr(offset, a, n, s);
 717         free(s);
 718         qunlock(&ctlr->slock);
 719
 720         return n;
 721 }
 722
 723 static void
 724 i82563promiscuous(void *arg, int on)
 725 {
 726         int rctl;
 727         Ctlr *ctlr;
 728         Ether *edev;
 729
 730         edev = arg;
 731         ctlr = edev->ctlr;
 732
 733         rctl = csr32r(ctlr, Rctl);
 734         rctl &= ~MoMASK;
 735         if(on)
 736                 rctl |= Upe|Mpe;
 737         else
 738                 rctl &= ~(Upe|Mpe);
 739         csr32w(ctlr, Rctl, rctl);
 740 }
 741
 742 static void
 743 i82563multicast(void *arg, uchar *addr, int on)
 744 {
 745         int bit, x;
 746         Ctlr *ctlr;
 747         Ether *edev;
 748
 749         edev = arg;
 750         ctlr = edev->ctlr;
 751
 752         x = addr[5]>>1;
 753         if(ctlr->type == i82566)
 754                 x &= 31;
 755         bit = ((addr[5] & 1)<<4)|(addr[4]>>4);
 756         /*
 757          * multiple ether addresses can hash to the same filter bit,
 758          * so it's never safe to clear a filter bit.
 759          * if we want to clear filter bits, we need to keep track of
 760          * all the multicast addresses in use, clear all the filter bits,
 761          * then set the ones corresponding to in-use addresses.
 762          */
 763         if(on)
 764                 ctlr->mta[x] |= 1<<bit;
 765 //      else
 766 //              ctlr->mta[x] &= ~(1<<bit);
 767
 768         csr32w(ctlr, Mta+x*4, ctlr->mta[x]);
 769 }
 770
 771 static int
 772 icansleep(void *v)
 773 {
 774         Rbpool *p;
 775         int r;
 776
 777         p = v;
 778         ilock(p);
 779         r = p->starve == 0;
 780         iunlock(p);
 781
 782         return r;
 783 }
 784
 785 static Block*
 786 i82563rballoc(Rbpool *p)
 787 {
 788         Block *b;
 789
 790         for(;;){
 791                 if((b = p->x) != nil){
 792                         p->nfast++;
 793                         p->x = b->next;
 794                         b->next = nil;
 795                         _xinc(&b->ref);
 796                         return b;
 797                 }
 798
 799                 ilock(p);
 800                 b = p->b;
 801                 p->b = nil;
 802                 if(b == nil){
 803                         p->nstarve++;
 804                         iunlock(p);
 805                         return nil;
 806                 }
 807                 p->nslow++;
 808                 iunlock(p);
 809                 p->x = b;
 810         }
 811 }
 812
 813 static void
 814 rbfree(Block *b, int t)
 815 {
 816         Rbpool *p;
 817
 818         p = rbtab + t;
 819         b->rp = b->wp = (uchar*)ROUND((uintptr)b->base, Rbalign);
 820         b->flag &= ~(Bipck | Budpck | Btcpck | Bpktck);
 821
 822         ilock(p);
 823         b->next = p->b;
 824         p->b = b;
 825         if(p->starve){
 826                 if(0)
 827                         iprint("wakey %d; %d %d\n", t, p->nstarve, p->nwakey);
 828                 p->nwakey++;
 829                 p->starve = 0;
 830                 wakeup(p);
 831         }
 832         iunlock(p);
 833 }
 834
 835 static void
 836 rbfree0(Block *b)
 837 {
 838         rbfree(b, 0);
 839 }
 840
 841 static void
 842 rbfree1(Block *b)
 843 {
 844         rbfree(b, 1);
 845 }
 846
 847 static void
 848 rbfree2(Block *b)
 849 {
 850         rbfree(b, 2);
 851 }
 852
 853 static void
 854 rbfree3(Block *b)
 855 {
 856         rbfree(b, 3);
 857 }
 858
 859 static void
 860 rbfree4(Block *b)
 861 {
 862         rbfree(b, 4);
 863 }
 864
 865 static void
 866 rbfree5(Block *b)
 867 {
 868         rbfree(b, 5);
 869 }
 870
 871 static void
 872 rbfree6(Block *b)
 873 {
 874         rbfree(b, 6);
 875 }
 876
 877 static void
 878 rbfree7(Block *b)
 879 {
 880         rbfree(b, 7);
 881 }
 882
 883 static Freefn freetab[Npool] = {
 884         rbfree0,
 885         rbfree1,
 886         rbfree2,
 887         rbfree3,
 888         rbfree4,
 889         rbfree5,
 890         rbfree6,
 891         rbfree7,
 892 };
 893
 894 static int
 895 newpool(void)
 896 {
 897         static int seq;
 898
 899         if(seq == nelem(freetab))
 900                 return -1;
 901         if(freetab[seq] == nil){
 902                 print("82563: bad freetab\n");
 903                 return -1;
 904         }
 905         return seq++;
 906 }
 907
 908 static void
 909 i82563im(Ctlr *ctlr, int im)
 910 {
 911         ilock(&ctlr->imlock);
 912         ctlr->im |= im;
 913         csr32w(ctlr, Ims, ctlr->im);
 914         iunlock(&ctlr->imlock);
 915 }
 916
 917 static void
 918 i82563txinit(Ctlr *ctlr)
 919 {
 920         int i, r;
 921         Block *b;
 922
 923         if(cttab[ctlr->type].flag & F75)
 924                 csr32w(ctlr, Tctl, 0x0F<<CtSHIFT | Psp);
 925         else
 926                 csr32w(ctlr, Tctl, 0x0F<<CtSHIFT | Psp | 66<<ColdSHIFT | Mulr);
 927         csr32w(ctlr, Tipg, 6<<20 | 8<<10 | 8);          /* yb sez: 0x702008 */
 928         csr32w(ctlr, Tdbal, PCIWADDR(ctlr->tdba));
 929 //      csr32w(ctlr, Tdbah, Pciwaddrh(ctlr->tdba));
 930         csr32w(ctlr, Tdbah, 0);
 931         csr32w(ctlr, Tdlen, ctlr->ntd * sizeof(Td));
 932         ctlr->tdh = PREV(0, ctlr->ntd);
 933         csr32w(ctlr, Tdh, 0);
 934         ctlr->tdt = 0;
 935         csr32w(ctlr, Tdt, 0);
 936         for(i = 0; i < ctlr->ntd; i++){
 937                 if((b = ctlr->tb[i]) != nil){
 938                         ctlr->tb[i] = nil;
 939                         freeb(b);
 940                 }
 941                 memset(&ctlr->tdba[i], 0, sizeof(Td));
 942         }
 943         csr32w(ctlr, Tidv, 128);
 944         csr32w(ctlr, Tadv, 64);
 945         csr32w(ctlr, Tctl, csr32r(ctlr, Tctl) | Ten);
 946         r = csr32r(ctlr, Txdctl) & ~WthreshMASK;
 947         r |= 4<<WthreshSHIFT | 4<<PthreshSHIFT;
 948         if(cttab[ctlr->type].flag & F75)
 949                 r |= Enable;
 950         csr32w(ctlr, Txdctl, r);
 951 }
 952
 953 #define Next(x, m)      (((x)+1) & (m))
 954
 955 static int
 956 i82563cleanup(Ctlr *c)
 957 {
 958         Block *b;
 959         int tdh, m, n;
 960
 961         tdh = c->tdh;
 962         m = c->ntd-1;
 963         while(c->tdba[n = Next(tdh, m)].status & Tdd){
 964                 tdh = n;
 965                 if((b = c->tb[tdh]) != nil){
 966                         c->tb[tdh] = nil;
 967                         freeb(b);
 968                 }else
 969                         iprint("82563 tx underrun!\n");
 970                 c->tdba[tdh].status = 0;
 971         }
 972
 973         return c->tdh = tdh;
 974 }
 975
 976 static int
 977 notrim(void *v)
 978 {
 979         Ctlr *c;
 980
 981         c = v;
 982         return (c->im & Txdw) == 0;
 983 }
 984
 985 static void
 986 i82563tproc(void *v)
 987 {
 988         Td *td;
 989         Block *bp;
 990         Ether *edev;
 991         Ctlr *ctlr;
 992         int tdh, tdt, m;
 993
 994         edev = v;
 995         ctlr = edev->ctlr;
 996         tdt = ctlr->tdt;
 997         m = ctlr->ntd-1;
 998
 999         i82563txinit(ctlr);
1000
1001         for(;;){
1002                 tdh = i82563cleanup(ctlr);
1003
1004                 if(Next(tdt, m) == tdh){
1005                         ctlr->txdw++;
1006                         i82563im(ctlr, Txdw);
1007                         sleep(&ctlr->trendez, notrim, ctlr);
1008                 }
1009                 bp = qbread(edev->oq, 100000);
1010                 td = &ctlr->tdba[tdt];
1011                 td->addr[0] = PCIWADDR(bp->rp);
1012 //              td->addr[1] = Pciwaddrh(bp->rp);
1013                 td->control = Ide|Rs|Ifcs|Teop|BLEN(bp);
1014                 ctlr->tb[tdt] = bp;
1015                 tdt = Next(tdt, m);
1016                 csr32w(ctlr, Tdt, tdt);
1017         }
1018 }
1019
1020 static int
1021 i82563replenish(Ctlr *ctlr, int maysleep)
1022 {
1023         uint rdt, m, i;
1024         Block *bp;
1025         Rbpool *p;
1026         Rd *rd;
1027
1028         rdt = ctlr->rdt;
1029         m = ctlr->nrd-1;
1030         p = rbtab + ctlr->pool;
1031         i = 0;
1032         for(; Next(rdt, m) != ctlr->rdh; rdt = Next(rdt, m)){
1033                 rd = &ctlr->rdba[rdt];
1034                 if(ctlr->rb[rdt] != nil){
1035                         iprint("82563: tx overrun\n");
1036                         break;
1037                 }
1038         redux:
1039                 bp = i82563rballoc(p);
1040                 if(bp == nil){
1041                         if(rdt - ctlr->rdh >= 16)
1042                                 break;
1043                         print("i82563%d: no rx buffers\n", ctlr->pool);
1044                         if(maysleep == 0)
1045                                 return -1;
1046                         ilock(p);
1047                         p->starve = 1;
1048                         iunlock(p);
1049                         sleep(p, icansleep, p);
1050                         goto redux;
1051                 }
1052                 i++;
1053                 ctlr->rb[rdt] = bp;
1054                 rd->addr[0] = PCIWADDR(bp->rp);
1055         //      rd->addr[1] = Pciwaddrh(bp->rp);
1056                 rd->status = 0;
1057                 ctlr->rdfree++;
1058         }
1059         if(i != 0){
1060                 ctlr->rdt = rdt;
1061                 csr32w(ctlr, Rdt, rdt);
1062         }
1063         return 0;
1064 }
1065
1066 static void
1067 i82563rxinit(Ctlr *ctlr)
1068 {
1069         int i;
1070         Block *bp;
1071
1072         if(ctlr->rbsz <= 2048)
1073                 csr32w(ctlr, Rctl, Dpf|Bsize2048|Bam|RdtmsHALF);
1074         else{
1075                 i = ctlr->rbsz / 1024;
1076                 if(ctlr->rbsz % 1024)
1077                         i++;
1078                 if(cttab[ctlr->type].flag & F75){
1079                         csr32w(ctlr, Rctl, Lpe|Dpf|Bsize2048|Bam|RdtmsHALF|Secrc);
1080                         if(ctlr->type != i82575)
1081                                 i |= (ctlr->nrd/2>>4)<<20;              /* RdmsHalf */
1082                         csr32w(ctlr, Srrctl, i | Dropen);
1083                         csr32w(ctlr, Rmpl, ctlr->rbsz);
1084 //                      csr32w(ctlr, Drxmxod, 0x7ff);
1085                 }else
1086                         csr32w(ctlr, Rctl, Lpe|Dpf|BsizeFlex*i|Bam|RdtmsHALF|Secrc);
1087         }
1088
1089         if(cttab[ctlr->type].flag & Fert)
1090                 csr32w(ctlr, Ert, 1024/8);
1091
1092         if(ctlr->type == i82566)
1093                 csr32w(ctlr, Pbs, 16);
1094
1095         csr32w(ctlr, Rdbal, PCIWADDR(ctlr->rdba));
1096 //      csr32w(ctlr, Rdbah, Pciwaddrh(ctlr->rdba));
1097         csr32w(ctlr, Rdbah, 0);
1098         csr32w(ctlr, Rdlen, ctlr->nrd * sizeof(Rd));
1099         ctlr->rdh = 0;
1100         csr32w(ctlr, Rdh, 0);
1101         ctlr->rdt = 0;
1102         csr32w(ctlr, Rdt, 0);
1103         ctlr->rdtr = 25;
1104         ctlr->radv = 500;
1105         csr32w(ctlr, Rdtr, ctlr->rdtr);
1106         csr32w(ctlr, Radv, ctlr->radv);
1107
1108         for(i = 0; i < ctlr->nrd; i++)
1109                 if((bp = ctlr->rb[i]) != nil){
1110                         ctlr->rb[i] = nil;
1111                         freeb(bp);
1112                 }
1113         if(cttab[ctlr->type].flag & F75)
1114                 csr32w(ctlr, Rxdctl, 1<<WthreshSHIFT | 8<<PthreshSHIFT | 1<<HthreshSHIFT | Enable);
1115         else
1116                 csr32w(ctlr, Rxdctl, 2<<WthreshSHIFT | 2<<PthreshSHIFT);
1117
1118         /*
1119          * Enable checksum offload.
1120          */
1121         csr32w(ctlr, Rxcsum, Tuofl | Ipofl | ETHERHDRSIZE);
1122 }
1123
1124 static int
1125 i82563rim(void *v)
1126 {
1127         return ((Ctlr*)v)->rim != 0;
1128 }
1129
1130 static void
1131 i82563rproc(void *arg)
1132 {
1133         uint m, rdh, rim, im;
1134         Block *bp;
1135         Ctlr *ctlr;
1136         Ether *edev;
1137         Rd *rd;
1138
1139         edev = arg;
1140         ctlr = edev->ctlr;
1141
1142         i82563rxinit(ctlr);
1143         csr32w(ctlr, Rctl, csr32r(ctlr, Rctl) | Ren);
1144         if(cttab[ctlr->type].flag & F75){
1145                 csr32w(ctlr, Rxdctl, csr32r(ctlr, Rxdctl) | Enable);
1146                 im = Rxt0|Rxo|Rxdmt0|Rxseq|Ack;
1147         }else
1148                 im = Rxt0|Rxo|Rxdmt0|Rxseq|Ack;
1149         m = ctlr->nrd-1;
1150
1151         for(;;){
1152                 i82563im(ctlr, im);
1153                 ctlr->rsleep++;
1154                 i82563replenish(ctlr, 1);
1155                 sleep(&ctlr->rrendez, i82563rim, ctlr);
1156
1157                 rdh = ctlr->rdh;
1158                 for(;;){
1159                         rd = &ctlr->rdba[rdh];
1160                         rim = ctlr->rim;
1161                         ctlr->rim = 0;
1162                         if(!(rd->status & Rdd))
1163                                 break;
1164
1165                         /*
1166                          * Accept eop packets with no errors.
1167                          * With no errors and the Ixsm bit set,
1168                          * the descriptor status Tpcs and Ipcs bits give
1169                          * an indication of whether the checksums were
1170                          * calculated and valid.
1171                          */
1172                         bp = ctlr->rb[rdh];
1173                         if((rd->status & Reop) && rd->errors == 0){
1174                                 bp->wp += rd->length;
1175                                 bp->lim = bp->wp;       /* lie like a dog.  avoid packblock. */
1176                                 if(!(rd->status & Ixsm)){
1177                                         ctlr->ixsm++;
1178                                         if(rd->status & Ipcs){
1179                                                 /*
1180                                                  * IP checksum calculated
1181                                                  * (and valid as errors == 0).
1182                                                  */
1183                                                 ctlr->ipcs++;
1184                                                 bp->flag |= Bipck;
1185                                         }
1186                                         if(rd->status & Tcpcs){
1187                                                 /*
1188                                                  * TCP/UDP checksum calculated
1189                                                  * (and valid as errors == 0).
1190                                                  */
1191                                                 ctlr->tcpcs++;
1192                                                 bp->flag |= Btcpck|Budpck;
1193                                         }
1194                                         bp->checksum = rd->checksum;
1195                                         bp->flag |= Bpktck;
1196                                 }
1197                                 etheriq(edev, bp, 1);
1198                         } else
1199                                 freeb(bp);
1200                         ctlr->rb[rdh] = nil;
1201                         rd->status = 0;
1202                         ctlr->rdfree--;
1203                         ctlr->rdh = rdh = Next(rdh, m);
1204                         if(ctlr->nrd-ctlr->rdfree >= 32 || (rim & Rxdmt0))
1205                                 if(i82563replenish(ctlr, 0) == -1)
1206                                         break;
1207                 }
1208         }
1209 }
1210
1211 static int
1212 i82563lim(void *v)
1213 {
1214         return ((Ctlr*)v)->lim != 0;
1215 }
1216
1217 static int speedtab[] = {
1218         10, 100, 1000, 0
1219 };
1220
1221 static uint
1222 phyread(Ctlr *c, int phyno, int reg)
1223 {
1224         uint phy, i;
1225
1226         csr32w(c, Mdic, MDIrop | phyno<<MDIpSHIFT | reg<<MDIrSHIFT);
1227         phy = 0;
1228         for(i = 0; i < 64; i++){
1229                 phy = csr32r(c, Mdic);
1230                 if(phy & (MDIe|MDIready))
1231                         break;
1232                 microdelay(1);
1233         }
1234         if((phy & (MDIe|MDIready)) != MDIready){
1235                 print("%s: phy %d wedged %.8ux\n", cttab[c->type].name, phyno, phy);
1236                 return ~0;
1237         }
1238         return phy & 0xffff;
1239 }
1240
1241 static uint
1242 phywrite0(Ctlr *c, int phyno, int reg, ushort val)
1243 {
1244         uint phy, i;
1245
1246         csr32w(c, Mdic, MDIwop | phyno<<MDIpSHIFT | reg<<MDIrSHIFT | val);
1247         phy = 0;
1248         for(i = 0; i < 64; i++){
1249                 phy = csr32r(c, Mdic);
1250                 if(phy & (MDIe|MDIready))
1251                         break;
1252                 microdelay(1);
1253         }
1254         if((phy & (MDIe|MDIready)) != MDIready)
1255                 return ~0;
1256         return 0;
1257 }
1258
1259 static uint
1260 setpage(Ctlr *c, uint phyno, uint p, uint r)
1261 {
1262         uint pr;
1263
1264         if(c->type == i82563){
1265                 if(r >= 16 && r <= 28 && r != 22)
1266                         pr = Phypage;
1267                 else if(r == 30 || r == 31)
1268                         pr = Phyapage;
1269                 else
1270                         return 0;
1271                 return phywrite0(c, phyno, pr, p);
1272         }else if(p == 0)
1273                 return 0;
1274         return ~0;
1275 }
1276
1277 static uint
1278 phywrite(Ctlr *c, uint phyno, uint reg, ushort v)
1279 {
1280         if(setpage(c, phyno, reg>>8, reg & 0xff) == ~0)
1281                 panic("%s: bad phy reg %.4ux", cname(c), reg);
1282         return phywrite0(c, phyno, reg & 0xff, v);
1283 }
1284
1285 static void
1286 phyerrata(Ether *e, Ctlr *c)
1287 {
1288         if(e->mbps == 0)
1289                 if(c->phyerrata == 0){
1290                         c->phyerrata++;
1291                         phywrite(c, 1, Phyprst, Prst);  /* try a port reset */
1292                         print("%s: phy port reset\n", cname(c));
1293                 }
1294         else
1295                 c->phyerrata = 0;
1296 }
1297
1298 static void
1299 phyl79proc(void *v)
1300 {
1301         uint a, i, r, phy, phyno;
1302         Ctlr *c;
1303         Ether *e;
1304
1305         e = v;
1306         c = e->ctlr;
1307
1308         phyno = 1;
1309         if(c->type == i82579)
1310                 phyno = 2;
1311
1312         for(;;){
1313                 phy = phyread(c, phyno, Phystat);
1314                 if(phy == ~0)
1315                         goto next;
1316                 i = (phy>>8) & 3;
1317                 a = phy & Ans;
1318                 if(a){
1319                         r = phyread(c, phyno, Phyctl);
1320                         phywrite(c, phyno, Phyctl, r | Ran | Ean);
1321                 }
1322                 e->link = (phy & Link) != 0;
1323                 if(e->link == 0)
1324                         i = 3;
1325                 c->speeds[i]++;
1326                 e->mbps = speedtab[i];
1327 next:
1328                 c->lim = 0;
1329                 i82563im(c, Lsc);
1330                 c->lsleep++;
1331                 sleep(&c->lrendez, i82563lim, c);
1332         }
1333 }
1334
1335 static void
1336 phylproc(void *v)
1337 {
1338         uint a, i, phy;
1339         Ctlr *c;
1340         Ether *e;
1341
1342         e = v;
1343         c = e->ctlr;
1344
1345         if(c->type == i82573 && (phy = phyread(c, 1, Phyier)) != ~0)
1346                 phywrite(c, 1, Phyier, phy | Lscie | Ancie | Spdie | Panie);
1347         for(;;){
1348                 phy = phyread(c, 1, Physsr);
1349                 if(phy == ~0)
1350                         goto next;
1351                 i = (phy>>14) & 3;
1352                 switch(c->type){
1353                 default:
1354                         a = 0;
1355                         break;
1356                 case i82563:
1357                 case i82578:
1358                 case i82578m:
1359                 case i82583:
1360                         a = phyread(c, 1, Phyisr) & Ane;
1361                         break;
1362                 case i82571:
1363                 case i82572:
1364                 case i82575:
1365                 case i82576:
1366                         a = phyread(c, 1, Phylhr) & Anf;
1367                         i = (i-1) & 3;
1368                         break;
1369                 }
1370                 if(a)
1371                         phywrite(c, 1, Phyctl, phyread(c, 1, Phyctl) | Ran | Ean);
1372                 e->link = (phy & Rtlink) != 0;
1373                 if(e->link == 0)
1374                         i = 3;
1375                 c->speeds[i]++;
1376                 e->mbps = speedtab[i];
1377                 if(c->type == i82563)
1378                         phyerrata(e, c);
1379 next:
1380                 c->lim = 0;
1381                 i82563im(c, Lsc);
1382                 c->lsleep++;
1383                 sleep(&c->lrendez, i82563lim, c);
1384         }
1385 }
1386
1387 static void
1388 pcslproc(void *v)
1389 {
1390         uint i, phy;
1391         Ctlr *c;
1392         Ether *e;
1393
1394         e = v;
1395         c = e->ctlr;
1396
1397         for(;;){
1398                 phy = csr32r(c, Pcsstat);
1399                 e->link = phy & Linkok;
1400                 i = 3;
1401                 if(e->link)
1402                         i = (phy & 6) >> 1;
1403                 else if(phy & Anbad)
1404                         csr32w(c, Pcsctl, csr32r(c, Pcsctl) | Pan | Prestart);
1405                 c->speeds[i]++;
1406                 e->mbps = speedtab[i];
1407                 c->lim = 0;
1408                 i82563im(c, Lsc);
1409                 c->lsleep++;
1410                 sleep(&c->lrendez, i82563lim, c);
1411         }
1412 }
1413
1414 static void
1415 serdeslproc(void *v)
1416 {
1417         uint i, tx, rx;
1418         Ctlr *c;
1419         Ether *e;
1420
1421         e = v;
1422         c = e->ctlr;
1423
1424         for(;;){
1425                 rx = csr32r(c, Rxcw);
1426                 tx = csr32r(c, Txcw);
1427                 USED(tx);
1428                 e->link = (rx & 1<<31) != 0;
1429 //              e->link = (csr32r(c, Status) & Lu) != 0;
1430                 i = 3;
1431                 if(e->link)
1432                         i = 2;
1433                 c->speeds[i]++;
1434                 e->mbps = speedtab[i];
1435                 c->lim = 0;
1436                 i82563im(c, Lsc);
1437                 c->lsleep++;
1438                 sleep(&c->lrendez, i82563lim, c);
1439         }
1440 }
1441
1442 static void
1443 i82563attach(Ether *edev)
1444 {
1445         char name[KNAMELEN];
1446         int i;
1447         Block *bp;
1448         Ctlr *ctlr;
1449
1450         ctlr = edev->ctlr;
1451         qlock(&ctlr->alock);
1452         if(ctlr->alloc != nil){
1453                 qunlock(&ctlr->alock);
1454                 return;
1455         }
1456
1457         ctlr->nrd = Nrd;
1458         ctlr->ntd = Ntd;
1459         ctlr->alloc = malloc(ctlr->nrd*sizeof(Rd)+ctlr->ntd*sizeof(Td) + 255);
1460         if(ctlr->alloc == nil){
1461                 qunlock(&ctlr->alock);
1462                 error(Enomem);
1463         }
1464         ctlr->rdba = (Rd*)ROUNDUP((uintptr)ctlr->alloc, 256);
1465         ctlr->tdba = (Td*)(ctlr->rdba + ctlr->nrd);
1466
1467         ctlr->rb = malloc(ctlr->nrd * sizeof(Block*));
1468         ctlr->tb = malloc(ctlr->ntd * sizeof(Block*));
1469
1470         if(waserror()){
1471                 while(bp = i82563rballoc(rbtab + ctlr->pool)){
1472                         bp->free = nil;
1473                         freeb(bp);
1474                 }
1475                 free(ctlr->tb);
1476                 ctlr->tb = nil;
1477                 free(ctlr->rb);
1478                 ctlr->rb = nil;
1479                 free(ctlr->alloc);
1480                 ctlr->alloc = nil;
1481                 qunlock(&ctlr->alock);
1482                 nexterror();
1483         }
1484
1485         for(i = 0; i < Nrb; i++){
1486                 bp = allocb(ctlr->rbsz + Rbalign);
1487                 bp->free = freetab[ctlr->pool];
1488                 freeb(bp);
1489         }
1490
1491         snprint(name, sizeof name, "#l%dl", edev->ctlrno);
1492         if((csr32r(ctlr, Ctrlext) & Linkmode) == Serdes)
1493                 kproc(name, pcslproc, edev);            /* phy based serdes */
1494         else if(csr32r(ctlr, Status) & Tbimode)
1495                 kproc(name, serdeslproc, edev);         /* mac based serdes */
1496         else if(ctlr->type == i82579 || ctlr->type == i82580)
1497                 kproc(name, phyl79proc, edev);
1498         else
1499                 kproc(name, phylproc, edev);
1500
1501         snprint(name, sizeof name, "#l%dr", edev->ctlrno);
1502         kproc(name, i82563rproc, edev);
1503
1504         snprint(name, sizeof name, "#l%dt", edev->ctlrno);
1505         kproc(name, i82563tproc, edev);
1506
1507         qunlock(&ctlr->alock);
1508         poperror();
1509 }
1510
1511 static void
1512 i82563interrupt(Ureg*, void *arg)
1513 {
1514         Ctlr *ctlr;
1515         Ether *edev;
1516         int icr, im;
1517
1518         edev = arg;
1519         ctlr = edev->ctlr;
1520
1521         ilock(&ctlr->imlock);
1522         csr32w(ctlr, Imc, ~0);
1523         im = ctlr->im;
1524
1525         while(icr = csr32r(ctlr, Icr) & ctlr->im){
1526                 if(icr & Lsc){
1527                         im &= ~Lsc;
1528                         ctlr->lim = icr & Lsc;
1529                         wakeup(&ctlr->lrendez);
1530                         ctlr->lintr++;
1531                 }
1532                 if(icr & (Rxt0|Rxo|Rxdmt0|Rxseq|Ack)){
1533                         ctlr->rim = icr & (Rxt0|Rxo|Rxdmt0|Rxseq|Ack);
1534                         im &= ~(Rxt0|Rxo|Rxdmt0|Rxseq|Ack);
1535                         wakeup(&ctlr->rrendez);
1536                         ctlr->rintr++;
1537                 }
1538                 if(icr & Txdw){
1539                         im &= ~Txdw;
1540                         ctlr->tintr++;
1541                         wakeup(&ctlr->trendez);
1542                 }
1543         }
1544
1545         ctlr->im = im;
1546         csr32w(ctlr, Ims, im);
1547         iunlock(&ctlr->imlock);
1548 }
1549
1550 static int
1551 i82563detach(Ctlr *ctlr)
1552 {
1553         int r, timeo;
1554
1555         /* balance rx/tx packet buffer; survives reset */
1556         if(ctlr->rbsz > 8192 && cttab[ctlr->type].flag & Fpba){
1557                 ctlr->pba = csr32r(ctlr, Pba);
1558                 r = ctlr->pba >> 16;
1559                 r += ctlr->pba & 0xffff;
1560                 r >>= 1;
1561                 csr32w(ctlr, Pba, r);
1562         }else if(ctlr->type == i82573 && ctlr->rbsz > 1514)
1563                 csr32w(ctlr, Pba, 14);
1564         ctlr->pba = csr32r(ctlr, Pba);
1565
1566         /*
1567          * Perform a device reset to get the chip back to the
1568          * power-on state, followed by an EEPROM reset to read
1569          * the defaults for some internal registers.
1570          */
1571         csr32w(ctlr, Imc, ~0);
1572         csr32w(ctlr, Rctl, 0);
1573         csr32w(ctlr, Tctl, csr32r(ctlr, Tctl) & ~Ten);
1574
1575         delay(10);
1576
1577         r = csr32r(ctlr, Ctrl);
1578         if(ctlr->type == i82566 || ctlr->type == i82579)
1579                 r |= Phyrst;
1580         csr32w(ctlr, Ctrl, Devrst | r);
1581         delay(1);
1582         for(timeo = 0;; timeo++){
1583                 if((csr32r(ctlr, Ctrl) & (Devrst|Phyrst)) == 0)
1584                         break;
1585                 if(timeo >= 1000)
1586                         return -1;
1587                 delay(1);
1588         }
1589
1590         r = csr32r(ctlr, Ctrl);
1591         csr32w(ctlr, Ctrl, Slu|r);
1592
1593         r = csr32r(ctlr, Ctrlext);
1594         csr32w(ctlr, Ctrlext, r|Eerst);
1595         delay(1);
1596         for(timeo = 0; timeo < 1000; timeo++){
1597                 if(!(csr32r(ctlr, Ctrlext) & Eerst))
1598                         break;
1599                 delay(1);
1600         }
1601         if(csr32r(ctlr, Ctrlext) & Eerst)
1602                 return -1;
1603
1604         csr32w(ctlr, Imc, ~0);
1605         delay(1);
1606         for(timeo = 0; timeo < 1000; timeo++){
1607                 if((csr32r(ctlr, Icr) & ~Rxcfg) == 0)
1608                         break;
1609                 delay(1);
1610         }
1611         if(csr32r(ctlr, Icr) & ~Rxcfg)
1612                 return -1;
1613
1614         return 0;
1615 }
1616
1617 static void
1618 i82563shutdown(Ether *edev)
1619 {
1620         i82563detach(edev->ctlr);
1621 }
1622
1623 static ushort
1624 eeread(Ctlr *ctlr, int adr)
1625 {
1626         csr32w(ctlr, Eerd, EEstart | adr << 2);
1627         while ((csr32r(ctlr, Eerd) & EEdone) == 0)
1628                 ;
1629         return csr32r(ctlr, Eerd) >> 16;
1630 }
1631
1632 static int
1633 eeload(Ctlr *ctlr)
1634 {
1635         ushort sum;
1636         int data, adr;
1637
1638         sum = 0;
1639         for (adr = 0; adr < 0x40; adr++) {
1640                 data = eeread(ctlr, adr);
1641                 ctlr->eeprom[adr] = data;
1642                 sum += data;
1643         }
1644         return sum;
1645 }
1646
1647 static int
1648 fcycle(Ctlr *, Flash *f)
1649 {
1650         ushort s, i;
1651
1652         s = f->reg[Fsts];
1653         if((s&Fvalid) == 0)
1654                 return -1;
1655         f->reg[Fsts] |= Fcerr | Ael;
1656         for(i = 0; i < 10; i++){
1657                 if((s&Scip) == 0)
1658                         return 0;
1659                 delay(1);
1660                 s = f->reg[Fsts];
1661         }
1662         return -1;
1663 }
1664
1665 static int
1666 fread(Ctlr *c, Flash *f, int ladr)
1667 {
1668         ushort s;
1669
1670         delay(1);
1671         if(fcycle(c, f) == -1)
1672                 return -1;
1673         f->reg[Fsts] |= Fdone;
1674         f->reg32[Faddr] = ladr;
1675
1676         /* setup flash control register */
1677         s = f->reg[Fctl] & ~0x3ff;
1678         f->reg[Fctl] = s | 1<<8 | Fgo;  /* 2 byte read */
1679
1680         while((f->reg[Fsts] & Fdone) == 0)
1681                 ;
1682         if(f->reg[Fsts] & (Fcerr|Ael))
1683                 return -1;
1684         return f->reg32[Fdata] & 0xffff;
1685 }
1686
1687 static int
1688 fload(Ctlr *c)
1689 {
1690         ulong data, io, r, adr;
1691         ushort sum;
1692         Flash f;
1693
1694         io = c->pcidev->mem[1].bar & ~0x0f;
1695         f.reg = vmap(io, c->pcidev->mem[1].size);
1696         if(f.reg == nil)
1697                 return -1;
1698         f.reg32 = (ulong*)f.reg;
1699         f.sz = f.reg32[Bfpr];
1700         if(csr32r(c, Eec) & 1<<22){
1701                 if(c->type == i82579)
1702                         f.sz  += 16;            /* sector size: 64k */
1703                 else
1704                         f.sz  += 1;             /* sector size: 4k */
1705         }
1706         r = (f.sz & 0x1fff) << 12;
1707         sum = 0;
1708         for(adr = 0; adr < 0x40; adr++) {
1709                 data = fread(c, &f, r + adr*2);
1710                 if(data == -1)
1711                         return -1;
1712                 c->eeprom[adr] = data;
1713                 sum += data;
1714         }
1715         vunmap(f.reg, c->pcidev->mem[1].size);
1716         return sum;
1717 }
1718
1719 static void
1720 defaultea(Ctlr *ctlr, uchar *ra)
1721 {
1722         uint i, r;
1723         uvlong u;
1724         static uchar nilea[Eaddrlen];
1725
1726         if(memcmp(ra, nilea, Eaddrlen) != 0)
1727                 return;
1728         if(cttab[ctlr->type].flag & Fflashea){
1729                 /* intel mb bug */
1730                 u = (uvlong)csr32r(ctlr, Rah)<<32u | (ulong)csr32r(ctlr, Ral);
1731                 for(i = 0; i < Eaddrlen; i++)
1732                         ra[i] = u >> 8*i;
1733         }
1734         if(memcmp(ra, nilea, Eaddrlen) != 0)
1735                 return;
1736         for(i = 0; i < Eaddrlen/2; i++){
1737                 ra[2*i] = ctlr->eeprom[Ea+i];
1738                 ra[2*i+1] = ctlr->eeprom[Ea+i] >> 8;
1739         }
1740         r = (csr32r(ctlr, Status) & Lanid) >> 2;
1741         ra[5] += r;                             /* ea ctlr[n] = ea ctlr[0]+n */
1742 }
1743
1744 static int
1745 i82563reset(Ctlr *ctlr)
1746 {
1747         uchar *ra;
1748         int i, r;
1749
1750         if(i82563detach(ctlr))
1751                 return -1;
1752         if(cttab[ctlr->type].flag & Fload)
1753                 r = fload(ctlr);
1754         else
1755                 r = eeload(ctlr);
1756         if(r != 0 && r != 0xbaba){
1757                 print("%s: bad eeprom checksum - %#.4ux\n",
1758                         cname(ctlr), r);
1759                 return -1;
1760         }
1761
1762         ra = ctlr->ra;
1763         defaultea(ctlr, ra);
1764         csr32w(ctlr, Ral, ra[3]<<24 | ra[2]<<16 | ra[1]<<8 | ra[0]);
1765         csr32w(ctlr, Rah, 1<<31 | ra[5]<<8 | ra[4]);
1766         for(i = 1; i < 16; i++){
1767                 csr32w(ctlr, Ral+i*8, 0);
1768                 csr32w(ctlr, Rah+i*8, 0);
1769         }
1770         memset(ctlr->mta, 0, sizeof(ctlr->mta));
1771         for(i = 0; i < 128; i++)
1772                 csr32w(ctlr, Mta + i*4, 0);
1773         csr32w(ctlr, Fcal, 0x00C28001);
1774         csr32w(ctlr, Fcah, 0x0100);
1775         if(ctlr->type != i82579)
1776                 csr32w(ctlr, Fct, 0x8808);
1777         csr32w(ctlr, Fcttv, 0x0100);
1778         csr32w(ctlr, Fcrtl, ctlr->fcrtl);
1779         csr32w(ctlr, Fcrth, ctlr->fcrth);
1780         if(cttab[ctlr->type].flag & F75)
1781                 csr32w(ctlr, Eitr, 128<<2);             /* 128 ¼ microsecond intervals */
1782         return 0;
1783 }
1784
1785 enum {
1786         CMrdtr,
1787         CMradv,
1788         CMpause,
1789         CMan,
1790 };
1791
1792 static Cmdtab i82563ctlmsg[] = {
1793         CMrdtr, "rdtr", 2,
1794         CMradv, "radv", 2,
1795         CMpause, "pause", 1,
1796         CMan,   "an",   1,
1797 };
1798
1799 static long
1800 i82563ctl(Ether *edev, void *buf, long n)
1801 {
1802         char *p;
1803         ulong v;
1804         Ctlr *ctlr;
1805         Cmdbuf *cb;
1806         Cmdtab *ct;
1807
1808         if((ctlr = edev->ctlr) == nil)
1809                 error(Enonexist);
1810
1811         cb = parsecmd(buf, n);
1812         if(waserror()){
1813                 free(cb);
1814                 nexterror();
1815         }
1816
1817         ct = lookupcmd(cb, i82563ctlmsg, nelem(i82563ctlmsg));
1818         switch(ct->index){
1819         case CMrdtr:
1820                 v = strtoul(cb->f[1], &p, 0);
1821                 if(*p || v > 0xffff)
1822                         error(Ebadarg);
1823                 ctlr->rdtr = v;
1824                 csr32w(ctlr, Rdtr, v);
1825                 break;
1826         case CMradv:
1827                 v = strtoul(cb->f[1], &p, 0);
1828                 if(*p || v > 0xffff)
1829                         error(Ebadarg);
1830                 ctlr->radv = v;
1831                 csr32w(ctlr, Radv, v);
1832                 break;
1833         case CMpause:
1834                 csr32w(ctlr, Ctrl, csr32r(ctlr, Ctrl) ^ (1<<27 | 1<<28));
1835                 break;
1836         case CMan:
1837                 csr32w(ctlr, Ctrl, csr32r(ctlr, Ctrl) | Lrst | Phyrst);
1838                 break;
1839         }
1840         free(cb);
1841         poperror();
1842
1843         return n;
1844 }
1845
1846 static int
1847 didtype(int d)
1848 {
1849         switch(d){
1850         case 0x1096:
1851         case 0x10ba:            /* “gilgal” */
1852         // case 0x1098:         /* serdes; not seen */
1853         // case 0x10bb:         /* serdes */
1854                 return i82563;
1855         case 0x1049:            /* mm */
1856         case 0x104a:            /* dm */
1857         case 0x104b:            /* dc */
1858         case 0x104d:            /* v “ninevah” */
1859         case 0x10bd:            /* dm-2 */
1860         case 0x294c:            /* ich 9 */
1861                 return i82566;
1862         case 0x10de:            /* lm ich10d */
1863         case 0x10df:            /* lf ich10 */
1864         case 0x10e5:            /* lm ich9 */
1865         case 0x10f5:            /* lm ich9m; “boazman” */
1866                 return i82567;
1867         case 0x10bf:            /* lf ich9m */
1868         case 0x10cb:            /* v ich9m */
1869         case 0x10cd:            /* lf ich10 */
1870         case 0x10ce:            /* v ich10 */
1871         case 0x10cc:            /* lm ich10 */
1872                 return i82567m;
1873         case 0x105e:            /* eb */
1874         case 0x105f:            /* eb */
1875         case 0x1060:            /* eb */
1876         case 0x10a4:            /* eb */
1877         case 0x10a5:            /* eb  fiber */
1878         case 0x10bc:            /* eb */
1879         case 0x10d9:            /* eb serdes */
1880         case 0x10da:            /* eb serdes “ophir” */
1881                 return i82571;
1882         case 0x107d:            /* eb copper */
1883         case 0x107e:            /* ei fiber */
1884         case 0x107f:            /* ei */
1885         case 0x10b9:            /* ei “rimon” */
1886                 return i82572;
1887         case 0x108b:            /*  e “vidalia” */
1888         case 0x108c:            /*  e (iamt) */
1889         case 0x109a:            /*  l “tekoa” */
1890                 return i82573;
1891         case 0x10d3:            /* l or it; “hartwell” */
1892                 return i82574;
1893         case 0x10a7:
1894         case 0x10a9:            /* fiber/serdes */
1895                 return i82575;
1896         case 0x10c9:            /* copper */
1897         case 0x10e6:            /* fiber */
1898         case 0x10e7:            /* serdes; “kawela” */
1899                 return i82576;
1900         case 0x10ea:            /* lc “calpella”; aka pch lan */
1901                 return i82577;
1902         case 0x10eb:            /* lm “calpella” */
1903                 return i82577m;
1904         case 0x10ef:            /* dc “piketon” */
1905                 return i82578;
1906         case 0x1502:            /* lm */
1907         case 0x1503:            /* v */
1908                 return i82579;
1909         case 0x10f0:            /* dm “king's creek” */
1910                 return i82578m;
1911         case 0x150e:            /* “barton hills” */
1912         case 0x150f:            /* fiber */
1913         case 0x1510:            /* backplane */
1914         case 0x1511:            /* sfp */
1915         case 0x1516:
1916                 return i82580;
1917         case 0x1506:            /* v */
1918                 return i82583;
1919         }
1920         return -1;
1921 }
1922
1923 static void
1924 hbafixup(Pcidev *p)
1925 {
1926         uint i;
1927
1928         i = pcicfgr32(p, PciSVID);
1929         if((i & 0xffff) == 0x1b52 && p->did == 1)
1930                 p->did = i>>16;
1931 }
1932
1933 static void
1934 i82563pci(void)
1935 {
1936         int type;
1937         Ctlr *ctlr;
1938         Pcidev *p;
1939
1940         for(p = nil; p = pcimatch(p, 0x8086, 0);){
1941                 hbafixup(p);
1942                 if((type = didtype(p->did)) == -1)
1943                         continue;
1944                 ctlr = malloc(sizeof(Ctlr));
1945                 ctlr->type = type;
1946                 ctlr->pcidev = p;
1947                 ctlr->rbsz = cttab[type].mtu;
1948                 ctlr->port = p->mem[0].bar & ~0x0F;
1949                 if(i82563ctlrhead != nil)
1950                         i82563ctlrtail->next = ctlr;
1951                 else
1952                         i82563ctlrhead = ctlr;
1953                 i82563ctlrtail = ctlr;
1954         }
1955 }
1956
1957 static int
1958 setup(Ctlr *ctlr)
1959 {
1960         Pcidev *p;
1961
1962         if((ctlr->pool = newpool()) == -1){
1963                 print("%s: no pool\n", cname(ctlr));
1964                 return -1;
1965         }
1966         p = ctlr->pcidev;
1967         ctlr->nic = vmap(ctlr->port, p->mem[0].size);
1968         if(ctlr->nic == nil){
1969                 print("%s: can't map %#p\n", cname(ctlr), ctlr->port);
1970                 return -1;
1971         }
1972         if(i82563reset(ctlr)){
1973                 vunmap(ctlr->nic, p->mem[0].size);
1974                 return -1;
1975         }
1976         pcisetbme(ctlr->pcidev);
1977         return 0;
1978 }
1979
1980 static int
1981 pnp(Ether *edev, int type)
1982 {
1983         Ctlr *ctlr;
1984         static int done;
1985
1986         if(!done) {
1987                 i82563pci();
1988                 done = 1;
1989         }
1990
1991         /*
1992          * Any adapter matches if no edev->port is supplied,
1993          * otherwise the ports must match.
1994          */
1995         for(ctlr = i82563ctlrhead; ; ctlr = ctlr->next){
1996                 if(ctlr == nil)
1997                         return -1;
1998                 if(ctlr->active)
1999                         continue;
2000                 if(type != -1 && ctlr->type != type)
2001                         continue;
2002                 if(edev->port == 0 || edev->port == ctlr->port){
2003                         ctlr->active = 1;
2004                         memmove(ctlr->ra, edev->ea, Eaddrlen);
2005                         if(setup(ctlr) == 0)
2006                                 break;
2007                 }
2008         }
2009
2010         edev->ctlr = ctlr;
2011         edev->port = ctlr->port;
2012         edev->irq = ctlr->pcidev->intl;
2013         edev->tbdf = ctlr->pcidev->tbdf;
2014         edev->mbps = 1000;
2015         edev->maxmtu = ctlr->rbsz;
2016         memmove(edev->ea, ctlr->ra, Eaddrlen);
2017
2018         /*
2019          * Linkage to the generic ethernet driver.
2020          */
2021         edev->attach = i82563attach;
2022 //      edev->transmit = i82563transmit;
2023         edev->interrupt = i82563interrupt;
2024         edev->ifstat = i82563ifstat;
2025         edev->ctl = i82563ctl;
2026
2027         edev->arg = edev;
2028         edev->promiscuous = i82563promiscuous;
2029         edev->shutdown = i82563shutdown;
2030         edev->multicast = i82563multicast;
2031
2032         return 0;
2033 }
2034
2035 static int
2036 anypnp(Ether *e)
2037 {
2038         return pnp(e, -1);
2039 }
2040
2041 static int
2042 i82563pnp(Ether *e)
2043 {
2044         return pnp(e, i82563);
2045 }
2046
2047 static int
2048 i82566pnp(Ether *e)
2049 {
2050         return pnp(e, i82566);
2051 }
2052
2053 static int
2054 i82567pnp(Ether *e)
2055 {
2056         return pnp(e, i82567m) & pnp(e, i82567);
2057 }
2058
2059 static int
2060 i82571pnp(Ether *e)
2061 {
2062         return pnp(e, i82571);
2063 }
2064
2065 static int
2066 i82572pnp(Ether *e)
2067 {
2068         return pnp(e, i82572);
2069 }
2070
2071 static int
2072 i82573pnp(Ether *e)
2073 {
2074         return pnp(e, i82573);
2075 }
2076
2077 static int
2078 i82574pnp(Ether *e)
2079 {
2080         return pnp(e, i82574);
2081 }
2082
2083 static int
2084 i82575pnp(Ether *e)
2085 {
2086         return pnp(e, i82575);
2087 }
2088
2089 static int
2090 i82576pnp(Ether *e)
2091 {
2092         return pnp(e, i82576);
2093 }
2094
2095 static int
2096 i82577pnp(Ether *e)
2097 {
2098         return pnp(e, i82577m) & pnp(e, i82577);
2099 }
2100
2101 static int
2102 i82578pnp(Ether *e)
2103 {
2104         return pnp(e, i82578m) & pnp(e, i82578);
2105 }
2106
2107 static int
2108 i82579pnp(Ether *e)
2109 {
2110         return pnp(e, i82579);
2111 }
2112
2113 static int
2114 i82580pnp(Ether *e)
2115 {
2116         return pnp(e, i82580);
2117 }
2118
2119 static int
2120 i82583pnp(Ether *e)
2121 {
2122         return pnp(e, i82583);
2123 }
2124
2125 void
2126 ether82563link(void)
2127 {
2128         /*
2129          * recognise lots of model numbers for debugging
2130          * also good for forcing onboard nic(s) as ether0
2131          * try to make that unnecessary by listing lom first.
2132          */
2133         addethercard("i82563", i82563pnp);
2134         addethercard("i82566", i82566pnp);
2135         addethercard("i82574", i82574pnp);
2136         addethercard("i82576", i82576pnp);
2137         addethercard("i82567", i82567pnp);
2138         addethercard("i82573", i82573pnp);
2139
2140         addethercard("i82571", i82571pnp);
2141         addethercard("i82572", i82572pnp);
2142         addethercard("i82575", i82575pnp);
2143         addethercard("i82577", i82577pnp);
2144         addethercard("i82578", i82578pnp);
2145         addethercard("i82579", i82579pnp);
2146         addethercard("i82580", i82580pnp);
2147         addethercard("i82583", i82583pnp);
2148         addethercard("igbepcie", anypnp);
2149 }