sys/src/9/pc/ether82563.c

   1 /*
   2  * Intel Gigabit Ethernet PCI-Express Controllers.
   3  *      8256[36], 8257[12], 82573[ev]
   4  *      82575eb, 82576
   5  * Pretty basic, does not use many of the chip smarts.
   6  * The interrupt mitigation tuning for each chip variant
   7  * is probably different. The reset/initialisation
   8  * sequence needs straightened out. Doubt the PHY code
   9  * for the 82575eb is right.
  10  */
  11 #include "u.h"
  12 #include "../port/lib.h"
  13 #include "mem.h"
  14 #include "dat.h"
  15 #include "fns.h"
  16 #include "io.h"
  17 #include "../port/error.h"
  18 #include "../port/netif.h"
  19
  20 #include "etherif.h"
  21
  22 /*
  23  * these are in the order they appear in the manual, not numeric order.
  24  * It was too hard to find them in the book. Ref 21489, rev 2.6
  25  */
  26
  27 enum {
  28         /* General */
  29
  30         Ctrl            = 0x0000,       /* Device Control */
  31         Status          = 0x0008,       /* Device Status */
  32         Eec             = 0x0010,       /* EEPROM/Flash Control/Data */
  33         Eerd            = 0x0014,       /* EEPROM Read */
  34         Ctrlext         = 0x0018,       /* Extended Device Control */
  35         Fla             = 0x001c,       /* Flash Access */
  36         Mdic            = 0x0020,       /* MDI Control */
  37         Seresctl        = 0x0024,       /* Serdes ana */
  38         Fcal            = 0x0028,       /* Flow Control Address Low */
  39         Fcah            = 0x002C,       /* Flow Control Address High */
  40         Fct             = 0x0030,       /* Flow Control Type */
  41         Kumctrlsta      = 0x0034,       /* MAC-PHY Interface */
  42         Vet             = 0x0038,       /* VLAN EtherType */
  43         Fcttv           = 0x0170,       /* Flow Control Transmit Timer Value */
  44         Txcw            = 0x0178,       /* Transmit Configuration Word */
  45         Rxcw            = 0x0180,       /* Receive Configuration Word */
  46         Ledctl          = 0x0E00,       /* LED control */
  47         Pba             = 0x1000,       /* Packet Buffer Allocation */
  48         Pbs             = 0x1008,       /* Packet Buffer Size */
  49
  50         /* Interrupt */
  51
  52         Icr             = 0x00C0,       /* Interrupt Cause Read */
  53         Itr             = 0x00c4,       /* Interrupt Throttling Rate */
  54         Ics             = 0x00C8,       /* Interrupt Cause Set */
  55         Ims             = 0x00D0,       /* Interrupt Mask Set/Read */
  56         Imc             = 0x00D8,       /* Interrupt mask Clear */
  57         Iam             = 0x00E0,       /* Interrupt acknowledge Auto Mask */
  58
  59         /* Receive */
  60
  61         Rctl            = 0x0100,       /* Control */
  62         Ert             = 0x2008,       /* Early Receive Threshold (573[EVL] only) */
  63         Fcrtl           = 0x2160,       /* Flow Control RX Threshold Low */
  64         Fcrth           = 0x2168,       /* Flow Control Rx Threshold High */
  65         Psrctl          = 0x2170,       /* Packet Split Receive Control */
  66         Rdbal           = 0x2800,       /* Rdesc Base Address Low Queue 0 */
  67         Rdbah           = 0x2804,       /* Rdesc Base Address High Queue 0 */
  68         Rdlen           = 0x2808,       /* Descriptor Length Queue 0 */
  69         Rdh             = 0x2810,       /* Descriptor Head Queue 0 */
  70         Rdt             = 0x2818,       /* Descriptor Tail Queue 0 */
  71         Rdtr            = 0x2820,       /* Descriptor Timer Ring */
  72         Rxdctl          = 0x2828,       /* Descriptor Control */
  73         Radv            = 0x282C,       /* Interrupt Absolute Delay Timer */
  74         Rdbal1          = 0x2900,       /* Rdesc Base Address Low Queue 1 */
  75         Rdbah1          = 0x2804,       /* Rdesc Base Address High Queue 1 */
  76         Rdlen1          = 0x2908,       /* Descriptor Length Queue 1 */
  77         Rdh1            = 0x2910,       /* Descriptor Head Queue 1 */
  78         Rdt1            = 0x2918,       /* Descriptor Tail Queue 1 */
  79         Rxdctl1         = 0x2928,       /* Descriptor Control Queue 1 */
  80         Rsrpd           = 0x2c00,       /* Small Packet Detect */
  81         Raid            = 0x2c08,       /* ACK interrupt delay */
  82         Cpuvec          = 0x2c10,       /* CPU Vector */
  83         Rxcsum          = 0x5000,       /* Checksum Control */
  84         Rfctl           = 0x5008,       /* Filter Control */
  85         Mta             = 0x5200,       /* Multicast Table Array */
  86         Ral             = 0x5400,       /* Receive Address Low */
  87         Rah             = 0x5404,       /* Receive Address High */
  88         Vfta            = 0x5600,       /* VLAN Filter Table Array */
  89         Mrqc            = 0x5818,       /* Multiple Receive Queues Command */
  90         Rssim           = 0x5864,       /* RSS Interrupt Mask */
  91         Rssir           = 0x5868,       /* RSS Interrupt Request */
  92         Reta            = 0x5c00,       /* Redirection Table */
  93         Rssrk           = 0x5c80,       /* RSS Random Key */
  94
  95         /* Transmit */
  96
  97         Tctl            = 0x0400,       /* Transmit Control */
  98         Tipg            = 0x0410,       /* Transmit IPG */
  99         Tkabgtxd        = 0x3004,       /* glci afe band gap transmit ref data, or something */
 100         Tdbal           = 0x3800,       /* Tdesc Base Address Low */
 101         Tdbah           = 0x3804,       /* Tdesc Base Address High */
 102         Tdlen           = 0x3808,       /* Descriptor Length */
 103         Tdh             = 0x3810,       /* Descriptor Head */
 104         Tdt             = 0x3818,       /* Descriptor Tail */
 105         Tidv            = 0x3820,       /* Interrupt Delay Value */
 106         Txdctl          = 0x3828,       /* Descriptor Control */
 107         Tadv            = 0x382C,       /* Interrupt Absolute Delay Timer */
 108         Tarc0           = 0x3840,       /* Arbitration Counter Queue 0 */
 109         Tdbal1          = 0x3900,       /* Descriptor Base Low Queue 1 */
 110         Tdbah1          = 0x3904,       /* Descriptor Base High Queue 1 */
 111         Tdlen1          = 0x3908,       /* Descriptor Length Queue 1 */
 112         Tdh1            = 0x3910,       /* Descriptor Head Queue 1 */
 113         Tdt1            = 0x3918,       /* Descriptor Tail Queue 1 */
 114         Txdctl1         = 0x3928,       /* Descriptor Control 1 */
 115         Tarc1           = 0x3940,       /* Arbitration Counter Queue 1 */
 116
 117         /* Statistics */
 118
 119         Statistics      = 0x4000,       /* Start of Statistics Area */
 120         Gorcl           = 0x88/4,       /* Good Octets Received Count */
 121         Gotcl           = 0x90/4,       /* Good Octets Transmitted Count */
 122         Torl            = 0xC0/4,       /* Total Octets Received */
 123         Totl            = 0xC8/4,       /* Total Octets Transmitted */
 124         Nstatistics     = 0x124/4,
 125 };
 126
 127 enum {                                  /* Ctrl */
 128         GIOmd           = 1<<2,         /* BIO master disable */
 129         Lrst            = 1<<3,         /* link reset */
 130         Slu             = 1<<6,         /* Set Link Up */
 131         SspeedMASK      = 3<<8,         /* Speed Selection */
 132         SspeedSHIFT     = 8,
 133         Sspeed10        = 0x00000000,   /* 10Mb/s */
 134         Sspeed100       = 0x00000100,   /* 100Mb/s */
 135         Sspeed1000      = 0x00000200,   /* 1000Mb/s */
 136         Frcspd          = 1<<11,        /* Force Speed */
 137         Frcdplx         = 1<<12,        /* Force Duplex */
 138         SwdpinsloMASK   = 0x003C0000,   /* Software Defined Pins - lo nibble */
 139         SwdpinsloSHIFT  = 18,
 140         SwdpioloMASK    = 0x03C00000,   /* Software Defined Pins - I or O */
 141         SwdpioloSHIFT   = 22,
 142         Devrst          = 1<<26,        /* Device Reset */
 143         Rfce            = 1<<27,        /* Receive Flow Control Enable */
 144         Tfce            = 1<<28,        /* Transmit Flow Control Enable */
 145         Vme             = 1<<30,        /* VLAN Mode Enable */
 146         Phyrst          = 1<<31,        /* Phy Reset */
 147 };
 148
 149 enum {                                  /* Status */
 150         Lu              = 1<<1,         /* Link Up */
 151         Lanid           = 3<<2,         /* mask for Lan ID. */
 152         Txoff           = 1<<4,         /* Transmission Paused */
 153         Tbimode         = 1<<5,         /* TBI Mode Indication */
 154         Phyra           = 1<<10,        /* PHY Reset Asserted */
 155         GIOme           = 1<<19,        /* GIO Master Enable Status */
 156 };
 157
 158 enum {                                  /* Eerd */
 159         EEstart         = 1<<0,         /* Start Read */
 160         EEdone          = 1<<1,         /* Read done */
 161 };
 162
 163 enum {                                  /* Ctrlext */
 164         Asdchk          = 1<<12,        /* ASD Check */
 165         Eerst           = 1<<13,        /* EEPROM Reset */
 166         Spdbyps         = 1<<15,        /* Speed Select Bypass */
 167 };
 168
 169 enum {                                  /* EEPROM content offsets */
 170         Ea              = 0x00,         /* Ethernet Address */
 171         Cf              = 0x03,         /* Compatibility Field */
 172         Icw1            = 0x0A,         /* Initialization Control Word 1 */
 173         Sid             = 0x0B,         /* Subsystem ID */
 174         Svid            = 0x0C,         /* Subsystem Vendor ID */
 175         Did             = 0x0D,         /* Device ID */
 176         Vid             = 0x0E,         /* Vendor ID */
 177         Icw2            = 0x0F,         /* Initialization Control Word 2 */
 178 };
 179
 180 enum {                                  /* Mdic */
 181         MDIdMASK        = 0x0000FFFF,   /* Data */
 182         MDIdSHIFT       = 0,
 183         MDIrMASK        = 0x001F0000,   /* PHY Register Address */
 184         MDIrSHIFT       = 16,
 185         MDIpMASK        = 0x03E00000,   /* PHY Address */
 186         MDIpSHIFT       = 21,
 187         MDIwop          = 0x04000000,   /* Write Operation */
 188         MDIrop          = 0x08000000,   /* Read Operation */
 189         MDIready        = 0x10000000,   /* End of Transaction */
 190         MDIie           = 0x20000000,   /* Interrupt Enable */
 191         MDIe            = 0x40000000,   /* Error */
 192 };
 193
 194 enum {                                  /* phy interface registers */
 195         Phyctl          = 0,            /* phy ctl */
 196         Physsr          = 17,           /* phy secondary status */
 197         Phyier          = 18,           /* 82573 phy interrupt enable */
 198         Phyisr          = 19,           /* 82563 phy interrupt status */
 199         Phylhr          = 19,           /* 8257[12] link health */
 200
 201         Rtlink          = 1<<10,        /* realtime link status */
 202         Phyan           = 1<<11,        /* phy has auto-negotiated */
 203
 204         /* Phyctl bits */
 205         Ran             = 1<<9,         /* restart auto-negotiation */
 206         Ean             = 1<<12,        /* enable auto-negotiation */
 207
 208         /* 82573 Phyier bits */
 209         Lscie           = 1<<10,        /* link status changed ie */
 210         Ancie           = 1<<11,        /* auto-negotiation complete ie */
 211         Spdie           = 1<<14,        /* speed changed ie */
 212         Panie           = 1<<15,        /* phy auto-negotiation error ie */
 213
 214         /* Phylhr/Phyisr bits */
 215         Anf             = 1<<6,         /* lhr: auto-negotiation fault */
 216         Ane             = 1<<15,        /* isr: auto-negotiation error */
 217 };
 218
 219 enum {                                  /* Icr, Ics, Ims, Imc */
 220         Txdw            = 0x00000001,   /* Transmit Descriptor Written Back */
 221         Txqe            = 0x00000002,   /* Transmit Queue Empty */
 222         Lsc             = 0x00000004,   /* Link Status Change */
 223         Rxseq           = 0x00000008,   /* Receive Sequence Error */
 224         Rxdmt0          = 0x00000010,   /* Rdesc Minimum Threshold Reached */
 225         Rxo             = 0x00000040,   /* Receiver Overrun */
 226         Rxt0            = 0x00000080,   /* Receiver Timer Interrupt */
 227         Mdac            = 0x00000200,   /* MDIO Access Completed */
 228         Rxcfg           = 0x00000400,   /* Receiving /C/ ordered sets */
 229         Gpi0            = 0x00000800,   /* General Purpose Interrupts */
 230         Gpi1            = 0x00001000,
 231         Gpi2            = 0x00002000,
 232         Gpi3            = 0x00004000,
 233         Ack             = 0x00020000,   /* Receive ACK frame */
 234 };
 235
 236 enum {                                  /* Txcw */
 237         TxcwFd          = 0x00000020,   /* Full Duplex */
 238         TxcwHd          = 0x00000040,   /* Half Duplex */
 239         TxcwPauseMASK   = 0x00000180,   /* Pause */
 240         TxcwPauseSHIFT  = 7,
 241         TxcwPs          = 1<<TxcwPauseSHIFT,    /* Pause Supported */
 242         TxcwAs          = 2<<TxcwPauseSHIFT,    /* Asymmetric FC desired */
 243         TxcwRfiMASK     = 0x00003000,   /* Remote Fault Indication */
 244         TxcwRfiSHIFT    = 12,
 245         TxcwNpr         = 0x00008000,   /* Next Page Request */
 246         TxcwConfig      = 0x40000000,   /* Transmit Config Control */
 247         TxcwAne         = 0x80000000,   /* Auto-Negotiation Enable */
 248 };
 249
 250 enum {                                  /* Rctl */
 251         Rrst            = 0x00000001,   /* Receiver Software Reset */
 252         Ren             = 0x00000002,   /* Receiver Enable */
 253         Sbp             = 0x00000004,   /* Store Bad Packets */
 254         Upe             = 0x00000008,   /* Unicast Promiscuous Enable */
 255         Mpe             = 0x00000010,   /* Multicast Promiscuous Enable */
 256         Lpe             = 0x00000020,   /* Long Packet Reception Enable */
 257         LbmMASK         = 0x000000C0,   /* Loopback Mode */
 258         LbmOFF          = 0x00000000,   /* No Loopback */
 259         LbmTBI          = 0x00000040,   /* TBI Loopback */
 260         LbmMII          = 0x00000080,   /* GMII/MII Loopback */
 261         LbmXCVR         = 0x000000C0,   /* Transceiver Loopback */
 262         RdtmsMASK       = 0x00000300,   /* Rdesc Minimum Threshold Size */
 263         RdtmsHALF       = 0x00000000,   /* Threshold is 1/2 Rdlen */
 264         RdtmsQUARTER    = 0x00000100,   /* Threshold is 1/4 Rdlen */
 265         RdtmsEIGHTH     = 0x00000200,   /* Threshold is 1/8 Rdlen */
 266         MoMASK          = 0x00003000,   /* Multicast Offset */
 267         Bam             = 0x00008000,   /* Broadcast Accept Mode */
 268         BsizeMASK       = 0x00030000,   /* Receive Buffer Size */
 269         Bsize16384      = 0x00010000,   /* Bsex = 1 */
 270         Bsize8192       = 0x00020000,   /* Bsex = 1 */
 271         Bsize2048       = 0x00000000,
 272         Bsize1024       = 0x00010000,
 273         Bsize512        = 0x00020000,
 274         Bsize256        = 0x00030000,
 275         BsizeFlex       = 0x08000000,   /* Flexible Bsize in 1KB increments */
 276         Vfe             = 0x00040000,   /* VLAN Filter Enable */
 277         Cfien           = 0x00080000,   /* Canonical Form Indicator Enable */
 278         Cfi             = 0x00100000,   /* Canonical Form Indicator value */
 279         Dpf             = 0x00400000,   /* Discard Pause Frames */
 280         Pmcf            = 0x00800000,   /* Pass MAC Control Frames */
 281         Bsex            = 0x02000000,   /* Buffer Size Extension */
 282         Secrc           = 0x04000000,   /* Strip CRC from incoming packet */
 283 };
 284
 285 enum {                                  /* Tctl */
 286         Trst            = 0x00000001,   /* Transmitter Software Reset */
 287         Ten             = 0x00000002,   /* Transmit Enable */
 288         Psp             = 0x00000008,   /* Pad Short Packets */
 289         Mulr            = 0x10000000,   /* Allow multiple concurrent requests */
 290         CtMASK          = 0x00000FF0,   /* Collision Threshold */
 291         CtSHIFT         = 4,
 292         ColdMASK        = 0x003FF000,   /* Collision Distance */
 293         ColdSHIFT       = 12,
 294         Swxoff          = 0x00400000,   /* Sofware XOFF Transmission */
 295         Pbe             = 0x00800000,   /* Packet Burst Enable */
 296         Rtlc            = 0x01000000,   /* Re-transmit on Late Collision */
 297         Nrtu            = 0x02000000,   /* No Re-transmit on Underrrun */
 298 };
 299
 300 enum {                                  /* [RT]xdctl */
 301         PthreshMASK     = 0x0000003F,   /* Prefetch Threshold */
 302         PthreshSHIFT    = 0,
 303         HthreshMASK     = 0x00003F00,   /* Host Threshold */
 304         HthreshSHIFT    = 8,
 305         WthreshMASK     = 0x003F0000,   /* Writeback Threshold */
 306         WthreshSHIFT    = 16,
 307         Gran            = 0x01000000,   /* Granularity */
 308         Qenable         = 0x02000000,   /* Queue Enable (82575) */
 309 };
 310
 311 enum {                                  /* Rxcsum */
 312         PcssMASK        = 0x00FF,       /* Packet Checksum Start */
 313         PcssSHIFT       = 0,
 314         Ipofl           = 0x0100,       /* IP Checksum Off-load Enable */
 315         Tuofl           = 0x0200,       /* TCP/UDP Checksum Off-load Enable */
 316 };
 317
 318 enum {                                  /* Receive Delay Timer Ring */
 319         DelayMASK       = 0xFFFF,       /* delay timer in 1.024nS increments */
 320         DelaySHIFT      = 0,
 321         Fpd             = 0x80000000,   /* Flush partial Descriptor Block */
 322 };
 323
 324 typedef struct Rd {                     /* Receive Descriptor */
 325         u32int  addr[2];
 326         u16int  length;
 327         u16int  checksum;
 328         u8int   status;
 329         u8int   errors;
 330         u16int  special;
 331 } Rd;
 332
 333 enum {                                  /* Rd status */
 334         Rdd             = 0x01,         /* Descriptor Done */
 335         Reop            = 0x02,         /* End of Packet */
 336         Ixsm            = 0x04,         /* Ignore Checksum Indication */
 337         Vp              = 0x08,         /* Packet is 802.1Q (matched VET) */
 338         Tcpcs           = 0x20,         /* TCP Checksum Calculated on Packet */
 339         Ipcs            = 0x40,         /* IP Checksum Calculated on Packet */
 340         Pif             = 0x80,         /* Passed in-exact filter */
 341 };
 342
 343 enum {                                  /* Rd errors */
 344         Ce              = 0x01,         /* CRC Error or Alignment Error */
 345         Se              = 0x02,         /* Symbol Error */
 346         Seq             = 0x04,         /* Sequence Error */
 347         Cxe             = 0x10,         /* Carrier Extension Error */
 348         Tcpe            = 0x20,         /* TCP/UDP Checksum Error */
 349         Ipe             = 0x40,         /* IP Checksum Error */
 350         Rxe             = 0x80,         /* RX Data Error */
 351 };
 352
 353 typedef struct {                        /* Transmit Descriptor */
 354         u32int  addr[2];                /* Data */
 355         u32int  control;
 356         u32int  status;
 357 } Td;
 358
 359 enum {                                  /* Tdesc control */
 360         LenMASK         = 0x000FFFFF,   /* Data/Packet Length Field */
 361         LenSHIFT        = 0,
 362         DtypeCD         = 0x00000000,   /* Data Type 'Context Descriptor' */
 363         DtypeDD         = 0x00100000,   /* Data Type 'Data Descriptor' */
 364         PtypeTCP        = 0x01000000,   /* TCP/UDP Packet Type (CD) */
 365         Teop            = 0x01000000,   /* End of Packet (DD) */
 366         PtypeIP         = 0x02000000,   /* IP Packet Type (CD) */
 367         Ifcs            = 0x02000000,   /* Insert FCS (DD) */
 368         Tse             = 0x04000000,   /* TCP Segmentation Enable */
 369         Rs              = 0x08000000,   /* Report Status */
 370         Rps             = 0x10000000,   /* Report Status Sent */
 371         Dext            = 0x20000000,   /* Descriptor Extension */
 372         Vle             = 0x40000000,   /* VLAN Packet Enable */
 373         Ide             = 0x80000000,   /* Interrupt Delay Enable */
 374 };
 375
 376 enum {                                  /* Tdesc status */
 377         Tdd             = 0x0001,       /* Descriptor Done */
 378         Ec              = 0x0002,       /* Excess Collisions */
 379         Lc              = 0x0004,       /* Late Collision */
 380         Tu              = 0x0008,       /* Transmit Underrun */
 381         CssMASK         = 0xFF00,       /* Checksum Start Field */
 382         CssSHIFT        = 8,
 383 };
 384
 385 typedef struct {
 386         u16int  *reg;
 387         u32int  *reg32;
 388         int     sz;
 389 } Flash;
 390
 391 enum {
 392         /* 16 and 32-bit flash registers for ich flash parts */
 393         Bfpr    = 0x00/4,               /* flash base 0:12; lim 16:28 */
 394         Fsts    = 0x04/2,               /* flash status;  Hsfsts */
 395         Fctl    = 0x06/2,               /* flash control; Hsfctl */
 396         Faddr   = 0x08/4,               /* flash address to r/w */
 397         Fdata   = 0x10/4,               /* data @ address */
 398
 399         /* status register */
 400         Fdone   = 1<<0,                 /* flash cycle done */
 401         Fcerr   = 1<<1,                 /* cycle error; write 1 to clear */
 402         Ael     = 1<<2,                 /* direct access error log; 1 to clear */
 403         Scip    = 1<<5,                 /* spi cycle in progress */
 404         Fvalid  = 1<<14,                /* flash descriptor valid */
 405
 406         /* control register */
 407         Fgo     = 1<<0,                 /* start cycle */
 408         Flcycle = 1<<1,                 /* two bits: r=0; w=2 */
 409         Fdbc    = 1<<8,                 /* bytes to read; 5 bits */
 410 };
 411
 412 enum {
 413         Nrd             = 256,          /* power of two */
 414         Ntd             = 128,          /* power of two */
 415         Nrb             = 1024,         /* private receive buffers per Ctlr */
 416 };
 417
 418 enum {
 419         Iany,
 420         i82563,
 421         i82566,
 422         i82567,
 423         i82571,
 424         i82572,
 425         i82573,
 426         i82575,
 427         i82576,
 428 };
 429
 430 static int rbtab[] = {
 431         0,
 432         9014,
 433         1514,
 434         1514,
 435         9234,
 436         9234,
 437         8192,                           /* terrible performance above 8k */
 438         1514,
 439         1514,
 440 };
 441
 442 static char *tname[] = {
 443         "any",
 444         "i82563",
 445         "i82566",
 446         "i82567",
 447         "i82571",
 448         "i82572",
 449         "i82573",
 450         "i82575",
 451         "i82576",
 452 };
 453
 454 typedef struct Ctlr Ctlr;
 455 struct Ctlr {
 456         int     port;
 457         Pcidev  *pcidev;
 458         Ctlr    *next;
 459         Ether   *edev;
 460         int     active;
 461         int     type;
 462         ushort  eeprom[0x40];
 463
 464         QLock   alock;                  /* attach */
 465         int     attached;
 466         int     nrd;
 467         int     ntd;
 468         int     nrb;                    /* how many this Ctlr has in the pool */
 469         unsigned rbsz;                  /* unsigned for % and / by 1024 */
 470
 471         int     *nic;
 472         Lock    imlock;
 473         int     im;                     /* interrupt mask */
 474
 475         Rendez  lrendez;
 476         int     lim;
 477
 478         QLock   slock;
 479         uint    statistics[Nstatistics];
 480         uint    lsleep;
 481         uint    lintr;
 482         uint    rsleep;
 483         uint    rintr;
 484         uint    txdw;
 485         uint    tintr;
 486         uint    ixsm;
 487         uint    ipcs;
 488         uint    tcpcs;
 489         uint    speeds[4];
 490
 491         uchar   ra[Eaddrlen];           /* receive address */
 492         ulong   mta[128];               /* multicast table array */
 493
 494         Rendez  rrendez;
 495         int     rim;
 496         int     rdfree;
 497         Rd      *rdba;                  /* receive descriptor base address */
 498         Block   **rb;                   /* receive buffers */
 499         int     rdh;                    /* receive descriptor head */
 500         int     rdt;                    /* receive descriptor tail */
 501         int     rdtr;                   /* receive delay timer ring value */
 502         int     radv;                   /* receive interrupt absolute delay timer */
 503
 504         Rendez  trendez;
 505         QLock   tlock;
 506         int     tbusy;
 507         Td      *tdba;                  /* transmit descriptor base address */
 508         Block   **tb;                   /* transmit buffers */
 509         int     tdh;                    /* transmit descriptor head */
 510         int     tdt;                    /* transmit descriptor tail */
 511
 512         int     fcrtl;
 513         int     fcrth;
 514
 515         uint    pba;                    /* packet buffer allocation */
 516 };
 517
 518 #define csr32r(c, r)    (*((c)->nic+((r)/4)))
 519 #define csr32w(c, r, v) (*((c)->nic+((r)/4)) = (v))
 520
 521 static Ctlr* i82563ctlrhead;
 522 static Ctlr* i82563ctlrtail;
 523
 524 static Lock i82563rblock;               /* free receive Blocks */
 525 static Block* i82563rbpool;
 526
 527 static char* statistics[] = {
 528         "CRC Error",
 529         "Alignment Error",
 530         "Symbol Error",
 531         "RX Error",
 532         "Missed Packets",
 533         "Single Collision",
 534         "Excessive Collisions",
 535         "Multiple Collision",
 536         "Late Collisions",
 537         nil,
 538         "Collision",
 539         "Transmit Underrun",
 540         "Defer",
 541         "Transmit - No CRS",
 542         "Sequence Error",
 543         "Carrier Extension Error",
 544         "Receive Error Length",
 545         nil,
 546         "XON Received",
 547         "XON Transmitted",
 548         "XOFF Received",
 549         "XOFF Transmitted",
 550         "FC Received Unsupported",
 551         "Packets Received (64 Bytes)",
 552         "Packets Received (65-127 Bytes)",
 553         "Packets Received (128-255 Bytes)",
 554         "Packets Received (256-511 Bytes)",
 555         "Packets Received (512-1023 Bytes)",
 556         "Packets Received (1024-mtu Bytes)",
 557         "Good Packets Received",
 558         "Broadcast Packets Received",
 559         "Multicast Packets Received",
 560         "Good Packets Transmitted",
 561         nil,
 562         "Good Octets Received",
 563         nil,
 564         "Good Octets Transmitted",
 565         nil,
 566         nil,
 567         nil,
 568         "Receive No Buffers",
 569         "Receive Undersize",
 570         "Receive Fragment",
 571         "Receive Oversize",
 572         "Receive Jabber",
 573         "Management Packets Rx",
 574         "Management Packets Drop",
 575         "Management Packets Tx",
 576         "Total Octets Received",
 577         nil,
 578         "Total Octets Transmitted",
 579         nil,
 580         "Total Packets Received",
 581         "Total Packets Transmitted",
 582         "Packets Transmitted (64 Bytes)",
 583         "Packets Transmitted (65-127 Bytes)",
 584         "Packets Transmitted (128-255 Bytes)",
 585         "Packets Transmitted (256-511 Bytes)",
 586         "Packets Transmitted (512-1023 Bytes)",
 587         "Packets Transmitted (1024-mtu Bytes)",
 588         "Multicast Packets Transmitted",
 589         "Broadcast Packets Transmitted",
 590         "TCP Segmentation Context Transmitted",
 591         "TCP Segmentation Context Fail",
 592         "Interrupt Assertion",
 593         "Interrupt Rx Pkt Timer",
 594         "Interrupt Rx Abs Timer",
 595         "Interrupt Tx Pkt Timer",
 596         "Interrupt Tx Abs Timer",
 597         "Interrupt Tx Queue Empty",
 598         "Interrupt Tx Desc Low",
 599         "Interrupt Rx Min",
 600         "Interrupt Rx Overrun",
 601 };
 602
 603 static long
 604 i82563ifstat(Ether* edev, void* a, long n, ulong offset)
 605 {
 606         Ctlr *ctlr;
 607         char *s, *p, *e, *stat;
 608         int i, r;
 609         uvlong tuvl, ruvl;
 610
 611         ctlr = edev->ctlr;
 612         qlock(&ctlr->slock);
 613         p = s = malloc(READSTR);
 614         e = p + READSTR;
 615
 616         for(i = 0; i < Nstatistics; i++){
 617                 r = csr32r(ctlr, Statistics + i*4);
 618                 if((stat = statistics[i]) == nil)
 619                         continue;
 620                 switch(i){
 621                 case Gorcl:
 622                 case Gotcl:
 623                 case Torl:
 624                 case Totl:
 625                         ruvl = r;
 626                         ruvl += (uvlong)csr32r(ctlr, Statistics+(i+1)*4) << 32;
 627                         tuvl = ruvl;
 628                         tuvl += ctlr->statistics[i];
 629                         tuvl += (uvlong)ctlr->statistics[i+1] << 32;
 630                         if(tuvl == 0)
 631                                 continue;
 632                         ctlr->statistics[i] = tuvl;
 633                         ctlr->statistics[i+1] = tuvl >> 32;
 634                         p = seprint(p, e, "%s: %llud %llud\n", stat, tuvl, ruvl);
 635                         i++;
 636                         break;
 637
 638                 default:
 639                         ctlr->statistics[i] += r;
 640                         if(ctlr->statistics[i] == 0)
 641                                 continue;
 642                         p = seprint(p, e, "%s: %ud %ud\n", stat,
 643                                 ctlr->statistics[i], r);
 644                         break;
 645                 }
 646         }
 647
 648         p = seprint(p, e, "lintr: %ud %ud\n", ctlr->lintr, ctlr->lsleep);
 649         p = seprint(p, e, "rintr: %ud %ud\n", ctlr->rintr, ctlr->rsleep);
 650         p = seprint(p, e, "tintr: %ud %ud\n", ctlr->tintr, ctlr->txdw);
 651         p = seprint(p, e, "ixcs: %ud %ud %ud\n", ctlr->ixsm, ctlr->ipcs, ctlr->tcpcs);
 652         p = seprint(p, e, "rdtr: %ud\n", ctlr->rdtr);
 653         p = seprint(p, e, "radv: %ud\n", ctlr->radv);
 654         p = seprint(p, e, "ctrl: %.8ux\n", csr32r(ctlr, Ctrl));
 655         p = seprint(p, e, "ctrlext: %.8ux\n", csr32r(ctlr, Ctrlext));
 656         p = seprint(p, e, "status: %.8ux\n", csr32r(ctlr, Status));
 657         p = seprint(p, e, "txcw: %.8ux\n", csr32r(ctlr, Txcw));
 658         p = seprint(p, e, "txdctl: %.8ux\n", csr32r(ctlr, Txdctl));
 659         p = seprint(p, e, "pba: %.8ux\n", ctlr->pba);
 660
 661         p = seprint(p, e, "speeds: 10:%ud 100:%ud 1000:%ud ?:%ud\n",
 662                 ctlr->speeds[0], ctlr->speeds[1], ctlr->speeds[2], ctlr->speeds[3]);
 663         p = seprint(p, e, "type: %s\n", tname[ctlr->type]);
 664
 665 //      p = seprint(p, e, "eeprom:");
 666 //      for(i = 0; i < 0x40; i++){
 667 //              if(i && ((i & 7) == 0))
 668 //                      p = seprint(p, e, "\n       ");
 669 //              p = seprint(p, e, " %4.4ux", ctlr->eeprom[i]);
 670 //      }
 671 //      p = seprint(p, e, "\n");
 672
 673         USED(p);
 674         n = readstr(offset, a, n, s);
 675         free(s);
 676         qunlock(&ctlr->slock);
 677
 678         return n;
 679 }
 680
 681 enum {
 682         CMrdtr,
 683         CMradv,
 684 };
 685
 686 static Cmdtab i82563ctlmsg[] = {
 687         CMrdtr, "rdtr", 2,
 688         CMradv, "radv", 2,
 689 };
 690
 691 static long
 692 i82563ctl(Ether* edev, void* buf, long n)
 693 {
 694         ulong v;
 695         char *p;
 696         Ctlr *ctlr;
 697         Cmdbuf *cb;
 698         Cmdtab *ct;
 699
 700         if((ctlr = edev->ctlr) == nil)
 701                 error(Enonexist);
 702
 703         cb = parsecmd(buf, n);
 704         if(waserror()){
 705                 free(cb);
 706                 nexterror();
 707         }
 708
 709         ct = lookupcmd(cb, i82563ctlmsg, nelem(i82563ctlmsg));
 710         switch(ct->index){
 711         case CMrdtr:
 712                 v = strtoul(cb->f[1], &p, 0);
 713                 if(p == cb->f[1] || v > 0xFFFF)
 714                         error(Ebadarg);
 715                 ctlr->rdtr = v;
 716                 csr32w(ctlr, Rdtr, v);
 717                 break;
 718         case CMradv:
 719                 v = strtoul(cb->f[1], &p, 0);
 720                 if(p == cb->f[1] || v > 0xFFFF)
 721                         error(Ebadarg);
 722                 ctlr->radv = v;
 723                 csr32w(ctlr, Radv, v);
 724         }
 725         free(cb);
 726         poperror();
 727
 728         return n;
 729 }
 730
 731 static void
 732 i82563promiscuous(void* arg, int on)
 733 {
 734         int rctl;
 735         Ctlr *ctlr;
 736         Ether *edev;
 737
 738         edev = arg;
 739         ctlr = edev->ctlr;
 740
 741         rctl = csr32r(ctlr, Rctl);
 742         rctl &= ~MoMASK;
 743         if(on)
 744                 rctl |= Upe|Mpe;
 745         else
 746                 rctl &= ~(Upe|Mpe);
 747         csr32w(ctlr, Rctl, rctl);
 748 }
 749
 750 static void
 751 i82563multicast(void* arg, uchar* addr, int on)
 752 {
 753         int bit, x;
 754         Ctlr *ctlr;
 755         Ether *edev;
 756
 757         edev = arg;
 758         ctlr = edev->ctlr;
 759
 760         x = addr[5]>>1;
 761         if(ctlr->type == i82566 || ctlr->type == i82567)
 762                 x &= 31;
 763         bit = ((addr[5] & 1)<<4)|(addr[4]>>4);
 764         /*
 765          * multiple ether addresses can hash to the same filter bit,
 766          * so it's never safe to clear a filter bit.
 767          * if we want to clear filter bits, we need to keep track of
 768          * all the multicast addresses in use, clear all the filter bits,
 769          * then set the ones corresponding to in-use addresses.
 770          */
 771         if(on)
 772                 ctlr->mta[x] |= 1<<bit;
 773 //      else
 774 //              ctlr->mta[x] &= ~(1<<bit);
 775
 776         csr32w(ctlr, Mta+x*4, ctlr->mta[x]);
 777 }
 778
 779 static Block*
 780 i82563rballoc(void)
 781 {
 782         Block *bp;
 783
 784         ilock(&i82563rblock);
 785         if((bp = i82563rbpool) != nil){
 786                 i82563rbpool = bp->next;
 787                 bp->next = nil;
 788                 _xinc(&bp->ref);        /* prevent bp from being freed */
 789         }
 790         iunlock(&i82563rblock);
 791
 792         return bp;
 793 }
 794
 795 static void
 796 i82563rbfree(Block* b)
 797 {
 798         b->rp = b->wp = (uchar*)PGROUND((uintptr)b->base);
 799         b->flag &= ~(Bipck | Budpck | Btcpck | Bpktck);
 800         ilock(&i82563rblock);
 801         b->next = i82563rbpool;
 802         i82563rbpool = b;
 803         iunlock(&i82563rblock);
 804 }
 805
 806 static void
 807 i82563im(Ctlr* ctlr, int im)
 808 {
 809         ilock(&ctlr->imlock);
 810         ctlr->im |= im;
 811         csr32w(ctlr, Ims, ctlr->im);
 812         iunlock(&ctlr->imlock);
 813 }
 814
 815 static void
 816 i82563txinit(Ctlr* ctlr)
 817 {
 818         int i, r;
 819         Block *bp;
 820
 821         csr32w(ctlr, Tctl, 0x0F<<CtSHIFT | Psp | 66<<ColdSHIFT | Mulr);
 822         csr32w(ctlr, Tipg, 6<<20 | 8<<10 | 8);          /* yb sez: 0x702008 */
 823         csr32w(ctlr, Tdbal, PCIWADDR(ctlr->tdba));
 824         csr32w(ctlr, Tdbah, 0);
 825         csr32w(ctlr, Tdlen, ctlr->ntd * sizeof(Td));
 826         ctlr->tdh = PREV(0, ctlr->ntd);
 827         csr32w(ctlr, Tdh, 0);
 828         ctlr->tdt = 0;
 829         csr32w(ctlr, Tdt, 0);
 830         for(i = 0; i < ctlr->ntd; i++){
 831                 if((bp = ctlr->tb[i]) != nil){
 832                         ctlr->tb[i] = nil;
 833                         freeb(bp);
 834                 }
 835                 memset(&ctlr->tdba[i], 0, sizeof(Td));
 836         }
 837         csr32w(ctlr, Tidv, 128);
 838         r = csr32r(ctlr, Txdctl);
 839         r &= ~(WthreshMASK|PthreshMASK);
 840         r |= 4<<WthreshSHIFT | 4<<PthreshSHIFT;
 841         if(ctlr->type == i82575 || ctlr->type == i82576)
 842                 r |= Qenable;
 843         csr32w(ctlr, Tadv, 64);
 844         csr32w(ctlr, Txdctl, r);
 845         r = csr32r(ctlr, Tctl);
 846         r |= Ten;
 847         csr32w(ctlr, Tctl, r);
 848 //      if(ctlr->type == i82671)
 849 //              csr32w(ctlr, Tarc0, csr32r(ctlr, Tarc0) | 7<<24); /* yb sez? */
 850 }
 851
 852 #define Next(x, m)      (((x)+1) & (m))
 853
 854 static int
 855 i82563cleanup(Ctlr *c)
 856 {
 857         Block *b;
 858         int tdh, m, n;
 859
 860         tdh = c->tdh;
 861         m = c->ntd-1;
 862         while(c->tdba[n = Next(tdh, m)].status & Tdd){
 863                 tdh = n;
 864                 if((b = c->tb[tdh]) != nil){
 865                         c->tb[tdh] = nil;
 866                         freeb(b);
 867                 }else
 868                         iprint("82563 tx underrun!\n");
 869                 c->tdba[tdh].status = 0;
 870         }
 871
 872         return c->tdh = tdh;
 873 }
 874
 875 static void
 876 i82563transmit(Ether* edev)
 877 {
 878         Td *td;
 879         Block *bp;
 880         Ctlr *ctlr;
 881         int tdh, tdt, m;
 882
 883         ctlr = edev->ctlr;
 884
 885         qlock(&ctlr->tlock);
 886
 887         /*
 888          * Free any completed packets
 889          */
 890         tdh = i82563cleanup(ctlr);
 891
 892         /*
 893          * Try to fill the ring back up.
 894          */
 895         tdt = ctlr->tdt;
 896         m = ctlr->ntd-1;
 897         for(;;){
 898                 if(Next(tdt, m) == tdh){
 899                         ctlr->txdw++;
 900                         i82563im(ctlr, Txdw);
 901                         break;
 902                 }
 903                 if((bp = qget(edev->oq)) == nil)
 904                         break;
 905                 td = &ctlr->tdba[tdt];
 906                 td->addr[0] = PCIWADDR(bp->rp);
 907                 td->control = Ide|Rs|Ifcs|Teop|BLEN(bp);
 908                 ctlr->tb[tdt] = bp;
 909                 tdt = Next(tdt, m);
 910         }
 911         if(ctlr->tdt != tdt){
 912                 ctlr->tdt = tdt;
 913                 csr32w(ctlr, Tdt, tdt);
 914         }
 915         qunlock(&ctlr->tlock);
 916 }
 917
 918 static void
 919 i82563replenish(Ctlr* ctlr)
 920 {
 921         Rd *rd;
 922         int rdt, m;
 923         Block *bp;
 924
 925         rdt = ctlr->rdt;
 926         m = ctlr->nrd-1;
 927         while(Next(rdt, m) != ctlr->rdh){
 928                 rd = &ctlr->rdba[rdt];
 929                 if(ctlr->rb[rdt] != nil){
 930                         iprint("82563: tx overrun\n");
 931                         break;
 932                 }
 933                 bp = i82563rballoc();
 934                 if(bp == nil){
 935                         vlong now;
 936                         static vlong lasttime;
 937
 938                         /* don't flood the console */
 939                         now = tk2ms(MACHP(0)->ticks);
 940                         if (now - lasttime > 2000)
 941                                 iprint("#l%d: 82563: all %d rx buffers in use\n",
 942                                         ctlr->edev->ctlrno, ctlr->nrb);
 943                         lasttime = now;
 944                         break;
 945                 }
 946                 ctlr->rb[rdt] = bp;
 947                 rd->addr[0] = PCIWADDR(bp->rp);
 948 //              rd->addr[1] = 0;
 949                 rd->status = 0;
 950                 ctlr->rdfree++;
 951                 rdt = Next(rdt, m);
 952         }
 953         ctlr->rdt = rdt;
 954         csr32w(ctlr, Rdt, rdt);
 955 }
 956
 957 static void
 958 i82563rxinit(Ctlr* ctlr)
 959 {
 960         Block *bp;
 961         int i, r, rctl;
 962
 963         if(ctlr->rbsz <= 2048)
 964                 rctl = Dpf|Bsize2048|Bam|RdtmsHALF;
 965         else if(ctlr->rbsz <= 8192)
 966                 rctl = Lpe|Dpf|Bsize8192|Bsex|Bam|RdtmsHALF|Secrc;
 967         else if(ctlr->rbsz <= 12*1024){
 968                 i = ctlr->rbsz / 1024;
 969                 if(ctlr->rbsz % 1024)
 970                         i++;
 971                 rctl = Lpe|Dpf|BsizeFlex*i|Bam|RdtmsHALF|Secrc;
 972         }
 973         else
 974                 rctl = Lpe|Dpf|Bsize16384|Bsex|Bam|RdtmsHALF|Secrc;
 975
 976         if(ctlr->type == i82575 || ctlr->type == i82576){
 977                 /*
 978                  * Setting Qenable in Rxdctl does not
 979                  * appear to stick unless Ren is on.
 980                  */
 981                 csr32w(ctlr, Rctl, Ren|rctl);
 982                 r = csr32r(ctlr, Rxdctl);
 983                 r |= Qenable;
 984                 csr32w(ctlr, Rxdctl, r);
 985         }
 986         csr32w(ctlr, Rctl, rctl);
 987
 988         if(ctlr->type == i82573)
 989                 csr32w(ctlr, Ert, 1024/8);
 990
 991         if(ctlr->type == i82566 || ctlr->type == i82567)
 992                 csr32w(ctlr, Pbs, 16);
 993
 994         csr32w(ctlr, Rdbal, PCIWADDR(ctlr->rdba));
 995         csr32w(ctlr, Rdbah, 0);
 996         csr32w(ctlr, Rdlen, ctlr->nrd * sizeof(Rd));
 997         ctlr->rdh = 0;
 998         csr32w(ctlr, Rdh, 0);
 999         ctlr->rdt = 0;
1000         csr32w(ctlr, Rdt, 0);
1001         /* to hell with interrupt moderation, we've got fast cpus */
1002 //      ctlr->rdtr = 25;                /* µs units? */
1003 //      ctlr->radv = 500;               /* µs units? */
1004         ctlr->radv = ctlr->rdtr = 0;
1005         csr32w(ctlr, Rdtr, ctlr->rdtr);
1006         csr32w(ctlr, Radv, ctlr->radv);
1007
1008         for(i = 0; i < ctlr->nrd; i++){
1009                 if((bp = ctlr->rb[i]) != nil){
1010                         ctlr->rb[i] = nil;
1011                         freeb(bp);
1012                 }
1013         }
1014         i82563replenish(ctlr);
1015
1016         if(ctlr->type != i82575 || ctlr->type == i82576){
1017                 /*
1018                  * See comment above for Qenable.
1019                  * Could shuffle the code?
1020                  */
1021                 r = csr32r(ctlr, Rxdctl);
1022                 r &= ~(WthreshMASK|PthreshMASK);
1023                 r |= (2<<WthreshSHIFT)|(2<<PthreshSHIFT);
1024                 csr32w(ctlr, Rxdctl, r);
1025         }
1026
1027         /*
1028          * Don't enable checksum offload.  In practice, it interferes with
1029          * tftp booting on at least the 82575.
1030          */
1031 //      csr32w(ctlr, Rxcsum, Tuofl | Ipofl | ETHERHDRSIZE<<PcssSHIFT);
1032         csr32w(ctlr, Rxcsum, 0);
1033 }
1034
1035 static int
1036 i82563rim(void* ctlr)
1037 {
1038         return ((Ctlr*)ctlr)->rim != 0;
1039 }
1040
1041 static void
1042 i82563rproc(void* arg)
1043 {
1044         Rd *rd;
1045         Block *bp;
1046         Ctlr *ctlr;
1047         int r, m, rdh, rim;
1048         Ether *edev;
1049
1050         edev = arg;
1051         ctlr = edev->ctlr;
1052
1053         i82563rxinit(ctlr);
1054         r = csr32r(ctlr, Rctl);
1055         r |= Ren;
1056         csr32w(ctlr, Rctl, r);
1057         m = ctlr->nrd-1;
1058
1059         for(;;){
1060                 i82563im(ctlr, Rxt0|Rxo|Rxdmt0|Rxseq|Ack);
1061                 ctlr->rsleep++;
1062 //              coherence();
1063                 sleep(&ctlr->rrendez, i82563rim, ctlr);
1064
1065                 rdh = ctlr->rdh;
1066                 for(;;){
1067                         rd = &ctlr->rdba[rdh];
1068                         rim = ctlr->rim;
1069                         ctlr->rim = 0;
1070                         if(!(rd->status & Rdd))
1071                                 break;
1072
1073                         /*
1074                          * Accept eop packets with no errors.
1075                          * With no errors and the Ixsm bit set,
1076                          * the descriptor status Tpcs and Ipcs bits give
1077                          * an indication of whether the checksums were
1078                          * calculated and valid.
1079                          */
1080                         bp = ctlr->rb[rdh];
1081                         if((rd->status & Reop) && rd->errors == 0){
1082                                 bp->wp += rd->length;
1083                                 bp->lim = bp->wp;       /* lie like a dog. */
1084                                 if(!(rd->status & Ixsm)){
1085                                         ctlr->ixsm++;
1086                                         if(rd->status & Ipcs){
1087                                                 /*
1088                                                  * IP checksum calculated
1089                                                  * (and valid as errors == 0).
1090                                                  */
1091                                                 ctlr->ipcs++;
1092                                                 bp->flag |= Bipck;
1093                                         }
1094                                         if(rd->status & Tcpcs){
1095                                                 /*
1096                                                  * TCP/UDP checksum calculated
1097                                                  * (and valid as errors == 0).
1098                                                  */
1099                                                 ctlr->tcpcs++;
1100                                                 bp->flag |= Btcpck|Budpck;
1101                                         }
1102                                         bp->checksum = rd->checksum;
1103                                         bp->flag |= Bpktck;
1104                                 }
1105                                 etheriq(edev, bp, 1);
1106                         } else if (rd->status & Reop && rd->errors)
1107                                 print("%s: input packet error %#ux\n",
1108                                         tname[ctlr->type], rd->errors);
1109                         else
1110                                 freeb(bp);
1111                         ctlr->rb[rdh] = nil;
1112
1113                         rd->status = 0;
1114                         ctlr->rdfree--;
1115                         ctlr->rdh = rdh = Next(rdh, m);
1116                         if(ctlr->nrd-ctlr->rdfree >= 32 || (rim & Rxdmt0))
1117                                 i82563replenish(ctlr);
1118                 }
1119         }
1120 }
1121
1122 static int
1123 i82563lim(void* c)
1124 {
1125         return ((Ctlr*)c)->lim != 0;
1126 }
1127
1128 static int speedtab[] = {
1129         10, 100, 1000, 0
1130 };
1131
1132 static uint
1133 phyread(Ctlr *c, int reg)
1134 {
1135         uint phy, i;
1136
1137         csr32w(c, Mdic, MDIrop | 1<<MDIpSHIFT | reg<<MDIrSHIFT);
1138         phy = 0;
1139         for(i = 0; i < 64; i++){
1140                 phy = csr32r(c, Mdic);
1141                 if(phy & (MDIe|MDIready))
1142                         break;
1143                 microdelay(1);
1144         }
1145         if((phy & (MDIe|MDIready)) != MDIready)
1146                 return ~0;
1147         return phy & 0xffff;
1148 }
1149
1150 static uint
1151 phywrite(Ctlr *c, int reg, ushort val)
1152 {
1153         uint phy, i;
1154
1155         csr32w(c, Mdic, MDIwop | 1<<MDIpSHIFT | reg<<MDIrSHIFT | val);
1156         phy = 0;
1157         for(i = 0; i < 64; i++){
1158                 phy = csr32r(c, Mdic);
1159                 if(phy & (MDIe|MDIready))
1160                         break;
1161                 microdelay(1);
1162         }
1163         if((phy & (MDIe|MDIready)) != MDIready)
1164                 return ~0;
1165         return 0;
1166 }
1167
1168 /*
1169  * watch for changes of link state
1170  */
1171 static void
1172 i82563lproc(void *v)
1173 {
1174         uint phy, i, a;
1175         Ctlr *c;
1176         Ether *e;
1177
1178         e = v;
1179         c = e->ctlr;
1180
1181         if(c->type == i82573 && (phy = phyread(c, Phyier)) != ~0)
1182                 phywrite(c, Phyier, phy | Lscie | Ancie | Spdie | Panie);
1183         for(;;){
1184                 phy = phyread(c, Physsr);
1185                 if(phy == ~0)
1186                         goto next;
1187                 i = (phy>>14) & 3;
1188
1189                 switch(c->type){
1190                 case i82563:
1191                         a = phyread(c, Phyisr) & Ane;
1192                         break;
1193                 case i82571:
1194                 case i82572:
1195                         a = phyread(c, Phylhr) & Anf;
1196                         i = (i-1) & 3;
1197                         break;
1198                 default:
1199                         a = 0;
1200                         break;
1201                 }
1202                 if(a)
1203                         phywrite(c, Phyctl, phyread(c, Phyctl) | Ran | Ean);
1204                 e->link = (phy & Rtlink) != 0;
1205                 if(e->link){
1206                         c->speeds[i]++;
1207                         if (speedtab[i])
1208                                 e->mbps = speedtab[i];
1209                 }
1210 next:
1211                 c->lim = 0;
1212                 i82563im(c, Lsc);
1213                 c->lsleep++;
1214                 sleep(&c->lrendez, i82563lim, c);
1215         }
1216 }
1217
1218 static void
1219 i82563tproc(void *v)
1220 {
1221         Ether *e;
1222         Ctlr *c;
1223
1224         e = v;
1225         c = e->ctlr;
1226         for(;;){
1227                 sleep(&c->trendez, return0, 0);
1228                 i82563transmit(e);
1229         }
1230 }
1231
1232 static void
1233 i82563attach(Ether* edev)
1234 {
1235         Block *bp;
1236         Ctlr *ctlr;
1237         char name[KNAMELEN];
1238
1239         ctlr = edev->ctlr;
1240         ctlr->edev = edev;                      /* point back to Ether* */
1241         qlock(&ctlr->alock);
1242         if(ctlr->attached){
1243                 qunlock(&ctlr->alock);
1244                 return;
1245         }
1246
1247         ctlr->nrd = Nrd;
1248         ctlr->ntd = Ntd;
1249
1250         if(waserror()){
1251                 while(ctlr->nrb > 0){
1252                         bp = i82563rballoc();
1253                         bp->free = nil;
1254                         freeb(bp);
1255                         ctlr->nrb--;
1256                 }
1257                 free(ctlr->tb);
1258                 ctlr->tb = nil;
1259                 free(ctlr->rb);
1260                 ctlr->rb = nil;
1261                 free(ctlr->tdba);
1262                 ctlr->tdba = nil;
1263                 free(ctlr->rdba);
1264                 ctlr->rdba = nil;
1265                 qunlock(&ctlr->alock);
1266                 nexterror();
1267         }
1268
1269         if((ctlr->rdba = mallocalign(ctlr->nrd*sizeof(Rd), 128, 0, 0)) == nil ||
1270            (ctlr->tdba = mallocalign(ctlr->ntd*sizeof(Td), 128, 0, 0)) == nil ||
1271            (ctlr->rb = malloc(ctlr->nrd*sizeof(Block*))) == nil ||
1272            (ctlr->tb = malloc(ctlr->ntd*sizeof(Block*))) == nil)
1273                 error(Enomem);
1274
1275         for(ctlr->nrb = 0; ctlr->nrb < Nrb; ctlr->nrb++){
1276                 if((bp = allocb(ctlr->rbsz + BY2PG)) == nil)
1277                         break;
1278                 bp->free = i82563rbfree;
1279                 freeb(bp);
1280         }
1281
1282         ctlr->attached = 1;
1283
1284         snprint(name, sizeof name, "#l%dl", edev->ctlrno);
1285         kproc(name, i82563lproc, edev);
1286
1287         snprint(name, sizeof name, "#l%dr", edev->ctlrno);
1288         kproc(name, i82563rproc, edev);
1289
1290         snprint(name, sizeof name, "#l%dt", edev->ctlrno);
1291         kproc(name, i82563tproc, edev);
1292
1293         i82563txinit(ctlr);
1294
1295         qunlock(&ctlr->alock);
1296         poperror();
1297 }
1298
1299 static void
1300 i82563interrupt(Ureg*, void* arg)
1301 {
1302         Ctlr *ctlr;
1303         Ether *edev;
1304         int icr, im;
1305
1306         edev = arg;
1307         ctlr = edev->ctlr;
1308
1309         ilock(&ctlr->imlock);
1310         csr32w(ctlr, Imc, ~0);
1311         im = ctlr->im;
1312
1313         for(icr = csr32r(ctlr, Icr); icr & ctlr->im; icr = csr32r(ctlr, Icr)){
1314                 if(icr & Lsc){
1315                         im &= ~Lsc;
1316                         ctlr->lim = icr & Lsc;
1317                         wakeup(&ctlr->lrendez);
1318                         ctlr->lintr++;
1319                 }
1320                 if(icr & (Rxt0|Rxo|Rxdmt0|Rxseq|Ack)){
1321                         ctlr->rim = icr & (Rxt0|Rxo|Rxdmt0|Rxseq|Ack);
1322                         im &= ~(Rxt0|Rxo|Rxdmt0|Rxseq|Ack);
1323                         wakeup(&ctlr->rrendez);
1324                         ctlr->rintr++;
1325                 }
1326                 if(icr & Txdw){
1327                         im &= ~Txdw;
1328                         ctlr->tintr++;
1329                         wakeup(&ctlr->trendez);
1330                 }
1331         }
1332
1333         ctlr->im = im;
1334         csr32w(ctlr, Ims, im);
1335         iunlock(&ctlr->imlock);
1336 }
1337
1338 /* assume misrouted interrupts and check all controllers */
1339 static void
1340 i82575interrupt(Ureg*, void *)
1341 {
1342         Ctlr *ctlr;
1343
1344         for (ctlr = i82563ctlrhead; ctlr != nil; ctlr = ctlr->next)
1345                 i82563interrupt(nil, ctlr->edev);
1346 }
1347
1348 static int
1349 i82563detach(Ctlr* ctlr)
1350 {
1351         int r, timeo;
1352
1353         /*
1354          * Perform a device reset to get the chip back to the
1355          * power-on state, followed by an EEPROM reset to read
1356          * the defaults for some internal registers.
1357          */
1358         csr32w(ctlr, Imc, ~0);
1359         csr32w(ctlr, Rctl, 0);
1360         csr32w(ctlr, Tctl, 0);
1361
1362         delay(10);
1363
1364         r = csr32r(ctlr, Ctrl);
1365         if(ctlr->type == i82566 || ctlr->type == i82567)
1366                 r |= Phyrst;
1367         csr32w(ctlr, Ctrl, Devrst | r);
1368         delay(1);
1369         for(timeo = 0; timeo < 1000; timeo++){
1370                 if(!(csr32r(ctlr, Ctrl) & Devrst))
1371                         break;
1372                 delay(1);
1373         }
1374         if(csr32r(ctlr, Ctrl) & Devrst)
1375                 return -1;
1376
1377         r = csr32r(ctlr, Ctrlext);
1378         csr32w(ctlr, Ctrlext, r|Eerst);
1379         delay(1);
1380         for(timeo = 0; timeo < 1000; timeo++){
1381                 if(!(csr32r(ctlr, Ctrlext) & Eerst))
1382                         break;
1383                 delay(1);
1384         }
1385         if(csr32r(ctlr, Ctrlext) & Eerst)
1386                 return -1;
1387
1388         csr32w(ctlr, Imc, ~0);
1389         delay(1);
1390         for(timeo = 0; timeo < 1000; timeo++){
1391                 if(!csr32r(ctlr, Icr))
1392                         break;
1393                 delay(1);
1394         }
1395         if(csr32r(ctlr, Icr))
1396                 return -1;
1397
1398         /*
1399          * Balance Rx/Tx packet buffer.
1400          * No need to set PBA register unless using jumbo, defaults to 32KB
1401          * for receive. If it is changed, then have to do a MAC reset,
1402          * and need to do that at the the right time as it will wipe stuff.
1403          */
1404         if(ctlr->rbsz > 8192 && (ctlr->type == i82563 || ctlr->type == i82571 ||
1405             ctlr->type == i82572)){
1406                 ctlr->pba = csr32r(ctlr, Pba);
1407                 r = ctlr->pba >> 16;
1408                 r += ctlr->pba & 0xffff;
1409                 r >>= 1;
1410                 csr32w(ctlr, Pba, r);
1411         } else if(ctlr->type == i82573 && ctlr->rbsz > 1514)
1412                 csr32w(ctlr, Pba, 14);
1413         ctlr->pba = csr32r(ctlr, Pba);
1414
1415         r = csr32r(ctlr, Ctrl);
1416         csr32w(ctlr, Ctrl, Slu|r);
1417
1418         return 0;
1419 }
1420
1421 static void
1422 i82563shutdown(Ether* ether)
1423 {
1424         i82563detach(ether->ctlr);
1425 }
1426
1427 static ushort
1428 eeread(Ctlr *ctlr, int adr)
1429 {
1430         csr32w(ctlr, Eerd, EEstart | adr << 2);
1431         while ((csr32r(ctlr, Eerd) & EEdone) == 0)
1432                 ;
1433         return csr32r(ctlr, Eerd) >> 16;
1434 }
1435
1436 static int
1437 eeload(Ctlr *ctlr)
1438 {
1439         ushort sum;
1440         int data, adr;
1441
1442         sum = 0;
1443         for (adr = 0; adr < 0x40; adr++) {
1444                 data = eeread(ctlr, adr);
1445                 ctlr->eeprom[adr] = data;
1446                 sum += data;
1447         }
1448         return sum;
1449 }
1450
1451 static int
1452 fcycle(Ctlr *, Flash *f)
1453 {
1454         ushort s, i;
1455
1456         s = f->reg[Fsts];
1457         if((s&Fvalid) == 0)
1458                 return -1;
1459         f->reg[Fsts] |= Fcerr | Ael;
1460         for(i = 0; i < 10; i++){
1461                 if((s&Scip) == 0)
1462                         return 0;
1463                 delay(1);
1464                 s = f->reg[Fsts];
1465         }
1466         return -1;
1467 }
1468
1469 static int
1470 fread(Ctlr *c, Flash *f, int ladr)
1471 {
1472         ushort s;
1473
1474         delay(1);
1475         if(fcycle(c, f) == -1)
1476                 return -1;
1477         f->reg[Fsts] |= Fdone;
1478         f->reg32[Faddr] = ladr;
1479
1480         /* setup flash control register */
1481         s = f->reg[Fctl];
1482         s &= ~(0x1f << 8);
1483         s |= (2-1) << 8;                /* 2 bytes */
1484         s &= ~(2*Flcycle);              /* read */
1485         f->reg[Fctl] = s | Fgo;
1486
1487         while((f->reg[Fsts] & Fdone) == 0)
1488                 ;
1489         if(f->reg[Fsts] & (Fcerr|Ael))
1490                 return -1;
1491         return f->reg32[Fdata] & 0xffff;
1492 }
1493
1494 static int
1495 fload(Ctlr *c)
1496 {
1497         ulong data, io, r, adr;
1498         ushort sum;
1499         Flash f;
1500
1501         io = c->pcidev->mem[1].bar & ~0x0f;
1502         f.reg = vmap(io, c->pcidev->mem[1].size);
1503         if(f.reg == nil)
1504                 return -1;
1505         f.reg32 = (void*)f.reg;
1506         f.sz = f.reg32[Bfpr];
1507         r = f.sz & 0x1fff;
1508         if(csr32r(c, Eec) & (1<<22))
1509                 ++r;
1510         r <<= 12;
1511
1512         sum = 0;
1513         for (adr = 0; adr < 0x40; adr++) {
1514                 data = fread(c, &f, r + adr*2);
1515                 if(data == -1)
1516                         break;
1517                 c->eeprom[adr] = data;
1518                 sum += data;
1519         }
1520         vunmap(f.reg, c->pcidev->mem[1].size);
1521         return sum;
1522 }
1523
1524 static int
1525 i82563reset(Ctlr *ctlr)
1526 {
1527         int i, r;
1528
1529         if(i82563detach(ctlr))
1530                 return -1;
1531         if(ctlr->type == i82566 || ctlr->type == i82567)
1532                 r = fload(ctlr);
1533         else
1534                 r = eeload(ctlr);
1535         if (r != 0 && r != 0xBABA){
1536                 print("%s: bad EEPROM checksum - %#.4ux\n",
1537                         tname[ctlr->type], r);
1538                 return -1;
1539         }
1540
1541         for(i = 0; i < Eaddrlen/2; i++){
1542                 ctlr->ra[2*i]   = ctlr->eeprom[Ea+i];
1543                 ctlr->ra[2*i+1] = ctlr->eeprom[Ea+i] >> 8;
1544         }
1545         r = (csr32r(ctlr, Status) & Lanid) >> 2;
1546         ctlr->ra[5] += r;               /* ea ctlr[1] = ea ctlr[0]+1 */
1547
1548         r = ctlr->ra[3]<<24 | ctlr->ra[2]<<16 | ctlr->ra[1]<<8 | ctlr->ra[0];
1549         csr32w(ctlr, Ral, r);
1550         r = 0x80000000 | ctlr->ra[5]<<8 | ctlr->ra[4];
1551         csr32w(ctlr, Rah, r);
1552         for(i = 1; i < 16; i++){
1553                 csr32w(ctlr, Ral+i*8, 0);
1554                 csr32w(ctlr, Rah+i*8, 0);
1555         }
1556         memset(ctlr->mta, 0, sizeof(ctlr->mta));
1557         for(i = 0; i < 128; i++)
1558                 csr32w(ctlr, Mta + i*4, 0);
1559
1560         /*
1561          * Does autonegotiation affect this manual setting?
1562          * The correct values here should depend on the PBA value
1563          * and maximum frame length, no?
1564          * ctlr->fcrt[lh] are never set, so default to 0.
1565          */
1566         csr32w(ctlr, Fcal, 0x00C28001);
1567         csr32w(ctlr, Fcah, 0x0100);
1568         csr32w(ctlr, Fct, 0x8808);
1569         csr32w(ctlr, Fcttv, 0x0100);
1570
1571         ctlr->fcrtl = ctlr->fcrth = 0;
1572         // ctlr->fcrtl = 0x00002000;
1573         // ctlr->fcrth = 0x00004000;
1574         csr32w(ctlr, Fcrtl, ctlr->fcrtl);
1575         csr32w(ctlr, Fcrth, ctlr->fcrth);
1576
1577         return 0;
1578 }
1579
1580 static void
1581 i82563pci(void)
1582 {
1583         int type;
1584         ulong io;
1585         void *mem;
1586         Pcidev *p;
1587         Ctlr *ctlr;
1588
1589         p = nil;
1590         while(p = pcimatch(p, 0x8086, 0)){
1591                 switch(p->did){
1592                 default:
1593                         continue;
1594                 case 0x1096:
1595                 case 0x10ba:
1596                         type = i82563;
1597                         break;
1598                 case 0x1049:            /* mm */
1599                 case 0x104a:            /* dm */
1600                 case 0x104b:            /* dc */
1601                 case 0x104d:            /* mc */
1602                 case 0x10bd:            /* dm */
1603                 case 0x294c:            /* dc-2 */
1604                         type = i82566;
1605                         break;
1606                 case 0x10cd:            /* lf */
1607                 case 0x10ce:            /* v-2 */
1608                 case 0x10de:            /* lm-3 */
1609                 case 0x10f5:            /* lm-2 */
1610                         type = i82567;
1611                         break;
1612                 case 0x10a4:
1613                 case 0x105e:
1614                         type = i82571;
1615                         break;
1616                 case 0x10b9:            /* sic, 82572gi */
1617                         type = i82572;
1618                         break;
1619                 case 0x108b:            /*  v */
1620                 case 0x108c:            /*  e (iamt) */
1621                 case 0x109a:            /*  l */
1622                         type = i82573;
1623                         break;
1624 //              case 0x10d3:            /* l */
1625 //                      type = i82574;  /* never heard of it */
1626 //                      break;
1627                 case 0x10a7:    /* 82575eb: one of a pair of controllers */
1628                         type = i82575;
1629                         break;
1630                 case 0x10c9:            /* 82576 copper */
1631                 case 0x10e6:            /* 82576 fiber */
1632                 case 0x10e7:            /* 82576 serdes */
1633                         type = i82576;
1634                         break;
1635                 }
1636
1637                 io = p->mem[0].bar & ~0x0F;
1638                 mem = vmap(io, p->mem[0].size);
1639                 if(mem == nil){
1640                         print("%s: can't map %.8lux\n", tname[type], io);
1641                         continue;
1642                 }
1643                 ctlr = malloc(sizeof(Ctlr));
1644                 ctlr->port = io;
1645                 ctlr->pcidev = p;
1646                 ctlr->type = type;
1647                 ctlr->rbsz = rbtab[type];
1648                 ctlr->nic = mem;
1649
1650                 if(i82563reset(ctlr)){
1651                         vunmap(mem, p->mem[0].size);
1652                         free(ctlr);
1653                         continue;
1654                 }
1655                 pcisetbme(p);
1656
1657                 if(i82563ctlrhead != nil)
1658                         i82563ctlrtail->next = ctlr;
1659                 else
1660                         i82563ctlrhead = ctlr;
1661                 i82563ctlrtail = ctlr;
1662         }
1663 }
1664
1665 static int
1666 pnp(Ether* edev, int type)
1667 {
1668         Ctlr *ctlr;
1669         static int done;
1670
1671         if(!done) {
1672                 i82563pci();
1673                 done = 1;
1674         }
1675
1676         /*
1677          * Any adapter matches if no edev->port is supplied,
1678          * otherwise the ports must match.
1679          */
1680         for(ctlr = i82563ctlrhead; ctlr != nil; ctlr = ctlr->next){
1681                 if(ctlr->active)
1682                         continue;
1683                 if(type != Iany && ctlr->type != type)
1684                         continue;
1685                 if(edev->port == 0 || edev->port == ctlr->port){
1686                         ctlr->active = 1;
1687                         break;
1688                 }
1689         }
1690         if(ctlr == nil)
1691                 return -1;
1692
1693         edev->ctlr = ctlr;
1694         ctlr->edev = edev;                      /* point back to Ether* */
1695         edev->port = ctlr->port;
1696         edev->irq = ctlr->pcidev->intl;
1697         edev->tbdf = ctlr->pcidev->tbdf;
1698         edev->mbps = 1000;
1699         edev->maxmtu = ctlr->rbsz;
1700         memmove(edev->ea, ctlr->ra, Eaddrlen);
1701
1702         /*
1703          * Linkage to the generic ethernet driver.
1704          */
1705         edev->attach = i82563attach;
1706         edev->transmit = i82563transmit;
1707         edev->interrupt = (ctlr->type == i82575?
1708                 i82575interrupt: i82563interrupt);
1709         edev->ifstat = i82563ifstat;
1710         edev->ctl = i82563ctl;
1711
1712         edev->arg = edev;
1713         edev->promiscuous = i82563promiscuous;
1714         edev->shutdown = i82563shutdown;
1715         edev->multicast = i82563multicast;
1716
1717         return 0;
1718 }
1719
1720 static int
1721 anypnp(Ether *e)
1722 {
1723         return pnp(e, Iany);
1724 }
1725
1726 static int
1727 i82563pnp(Ether *e)
1728 {
1729         return pnp(e, i82563);
1730 }
1731
1732 static int
1733 i82566pnp(Ether *e)
1734 {
1735         return pnp(e, i82566);
1736 }
1737
1738 static int
1739 i82571pnp(Ether *e)
1740 {
1741         return pnp(e, i82571);
1742 }
1743
1744 static int
1745 i82572pnp(Ether *e)
1746 {
1747         return pnp(e, i82572);
1748 }
1749
1750 static int
1751 i82573pnp(Ether *e)
1752 {
1753         return pnp(e, i82573);
1754 }
1755
1756 static int
1757 i82575pnp(Ether *e)
1758 {
1759         return pnp(e, i82575);
1760 }
1761
1762 void
1763 ether82563link(void)
1764 {
1765         /* recognise lots of model numbers for debugging assistance */
1766         addethercard("i82563", i82563pnp);
1767         addethercard("i82566", i82566pnp);
1768         addethercard("i82571", i82571pnp);
1769         addethercard("i82572", i82572pnp);
1770         addethercard("i82573", i82573pnp);
1771         addethercard("i82575", i82575pnp);
1772         addethercard("igbepcie", anypnp);
1773 }