[ena] Provide a host information page

Some versions of the ENA firmware (observed on a c6i.large instance in
eu-west-2) seem to require a host information page, without which the
CREATE_CQ command will fail with ENA_ADMIN_UNKNOWN_ERROR.

These firmware versions also seem to require us to claim that we are a
Linux kernel with a specific driver major version number.  This
appears to be a firmware bug, as revealed by Linux kernel commit
1a63443af ("net/amazon: Ensure that driver version is aligned to the
linux kernel"): this commit changed the value of the driver version
number field to be the Linux kernel version, and was hastily reverted
in commit 92040c6da ("net: ena: fix broken interface between ENA
driver and FW") which clarified that the version number field does
actually have some undocumented significance to some versions of the
firmware.

Fix by providing a host information page via the SET_FEATURE command,
incorporating the apparently necessary lies about our identity.

Signed-off-by: Michael Brown <mcb30@ipxe.org>

diff --git a/src/drivers/net/ena.c b/src/drivers/net/ena.c
index bad0238fe1c5880af836711e8073fe5052fbd3b1..c2a48a27d0d52b3f0eb2aea739f2893d8845b176 100644 (file)
--- a/src/drivers/net/ena.c
+++ b/src/drivers/net/ena.c
@@ -24,6 +24,7 @@
 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 
 #include <stdint.h>
+#include <stdio.h>
 #include <string.h>
 #include <unistd.h>
 #include <errno.h>
@@ -34,6 +35,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/iobuf.h>
 #include <ipxe/malloc.h>
 #include <ipxe/pci.h>
+#include <ipxe/version.h>
 #include "ena.h"
 
 /** @file
@@ -607,6 +609,32 @@ static int ena_get_device_attributes ( struct net_device *netdev ) {
        return 0;
 }
 
+/**
+ * Set host attributes
+ *
+ * @v ena              ENA device
+ * @ret rc             Return status code
+ */
+static int ena_set_host_attributes ( struct ena_nic *ena ) {
+       union ena_aq_req *req;
+       union ena_acq_rsp *rsp;
+       union ena_feature *feature;
+       int rc;
+
+       /* Construct request */
+       req = ena_admin_req ( ena );
+       req->header.opcode = ENA_SET_FEATURE;
+       req->set_feature.id = ENA_HOST_ATTRIBUTES;
+       feature = &req->set_feature.feature;
+       feature->host.info = cpu_to_le64 ( virt_to_bus ( ena->info ) );
+
+       /* Issue request */
+       if ( ( rc = ena_admin ( ena, req, &rsp ) ) != 0 )
+               return rc;
+
+       return 0;
+}
+
 /**
  * Get statistics (for debugging)
  *
@@ -965,6 +993,7 @@ static struct net_device_operations ena_operations = {
 static int ena_probe ( struct pci_device *pci ) {
        struct net_device *netdev;
        struct ena_nic *ena;
+       struct ena_host_info *info;
        int rc;
 
        /* Allocate and initialise net device */
@@ -998,6 +1027,25 @@ static int ena_probe ( struct pci_device *pci ) {
                goto err_ioremap;
        }
 
+       /* Allocate and initialise host info */
+       info = malloc_phys ( PAGE_SIZE, PAGE_SIZE );
+       if ( ! info ) {
+               rc = -ENOMEM;
+               goto err_info;
+       }
+       ena->info = info;
+       memset ( info, 0, PAGE_SIZE );
+       info->type = cpu_to_le32 ( ENA_HOST_INFO_TYPE_LINUX );
+       snprintf ( info->dist_str, sizeof ( info->dist_str ), "%s",
+                  ( product_name[0] ? product_name : product_short_name ) );
+       snprintf ( info->kernel_str, sizeof ( info->kernel_str ), "%s",
+                  product_version );
+       info->version = cpu_to_le32 ( ENA_HOST_INFO_VERSION_WTF );
+       info->spec = cpu_to_le16 ( ENA_HOST_INFO_SPEC_2_0 );
+       info->busdevfn = cpu_to_le16 ( pci->busdevfn );
+       DBGC2 ( ena, "ENA %p host info:\n", ena );
+       DBGC2_HDA ( ena, virt_to_phys ( info ), info, sizeof ( *info ) );
+
        /* Reset the NIC */
        if ( ( rc = ena_reset ( ena ) ) != 0 )
                goto err_reset;
@@ -1006,6 +1054,10 @@ static int ena_probe ( struct pci_device *pci ) {
        if ( ( rc = ena_create_admin ( ena ) ) != 0 )
                goto err_create_admin;
 
+       /* Set host attributes */
+       if ( ( rc = ena_set_host_attributes ( ena ) ) != 0 )
+               goto err_set_host_attributes;
+
        /* Fetch MAC address */
        if ( ( rc = ena_get_device_attributes ( netdev ) ) != 0 )
                goto err_get_device_attributes;
@@ -1024,10 +1076,13 @@ static int ena_probe ( struct pci_device *pci ) {
        unregister_netdev ( netdev );
  err_register_netdev:
  err_get_device_attributes:
+ err_set_host_attributes:
        ena_destroy_admin ( ena );
  err_create_admin:
        ena_reset ( ena );
  err_reset:
+       free_phys ( ena->info, PAGE_SIZE );
+ err_info:
        iounmap ( ena->regs );
  err_ioremap:
        netdev_nullify ( netdev );
@@ -1054,6 +1109,9 @@ static void ena_remove ( struct pci_device *pci ) {
        /* Reset card */
        ena_reset ( ena );
 
+       /* Free host info */
+       free_phys ( ena->info, PAGE_SIZE );
+
        /* Free network device */
        iounmap ( ena->regs );
        netdev_nullify ( netdev );

diff --git a/src/drivers/net/ena.h b/src/drivers/net/ena.h
index 78693e678d0d1ce30cd7b5a8c4642ec179c2f81c..cbee1e768a97ac82311934993717cad06b8f6233 100644 (file)
--- a/src/drivers/net/ena.h
+++ b/src/drivers/net/ena.h
@@ -127,10 +127,86 @@ struct ena_device_attributes {
        uint32_t mtu;
 } __attribute__ (( packed ));
 
+/** Host attributes */
+#define ENA_HOST_ATTRIBUTES 28
+
+/** Host attributes */
+struct ena_host_attributes {
+       /** Host info base address */
+       uint64_t info;
+       /** Debug area base address */
+       uint64_t debug;
+       /** Debug area size */
+       uint32_t debug_len;
+} __attribute__ (( packed ));
+
+/** Host information */
+struct ena_host_info {
+       /** Operating system type */
+       uint32_t type;
+       /** Operating system distribution (string) */
+       char dist_str[128];
+       /** Operating system distribution (numeric) */
+       uint32_t dist;
+       /** Kernel version (string) */
+       char kernel_str[32];
+       /** Kernel version (numeric) */
+       uint32_t kernel;
+       /** Driver version */
+       uint32_t version;
+       /** Linux network device features */
+       uint64_t linux_features;
+       /** ENA specification version */
+       uint16_t spec;
+       /** PCI bus:dev.fn address */
+       uint16_t busdevfn;
+       /** Number of CPUs */
+       uint16_t cpus;
+       /** Reserved */
+       uint8_t reserved_a[2];
+       /** Supported features */
+       uint32_t features;
+} __attribute__ (( packed ));
+
+/** Linux operating system type
+ *
+ * There is a defined "iPXE" operating system type (with value 5).
+ * However, some very broken versions of the ENA firmware will refuse
+ * to allow a completion queue to be created if the "iPXE" type is
+ * used.
+ */
+#define ENA_HOST_INFO_TYPE_LINUX 1
+
+/** Driver version
+ *
+ * The driver version field is nominally used to report a version
+ * number outside of the VM for consumption by humans (and potentially
+ * by automated monitoring tools that could e.g. check for outdated
+ * versions with known security flaws).
+ *
+ * However, at some point in the development of the ENA firmware, some
+ * unknown person at AWS thought it would be sensible to apply a
+ * machine interpretation to this field and adjust the behaviour of
+ * the firmware based on its value, thereby creating a maintenance and
+ * debugging nightmare for all existing and future drivers.
+ *
+ * Hint to engineers: if you ever find yourself writing code of the
+ * form "if (version == SOME_MAGIC_NUMBER)" then something has gone
+ * very, very wrong.  This *always* indicates that something is
+ * broken, either in your own code or in the code with which you are
+ * forced to interact.
+ */
+#define ENA_HOST_INFO_VERSION_WTF 0x00000002UL
+
+/** ENA specification version */
+#define ENA_HOST_INFO_SPEC_2_0 0x0200
+
 /** Feature */
 union ena_feature {
        /** Device attributes */
        struct ena_device_attributes device;
+       /** Host attributes */
+       struct ena_host_attributes host;
 };
 
 /** Submission queue direction */
@@ -300,6 +376,27 @@ struct ena_get_feature_rsp {
        union ena_feature feature;
 } __attribute__ (( packed ));
 
+/** Set feature */
+#define ENA_SET_FEATURE 9
+
+/** Set feature request */
+struct ena_set_feature_req {
+       /** Header */
+       struct ena_aq_header header;
+       /** Length */
+       uint32_t len;
+       /** Address */
+       uint64_t address;
+       /** Flags */
+       uint8_t flags;
+       /** Feature identifier */
+       uint8_t id;
+       /** Reserved */
+       uint8_t reserved[2];
+       /** Feature */
+       union ena_feature feature;
+} __attribute__ (( packed ));
+
 /** Get statistics */
 #define ENA_GET_STATS 11
 
@@ -360,6 +457,8 @@ union ena_aq_req {
        struct ena_destroy_cq_req destroy_cq;
        /** Get feature */
        struct ena_get_feature_req get_feature;
+       /** Set feature */
+       struct ena_set_feature_req set_feature;
        /** Get statistics */
        struct ena_get_stats_req get_stats;
        /** Padding */
@@ -590,6 +689,8 @@ struct ena_qp {
 struct ena_nic {
        /** Registers */
        void *regs;
+       /** Host info */
+       struct ena_host_info *info;
        /** Admin queue */
        struct ena_aq aq;
        /** Admin completion queue */