From 1a573123d7b70bba918c34615e4d0fceabe70d48 Mon Sep 17 00:00:00 2001 From: Alireza Sanaee Date: Thu, 11 Jan 2024 23:24:27 +0000 Subject: [PATCH 1/5] upgrading to dpdk-23.11 --- src/core/drivers/dpdk/pmd.cc | 26 +++++++++++++------------- src/include/channel.h | 1 + src/include/dpdk.h | 35 ++++++++++++++++++++++++++++------- 3 files changed, 42 insertions(+), 20 deletions(-) diff --git a/src/core/drivers/dpdk/pmd.cc b/src/core/drivers/dpdk/pmd.cc index d685ad3..d53a1c4 100644 --- a/src/core/drivers/dpdk/pmd.cc +++ b/src/core/drivers/dpdk/pmd.cc @@ -18,7 +18,7 @@ __attribute__((unused)) static void ReportLinkStatus(uint8_t port_id) { LOG(WARNING) << "rte_eth_link_get_nowait() failed."; } - if (link.link_status == ETH_LINK_UP) { + if (link.link_status == RTE_ETH_LINK_UP) { LOG(INFO) << "[PMDPORT: " << static_cast(port_id) << "] " << "Link is UP " << link.link_speed << (link.link_autoneg ? "(AutoNeg)" : "(Fixed)") @@ -38,18 +38,18 @@ static rte_eth_conf DefaultEthConf(const rte_eth_dev_info *devinfo) { // offloads so return a very basic ethernet configuration. if (std::string(devinfo->driver_name) == "net_null") return port_conf; - port_conf.link_speeds = ETH_LINK_SPEED_AUTONEG; - uint64_t rss_hf = ETH_RSS_IP | ETH_RSS_UDP | ETH_RSS_TCP | ETH_RSS_SCTP; + port_conf.link_speeds = RTE_ETH_LINK_SPEED_AUTONEG; + uint64_t rss_hf = RTE_ETH_RSS_IP | RTE_ETH_RSS_UDP | RTE_ETH_RSS_TCP | RTE_ETH_RSS_SCTP; if (devinfo->flow_type_rss_offloads) { rss_hf &= devinfo->flow_type_rss_offloads; } port_conf.lpbk_mode = 1; - port_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; + port_conf.rxmode.mq_mode = RTE_ETH_MQ_RX_RSS; port_conf.rxmode.mtu = PmdRing::kDefaultFrameSize; port_conf.rxmode.max_lro_pkt_size = PmdRing::kDefaultFrameSize; - port_conf.rxmode.split_hdr_size = 0; + // port_conf.rxmode.split_hdr_size = 0; const auto rx_offload_capa = devinfo->rx_offload_capa; port_conf.rxmode.offloads |= ((RTE_ETH_RX_OFFLOAD_CHECKSUM)&rx_offload_capa); @@ -60,21 +60,21 @@ static rte_eth_conf DefaultEthConf(const rte_eth_dev_info *devinfo) { }; const auto tx_offload_capa = devinfo->tx_offload_capa; - if (!(tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM) || - !(tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM)) { + if (!(tx_offload_capa & RTE_ETH_TX_OFFLOAD_IPV4_CKSUM) || + !(tx_offload_capa & RTE_ETH_TX_OFFLOAD_UDP_CKSUM)) { // Making this fatal; not sure what NIC does not support checksum offloads. LOG(FATAL) << "Hardware does not support checksum offloads."; } - port_conf.txmode.mq_mode = ETH_MQ_TX_NONE; + port_conf.txmode.mq_mode = RTE_ETH_MQ_TX_NONE; port_conf.txmode.offloads = - (DEV_TX_OFFLOAD_IPV4_CKSUM | DEV_TX_OFFLOAD_UDP_CKSUM); + (RTE_ETH_TX_OFFLOAD_IPV4_CKSUM | RTE_ETH_TX_OFFLOAD_UDP_CKSUM); - if (tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE) { + if (tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) { // TODO(ilias): Add option to the constructor to enable this offload. LOG(WARNING) << "Enabling FAST FREE: use always the same mempool for each queue."; - port_conf.txmode.offloads |= DEV_TX_OFFLOAD_MBUF_FAST_FREE; + port_conf.txmode.offloads |= RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; } return port_conf; @@ -282,7 +282,7 @@ void PmdPort::InitDriver(uint16_t mtu) { struct rte_eth_link link; memset(&link, '0', sizeof(link)); int nsecs = 30; - while (nsecs-- && link.link_status == ETH_LINK_DOWN) { + while (nsecs-- && link.link_status == RTE_ETH_LINK_DOWN) { memset(&link, '0', sizeof(link)); int ret = rte_eth_link_get_nowait(port_id_, &link); if (ret != 0) { @@ -292,7 +292,7 @@ void PmdPort::InitDriver(uint16_t mtu) { sleep(1); } - if (link.link_status == ETH_LINK_UP) { + if (link.link_status == RTE_ETH_LINK_UP) { LOG(INFO) << "[PMDPORT: " << static_cast(port_id_) << "] " << "Link is UP " << link.link_speed << (link.link_autoneg ? " (AutoNeg)" : " (Fixed)") diff --git a/src/include/channel.h b/src/include/channel.h index 0b53532..ea6ab85 100644 --- a/src/include/channel.h +++ b/src/include/channel.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include diff --git a/src/include/dpdk.h b/src/include/dpdk.h index 35bec44..f087be0 100644 --- a/src/include/dpdk.h +++ b/src/include/dpdk.h @@ -4,14 +4,20 @@ #include #include #include +// #include +// #include +// #include #include -#include +// #include +#include +#include #include #include #include #include #include +#include namespace juggler { namespace dpdk { @@ -19,6 +25,10 @@ namespace dpdk { [[maybe_unused]] static void FetchDpdkPortInfo( uint8_t port_id, struct rte_eth_dev_info *devinfo, juggler::net::Ethernet::Address *lladdr, std::string *pci_string) { + + // struct ethtool_drvinfo drvinfo; + + if (!rte_eth_dev_is_valid_port(port_id)) { LOG(INFO) << "Port id " << static_cast(port_id) << " is not valid."; return; @@ -37,12 +47,23 @@ namespace dpdk { reinterpret_cast(lladdr->bytes)); struct rte_bus *bus = rte_bus_find_by_device(devinfo->device); - if (bus && !strcmp(bus->name, "pci")) { - const struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(devinfo->device); - *pci_string = juggler::utils::Format( - "%08x:%02hhx:%02hhx.%02hhx %04hx:%04hx", pci_dev->addr.domain, - pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function, - pci_dev->id.vendor_id, pci_dev->id.device_id); + // if (bus && !strcmp(bus->name, "pci")) { + // const struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(devinfo->device); + // *pci_string = juggler::utils::Format( + // "%08x:%02hhx:%02hhx.%02hhx %04hx:%04hx", pci_dev->addr.domain, + // pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function, + // pci_dev->id.vendor_id, pci_dev->id.device_id); + if (bus) { + // strlcpy(drvinfo.driver, devinfo->driver_name, + // sizeof(drvinfo.driver)); + // strlcpy(drvinfo.version, rte_version(), sizeof(drvinfo.version)); + // strlcpy(drvinfo.bus_info, rte_dev_name(devinfo->device), + // sizeof(drvinfo.bus_info)); + // const struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(devinfo->device); + // *pci_string = juggler::utils::Format( + // "%08x:%02hhx:%02hhx.%02hhx %04hx:%04hx", pci_dev->addr.domain, + // pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function, + // pci_dev->id.vendor_id, pci_dev->id.device_id); } LOG(INFO) << "[PMDPORT] [port_id: " << static_cast(port_id) From 9c79c9b1f45f5ea3a06064dba64d35f33719b080 Mon Sep 17 00:00:00 2001 From: Alireza Sanaee Date: Fri, 12 Jan 2024 18:53:58 +0000 Subject: [PATCH 2/5] remove pci info string literal due to lack of clean API in DPDK. --- src/core/drivers/dpdk/pmd.cc | 8 +++----- src/include/dpdk.h | 36 +++++------------------------------- src/include/pmd.h | 1 - 3 files changed, 8 insertions(+), 37 deletions(-) diff --git a/src/core/drivers/dpdk/pmd.cc b/src/core/drivers/dpdk/pmd.cc index d53a1c4..393fd54 100644 --- a/src/core/drivers/dpdk/pmd.cc +++ b/src/core/drivers/dpdk/pmd.cc @@ -100,7 +100,7 @@ void RxRing::Init() { void PmdPort::InitDriver(uint16_t mtu) { if (is_dpdk_primary_process_) { // Get DPDK port info. - FetchDpdkPortInfo(port_id_, &devinfo_, &l2_addr_, &pci_info_); + FetchDpdkPortInfo(port_id_, &devinfo_, &l2_addr_); device_ = devinfo_.device; if (std::string(devinfo_.driver_name) == "net_netvsc") { @@ -115,9 +115,7 @@ void PmdPort::InitDriver(uint16_t mtu) { // Get DPDK port info. struct rte_eth_dev_info vf_devinfo_; struct net::Ethernet::Address vf_l2_addr_; - std::string vf_pci_info_; - FetchDpdkPortInfo(vf_port_id.value(), &vf_devinfo_, &vf_l2_addr_, - &vf_pci_info_); + FetchDpdkPortInfo(vf_port_id.value(), &vf_devinfo_, &vf_l2_addr_); // If the VF is using an 'mlx4*' driver, we need extra checks. if (std::string(vf_devinfo_.driver_name).find("mlx4") != @@ -302,7 +300,7 @@ void PmdPort::InitDriver(uint16_t mtu) { << "Link is DOWN."; } } else { - FetchDpdkPortInfo(port_id_, &devinfo_, &l2_addr_, &pci_info_); + FetchDpdkPortInfo(port_id_, &devinfo_, &l2_addr_); // For the rings, just set port and queue IDs here, which have been // pre-initialized by the DPDK primary process diff --git a/src/include/dpdk.h b/src/include/dpdk.h index f087be0..db5fc4a 100644 --- a/src/include/dpdk.h +++ b/src/include/dpdk.h @@ -24,10 +24,7 @@ namespace dpdk { [[maybe_unused]] static void FetchDpdkPortInfo( uint8_t port_id, struct rte_eth_dev_info *devinfo, - juggler::net::Ethernet::Address *lladdr, std::string *pci_string) { - - // struct ethtool_drvinfo drvinfo; - + juggler::net::Ethernet::Address *lladdr) { if (!rte_eth_dev_is_valid_port(port_id)) { LOG(INFO) << "Port id " << static_cast(port_id) << " is not valid."; @@ -46,32 +43,11 @@ namespace dpdk { rte_eth_macaddr_get(port_id, reinterpret_cast(lladdr->bytes)); - struct rte_bus *bus = rte_bus_find_by_device(devinfo->device); - // if (bus && !strcmp(bus->name, "pci")) { - // const struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(devinfo->device); - // *pci_string = juggler::utils::Format( - // "%08x:%02hhx:%02hhx.%02hhx %04hx:%04hx", pci_dev->addr.domain, - // pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function, - // pci_dev->id.vendor_id, pci_dev->id.device_id); - if (bus) { - // strlcpy(drvinfo.driver, devinfo->driver_name, - // sizeof(drvinfo.driver)); - // strlcpy(drvinfo.version, rte_version(), sizeof(drvinfo.version)); - // strlcpy(drvinfo.bus_info, rte_dev_name(devinfo->device), - // sizeof(drvinfo.bus_info)); - // const struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(devinfo->device); - // *pci_string = juggler::utils::Format( - // "%08x:%02hhx:%02hhx.%02hhx %04hx:%04hx", pci_dev->addr.domain, - // pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function, - // pci_dev->id.vendor_id, pci_dev->id.device_id); - } - LOG(INFO) << "[PMDPORT] [port_id: " << static_cast(port_id) << ", driver: " << devinfo->driver_name << ", RXQ: " << devinfo->max_rx_queues << ", TXQ: " << devinfo->max_tx_queues - << ", l2addr: " << lladdr->ToString() - << ", pci_info: " << *pci_string << "]"; + << ", l2addr: " << lladdr->ToString() << "]"; } [[maybe_unused]] static std::optional FindSlaveVfPortId( @@ -80,7 +56,7 @@ namespace dpdk { std::string pci_info; juggler::net::Ethernet::Address lladdr; - FetchDpdkPortInfo(port_id, &devinfo, &lladdr, &pci_info); + FetchDpdkPortInfo(port_id, &devinfo, &lladdr); uint16_t slave_port_id = 0; while (slave_port_id < RTE_MAX_ETHPORTS) { @@ -94,10 +70,8 @@ namespace dpdk { } struct rte_eth_dev_info slave_devinfo; - std::string slave_pci_info; juggler::net::Ethernet::Address slave_lladdr; - FetchDpdkPortInfo(slave_port_id, &slave_devinfo, &slave_lladdr, - &slave_pci_info); + FetchDpdkPortInfo(slave_port_id, &slave_devinfo, &slave_lladdr); if (slave_lladdr == lladdr) { return slave_port_id; } @@ -118,7 +92,7 @@ namespace dpdk { std::string pci_info; juggler::net::Ethernet::Address lladdr; - FetchDpdkPortInfo(port_id, &devinfo, &lladdr, &pci_info); + FetchDpdkPortInfo(port_id, &devinfo, &lladdr); } } diff --git a/src/include/pmd.h b/src/include/pmd.h index 76f0890..222199f 100644 --- a/src/include/pmd.h +++ b/src/include/pmd.h @@ -475,7 +475,6 @@ class PmdPort { std::vector rss_reta_conf_; struct rte_eth_stats port_stats_; std::vector rss_hash_key_; - std::string pci_info_; bool initialized_; }; } // namespace dpdk From 2a69c17699cef58b17f30cedc2516d534801e513 Mon Sep 17 00:00:00 2001 From: Alireza Sanaee Date: Fri, 12 Jan 2024 18:58:52 +0000 Subject: [PATCH 3/5] Update Dockerfile and documentation for new DPDK * Dockerfile now clones and installs new DPDK * Also our internal documentation suggests using DPDK 23.11 Signed-off-by: Alireza Sanaee sarsanaee@gmail.com --- Dockerfile | 2 +- docs/INTERNAL.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9dc987a..3340df4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,7 +42,7 @@ RUN git clone -b 'stable-v40' --single-branch --depth 1 https://github.com/linux ENV RTE_SDK /root/dpdk # Build DPDK -RUN git clone --depth 1 --branch 'v21.11' https://github.com/DPDK/dpdk.git ${RTE_SDK} && \ +RUN git clone --depth 1 --branch 'v23.11' https://github.com/DPDK/dpdk.git ${RTE_SDK} && \ cd ${RTE_SDK} && \ meson build -Dexamples='' -Dplatform=generic -Denable_kmods=false -Dtests=false -Ddisable_drivers='raw/*,crypto/*,baseband/*,dma/*' && \ cd build/ && \ diff --git a/docs/INTERNAL.md b/docs/INTERNAL.md index dd243b2..4d287bd 100644 --- a/docs/INTERNAL.md +++ b/docs/INTERNAL.md @@ -41,10 +41,10 @@ ldconfig ### Build DPDK -We use DPDK v21.11 LTS, like so: +We use DPDK v23.11 LTS, like so: ```bash -export RTE_SDK=/path/to/dpdk-21.11/ +export RTE_SDK=/path/to/dpdk-23.11/ cd ${RTE_SDK} && meson build && cd build && ninja && DESTDIR=${PWD}/install ninja install ``` From b6553dfafb306d3a756b3f960f1014a18f71799d Mon Sep 17 00:00:00 2001 From: Alireza Sanaee Date: Fri, 12 Jan 2024 23:50:54 +0000 Subject: [PATCH 4/5] Remove unnecessary includes and commented lines. Signed-off-by: Alireza Sanaee --- src/core/drivers/dpdk/pmd.cc | 1 - src/include/dpdk.h | 8 +------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/core/drivers/dpdk/pmd.cc b/src/core/drivers/dpdk/pmd.cc index 393fd54..0a0fefc 100644 --- a/src/core/drivers/dpdk/pmd.cc +++ b/src/core/drivers/dpdk/pmd.cc @@ -49,7 +49,6 @@ static rte_eth_conf DefaultEthConf(const rte_eth_dev_info *devinfo) { port_conf.rxmode.mtu = PmdRing::kDefaultFrameSize; port_conf.rxmode.max_lro_pkt_size = PmdRing::kDefaultFrameSize; - // port_conf.rxmode.split_hdr_size = 0; const auto rx_offload_capa = devinfo->rx_offload_capa; port_conf.rxmode.offloads |= ((RTE_ETH_RX_OFFLOAD_CHECKSUM)&rx_offload_capa); diff --git a/src/include/dpdk.h b/src/include/dpdk.h index db5fc4a..ffdd192 100644 --- a/src/include/dpdk.h +++ b/src/include/dpdk.h @@ -4,20 +4,14 @@ #include #include #include -// #include -// #include -// #include #include -// #include -#include -#include + #include #include #include #include #include -#include namespace juggler { namespace dpdk { From 2013a744a194ac6a45f83b18733714067a394031 Mon Sep 17 00:00:00 2001 From: Alireza Sanaee Date: Thu, 18 Jan 2024 16:41:56 +0000 Subject: [PATCH 5/5] Removing history file from commits. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9472006..02278da 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.so *.o build/ +.history