From 57731f2427042a8a78d676daf9ebf9aca5360743 Mon Sep 17 00:00:00 2001 From: Aaron U'Ren Date: Sun, 22 Sep 2024 11:58:37 -0500 Subject: [PATCH 1/4] fact(krnode): add node struct abstraction This prepares the way for broader refactors in the way that we handle nodes by: * Separating frequently used node logic from the controller creation steps * Keeping reused code DRY-er * Adding interface abstractions for key groups of node data and starting to rely on those more rather than concrete types * Separating node data from the rest of the controller data structure so that it smaller definitions of data can be passed around to functions that need it rather than always passing the entire controller which contains more data / surface area than most functions need. --- pkg/cmd/kube-router.go | 2 +- .../netpol/network_policy_controller.go | 38 +- .../netpol/network_policy_controller_test.go | 62 ++- pkg/controllers/netpol/pod.go | 2 +- .../proxy/network_services_controller.go | 76 +--- .../proxy/network_services_controller_test.go | 10 +- .../proxy/service_endpoints_sync.go | 32 +- pkg/controllers/proxy/utils.go | 12 +- pkg/controllers/proxy/utils_test.go | 20 +- pkg/controllers/routing/bgp_peers.go | 55 ++- pkg/controllers/routing/bgp_policies.go | 10 +- pkg/controllers/routing/bgp_policies_test.go | 19 +- pkg/controllers/routing/ecmp_vip.go | 21 +- pkg/controllers/routing/ecmp_vip_test.go | 103 ++--- .../routing/network_routes_controller.go | 161 +++----- .../routing/network_routes_controller_test.go | 219 ++++++----- pkg/controllers/routing/pbr.go | 8 +- pkg/controllers/routing/utils.go | 39 +- pkg/utils/linux_routing.go | 6 + pkg/utils/node.go | 371 +++++++++++++++--- pkg/utils/node_test.go | 23 +- 21 files changed, 757 insertions(+), 532 deletions(-) diff --git a/pkg/cmd/kube-router.go b/pkg/cmd/kube-router.go index 577e781c51..1c0b06e6b2 100644 --- a/pkg/cmd/kube-router.go +++ b/pkg/cmd/kube-router.go @@ -220,7 +220,7 @@ func (kr *KubeRouter) Run() error { return fmt.Errorf("failed to create iptables handlers: %v", err) } npc, err := netpol.NewNetworkPolicyController(kr.Client, - kr.Config, podInformer, npInformer, nsInformer, &ipsetMutex, iptablesCmdHandlers, ipSetHandlers) + kr.Config, podInformer, npInformer, nsInformer, &ipsetMutex, nil, iptablesCmdHandlers, ipSetHandlers) if err != nil { return fmt.Errorf("failed to create network policy controller: %v", err) } diff --git a/pkg/controllers/netpol/network_policy_controller.go b/pkg/controllers/netpol/network_policy_controller.go index b3bb233854..19b07ea7a3 100644 --- a/pkg/controllers/netpol/network_policy_controller.go +++ b/pkg/controllers/netpol/network_policy_controller.go @@ -63,7 +63,7 @@ var ( // NetworkPolicyController struct to hold information required by NetworkPolicyController type NetworkPolicyController struct { - nodeHostName string + krNode utils.NodeAware serviceClusterIPRanges []net.IPNet serviceExternalIPRanges []net.IPNet serviceLoadBalancerIPRanges []net.IPNet @@ -79,7 +79,6 @@ type NetworkPolicyController struct { iptablesSaveRestore map[v1core.IPFamily]utils.IPTablesSaveRestorer filterTableRules map[v1core.IPFamily]*bytes.Buffer ipSetHandlers map[v1core.IPFamily]utils.IPSetHandler - nodeIPs map[v1core.IPFamily]net.IP podLister cache.Indexer npLister cache.Indexer @@ -832,7 +831,7 @@ func NewIPTablesHandlers(config *options.KubeRouterConfig) ( func NewNetworkPolicyController(clientset kubernetes.Interface, config *options.KubeRouterConfig, podInformer cache.SharedIndexInformer, npInformer cache.SharedIndexInformer, nsInformer cache.SharedIndexInformer, - ipsetMutex *sync.Mutex, + ipsetMutex *sync.Mutex, linkQ utils.LocalLinkQuerier, iptablesCmdHandlers map[v1core.IPFamily]utils.IPTablesHandler, ipSetHandlers map[v1core.IPFamily]utils.IPSetHandler) (*NetworkPolicyController, error) { npc := NetworkPolicyController{ipsetMutex: ipsetMutex} @@ -941,44 +940,33 @@ func NewNetworkPolicyController(clientset kubernetes.Interface, return nil, err } - npc.nodeHostName = node.Name - - nodeIPv4, nodeIPv6 := utils.GetAllNodeIPs(node) + npc.krNode, err = utils.NewKRNode(node, linkQ, config.EnableIPv4, config.EnableIPv6) + if err != nil { + return nil, err + } npc.iptablesCmdHandlers = iptablesCmdHandlers npc.iptablesSaveRestore = make(map[v1core.IPFamily]utils.IPTablesSaveRestorer, 2) npc.filterTableRules = make(map[v1core.IPFamily]*bytes.Buffer, 2) npc.ipSetHandlers = ipSetHandlers - npc.nodeIPs = make(map[v1core.IPFamily]net.IP, 2) if config.EnableIPv4 { + if !npc.krNode.IsIPv4Capable() { + return nil, fmt.Errorf("IPv4 was enabled but no IPv4 address was found on node") + } + klog.V(2).Infof("IPv4 is enabled") npc.iptablesSaveRestore[v1core.IPv4Protocol] = utils.NewIPTablesSaveRestore(v1core.IPv4Protocol) var buf bytes.Buffer npc.filterTableRules[v1core.IPv4Protocol] = &buf - // TODO: assuming that NPC should only use a single IP here is short-sighted, fix it so it considers all IPs - switch { - case len(nodeIPv4[v1core.NodeInternalIP]) > 0: - npc.nodeIPs[v1core.IPv4Protocol] = nodeIPv4[v1core.NodeInternalIP][0] - case len(nodeIPv4[v1core.NodeExternalIP]) > 0: - npc.nodeIPs[v1core.IPv4Protocol] = nodeIPv4[v1core.NodeExternalIP][0] - default: - return nil, fmt.Errorf("IPv4 was enabled but no IPv4 address was found on node") - } } if config.EnableIPv6 { + if !npc.krNode.IsIPv6Capable() { + return nil, fmt.Errorf("IPv6 was enabled but no IPv6 address was found on node") + } klog.V(2).Infof("IPv6 is enabled") npc.iptablesSaveRestore[v1core.IPv6Protocol] = utils.NewIPTablesSaveRestore(v1core.IPv6Protocol) var buf bytes.Buffer npc.filterTableRules[v1core.IPv6Protocol] = &buf - // TODO: assuming that NPC should only use a single IP here is short-sighted, fix it so it considers all IPs - switch { - case len(nodeIPv6[v1core.NodeInternalIP]) > 0: - npc.nodeIPs[v1core.IPv6Protocol] = nodeIPv6[v1core.NodeInternalIP][0] - case len(nodeIPv6[v1core.NodeExternalIP]) > 0: - npc.nodeIPs[v1core.IPv6Protocol] = nodeIPv6[v1core.NodeExternalIP][0] - default: - return nil, fmt.Errorf("IPv6 was enabled but no IPv6 address was found on node") - } } npc.podLister = podInformer.GetIndexer() diff --git a/pkg/controllers/netpol/network_policy_controller_test.go b/pkg/controllers/netpol/network_policy_controller_test.go index f20faafb0a..cd187ee230 100644 --- a/pkg/controllers/netpol/network_policy_controller_test.go +++ b/pkg/controllers/netpol/network_policy_controller_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/coreos/go-iptables/iptables" + "github.com/vishvananda/netlink" netv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -207,7 +208,6 @@ func newUneventfulNetworkPolicyController(podInformer cache.SharedIndexInformer, npc.iptablesSaveRestore = make(map[v1.IPFamily]utils.IPTablesSaveRestorer) npc.filterTableRules = make(map[v1.IPFamily]*bytes.Buffer) npc.ipSetHandlers = make(map[v1.IPFamily]utils.IPSetHandler) - npc.nodeIPs = make(map[v1.IPFamily]net.IP) // TODO: Handle both IP families npc.iptablesCmdHandlers[v1.IPv4Protocol] = newFakeIPTables(iptables.ProtocolIPv4) @@ -215,9 +215,13 @@ func newUneventfulNetworkPolicyController(podInformer cache.SharedIndexInformer, var buf bytes.Buffer npc.filterTableRules[v1.IPv4Protocol] = &buf npc.ipSetHandlers[v1.IPv4Protocol] = &fakeIPSet{} - npc.nodeIPs[v1.IPv4Protocol] = net.IPv4(10, 10, 10, 10) - npc.nodeHostName = "node" + krNode := utils.KRNode{ + NodeName: "node", + NodeIPv4Addrs: map[v1.NodeAddressType][]net.IP{v1.NodeInternalIP: {net.IPv4(10, 10, 10, 10)}}, + } + npc.krNode = &krNode + npc.podLister = podInformer.GetIndexer() npc.nsLister = nsInformer.GetIndexer() npc.npLister = npInformer.GetIndexer() @@ -754,6 +758,51 @@ func (ips *fakeIPSet) Name(name string) string { return name } +type fakeLocalLinkQuerier struct { + links []netlink.Link + addrs []*net.IPNet +} + +func newFakeLocalLinkQuerier(addrStrings []string) *fakeLocalLinkQuerier { + links := make([]netlink.Link, len(addrStrings)) + for idx := range addrStrings { + linkAttrs := netlink.LinkAttrs{ + Index: idx, + } + linkDevice := netlink.Device{LinkAttrs: linkAttrs} + links[idx] = &linkDevice + } + addrs := make([]*net.IPNet, len(addrStrings)) + for idx, addr := range addrStrings { + ip := net.ParseIP(addr) + var netMask net.IPMask + if ip.To4() != nil { + netMask = net.CIDRMask(24, 32) + } else { + netMask = net.CIDRMask(64, 128) + } + ipNet := &net.IPNet{ + IP: ip, + Mask: netMask, + } + addrs[idx] = ipNet + } + return &fakeLocalLinkQuerier{ + links: links, + addrs: addrs, + } +} + +func (f *fakeLocalLinkQuerier) LinkList() ([]netlink.Link, error) { + return f.links, nil +} + +func (f *fakeLocalLinkQuerier) AddrList(link netlink.Link, family int) ([]netlink.Addr, error) { + addrs := make([]netlink.Addr, 1) + addrs[0] = netlink.Addr{IPNet: f.addrs[link.Attrs().Index]} + return addrs, nil +} + func TestNetworkPolicyController(t *testing.T) { curHostname, _ := os.Hostname() testCases := []tNetPolConfigTestCase{ @@ -939,7 +988,9 @@ func TestNetworkPolicyController(t *testing.T) { "", }, } - client := fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{*newFakeNode("node", []string{"10.10.10.10", "2001:0db8:0042:0001:0000:0000:0000:0000"})}}) + fakeNodeIPs := []string{"10.10.10.10", "2001:0db8:0042:0001:0000:0000:0000:0000"} + fakeLinkQuerier := newFakeLocalLinkQuerier(fakeNodeIPs) + client := fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{*newFakeNode("node", fakeNodeIPs)}}) _, podInformer, nsInformer, netpolInformer := newFakeInformersFromClient(client) for _, test := range testCases { t.Run(test.name, func(t *testing.T) { @@ -948,7 +999,8 @@ func TestNetworkPolicyController(t *testing.T) { iptablesHandlers[v1.IPv4Protocol] = newFakeIPTables(iptables.ProtocolIPv4) ipSetHandlers := make(map[v1.IPFamily]utils.IPSetHandler, 1) ipSetHandlers[v1.IPv4Protocol] = &fakeIPSet{} - _, err := NewNetworkPolicyController(client, test.config, podInformer, netpolInformer, nsInformer, &sync.Mutex{}, iptablesHandlers, ipSetHandlers) + _, err := NewNetworkPolicyController(client, test.config, podInformer, netpolInformer, nsInformer, + &sync.Mutex{}, fakeLinkQuerier, iptablesHandlers, ipSetHandlers) if err == nil && test.expectError { t.Error("This config should have failed, but it was successful instead") } else if err != nil { diff --git a/pkg/controllers/netpol/pod.go b/pkg/controllers/netpol/pod.go index c2ce56dca7..b42175906d 100644 --- a/pkg/controllers/netpol/pod.go +++ b/pkg/controllers/netpol/pod.go @@ -115,7 +115,7 @@ func (npc *NetworkPolicyController) syncPodFirewallChains(networkPoliciesInfo [] // loop through the pods running on the node allLocalPods := make(map[string]podInfo) - for _, nodeIP := range npc.nodeIPs { + for _, nodeIP := range npc.krNode.GetNodeIPAddrs() { npc.getLocalPods(allLocalPods, nodeIP.String()) } for _, pod := range allLocalPods { diff --git a/pkg/controllers/proxy/network_services_controller.go b/pkg/controllers/proxy/network_services_controller.go index e7be3d1dbe..45e9d213e2 100644 --- a/pkg/controllers/proxy/network_services_controller.go +++ b/pkg/controllers/proxy/network_services_controller.go @@ -108,8 +108,7 @@ const ( // NetworkServicesController struct stores information needed by the controller type NetworkServicesController struct { - primaryIP net.IP - nodeHostName string + krNode utils.NodeAware syncPeriod time.Duration mu sync.Mutex serviceMap serviceInfoMap @@ -145,12 +144,8 @@ type NetworkServicesController struct { iptablesCmdHandlers map[v1.IPFamily]utils.IPTablesHandler ipSetHandlers map[v1.IPFamily]utils.IPSetHandler - nodeIPv4Addrs map[v1.NodeAddressType][]net.IP - nodeIPv6Addrs map[v1.NodeAddressType][]net.IP podIPv4CIDRs []string podIPv6CIDRs []string - isIPv4Capable bool - isIPv6Capable bool hpc *hairpinController hpEndpointReceiver chan string @@ -294,7 +289,7 @@ func (nsc *NetworkServicesController) Run(healthChan chan<- *healthcheck.Control // Ensure rp_filter=2 for DSR capability, see: // * https://access.redhat.com/solutions/53031 // * https://github.com/cloudnativelabs/kube-router/pull/1651#issuecomment-2072851683 - if nsc.isIPv4Capable { + if nsc.krNode.IsIPv4Capable() { sysctlErr := utils.SetSysctlSingleTemplate(utils.IPv4ConfRPFilterTemplate, "all", 2) if sysctlErr != nil { if sysctlErr.IsFatal() { @@ -618,10 +613,10 @@ func (nsc *NetworkServicesController) syncIpvsFirewall() error { for family, addrs := range addrsMap { // Don't run for families that we don't support - if family == v1.IPv4Protocol && !nsc.isIPv4Capable { + if family == v1.IPv4Protocol && !nsc.krNode.IsIPv4Capable() { continue } - if family == v1.IPv6Protocol && !nsc.isIPv6Capable { + if family == v1.IPv6Protocol && !nsc.krNode.IsIPv6Capable() { continue } @@ -744,10 +739,10 @@ func (nsc *NetworkServicesController) publishMetrics(serviceInfoMap serviceInfoM } else { pushMetric = false } - case nsc.primaryIP.String(): + case nsc.krNode.GetPrimaryNodeIP().String(): if protocol == ipvsSvc.Protocol && uint16(svc.port) == ipvsSvc.Port { pushMetric = true - svcVip = nsc.primaryIP.String() + svcVip = nsc.krNode.GetPrimaryNodeIP().String() } else { pushMetric = false } @@ -1106,7 +1101,7 @@ func (nsc *NetworkServicesController) buildEndpointSliceInfo() endpointSliceInfo } for _, addr := range ep.Addresses { - isLocal := ep.NodeName != nil && *ep.NodeName == nsc.nodeHostName + isLocal := ep.NodeName != nil && *ep.NodeName == nsc.krNode.GetNodeName() endpoints = append(endpoints, endpointSliceInfo{ ip: addr, port: int(*port.Port), @@ -1287,17 +1282,13 @@ func (nsc *NetworkServicesController) syncHairpinIptablesRules() error { family = v1.IPv4Protocol familyClusterIPs = clusterIPs[v1.IPv4Protocol] familyExternalIPs = externalIPs[v1.IPv4Protocol] - //nolint:gocritic // we intend to append to separate maps here - familyNodeIPs = append(nsc.nodeIPv4Addrs[v1.NodeInternalIP], - nsc.nodeIPv4Addrs[v1.NodeExternalIP]...) + familyNodeIPs = nsc.krNode.GetNodeIPv4Addrs() rulesMap = ipv4RulesNeeded } else { family = v1.IPv6Protocol familyClusterIPs = clusterIPs[v1.IPv6Protocol] familyExternalIPs = externalIPs[v1.IPv6Protocol] - //nolint:gocritic // we intend to append to separate maps here - familyNodeIPs = append(nsc.nodeIPv6Addrs[v1.NodeInternalIP], - nsc.nodeIPv6Addrs[v1.NodeExternalIP]...) + familyNodeIPs = nsc.krNode.GetNodeIPv6Addrs() rulesMap = ipv6RulesNeeded } if len(familyClusterIPs) < 1 { @@ -1336,14 +1327,14 @@ func (nsc *NetworkServicesController) syncHairpinIptablesRules() error { } // Cleanup (if needed) and return if there's no hairpin-mode Services - if len(ipv4RulesNeeded) == 0 && nsc.isIPv4Capable { + if len(ipv4RulesNeeded) == 0 && nsc.krNode.IsIPv4Capable() { klog.V(1).Info("No IPv4 hairpin-mode enabled services found -- no hairpin rules created") err := nsc.deleteHairpinIptablesRules(v1.IPv4Protocol) if err != nil { return fmt.Errorf("error deleting hairpin rules: %v", err) } } - if len(ipv6RulesNeeded) == 0 && nsc.isIPv6Capable { + if len(ipv6RulesNeeded) == 0 && nsc.krNode.IsIPv6Capable() { klog.V(1).Info("No IPv6 hairpin-mode enabled services found -- no hairpin rules created") err := nsc.deleteHairpinIptablesRules(v1.IPv6Protocol) if err != nil { @@ -1777,7 +1768,7 @@ func (nsc *NetworkServicesController) Cleanup() { if len(nsc.iptablesCmdHandlers) < 1 { // Even though we have a config at this point (via passed param), we want to send nil so that the node will // discover which IP address families it has and act accordingly - err = nsc.setupHandlers(nil, nil) + err = nsc.setupHandlers(nil) if err != nil { klog.Errorf("could not cleanup because we couldn't create iptables/ipset command handlers due to: %v", err) } @@ -1936,35 +1927,13 @@ func (nsc *NetworkServicesController) handleServiceDelete(obj interface{}) { // setupHandlers Here we test to see whether the node is IPv6 capable, if the user has enabled IPv6 (via command-line // options) and the node has an IPv6 address, the following method will return an IPv6 address -func (nsc *NetworkServicesController) setupHandlers(config *options.KubeRouterConfig, node *v1.Node) error { - // node being nil covers the case where this function is called by something that doesn't have a kube-apiserver - // connection like the cleanup code. In this instance we want all possible iptables and ipset handlers - if node != nil { - nsc.nodeIPv4Addrs, nsc.nodeIPv6Addrs = utils.GetAllNodeIPs(node) - } - - // We test for nil configs as the Cleanup() method often doesn't have a valid config in this respect, so rather - // than trying to guess options, it is better to just let the logic fallthrough. For the primary path to this func, - // NewNetworkServicesController, the config will not be nil and we want to check that we have options that match - // the node's capability to ensure sanity later down the road. - if config != nil { - if config.EnableIPv4 && len(nsc.nodeIPv4Addrs[v1.NodeInternalIP]) < 1 && - len(nsc.nodeIPv4Addrs[v1.NodeExternalIP]) < 1 { - return fmt.Errorf("IPv4 was enabled, but no IPv4 address was found on the node") - } - } - nsc.isIPv4Capable = len(nsc.nodeIPv4Addrs) > 0 - if config != nil { - if config.EnableIPv6 && len(nsc.nodeIPv6Addrs[v1.NodeInternalIP]) < 1 && - len(nsc.nodeIPv6Addrs[v1.NodeExternalIP]) < 1 { - return fmt.Errorf("IPv6 was enabled, but no IPv6 address was found on the node") - } - } - nsc.isIPv6Capable = len(nsc.nodeIPv6Addrs) > 0 - +func (nsc *NetworkServicesController) setupHandlers(node *v1.Node) error { nsc.ipSetHandlers = make(map[v1.IPFamily]utils.IPSetHandler) nsc.iptablesCmdHandlers = make(map[v1.IPFamily]utils.IPTablesHandler) - if node == nil || len(nsc.nodeIPv4Addrs) > 0 { + + // node being nil covers the case where this function is called by something that doesn't have a kube-apiserver + // connection like the cleanup code. In this instance we want all possible iptables and ipset handlers + if node == nil || nsc.krNode == nil || nsc.krNode.IsIPv4Capable() { iptHandler, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) if err != nil { klog.Fatalf("Failed to allocate IPv4 iptables handler: %v", err) @@ -1979,7 +1948,7 @@ func (nsc *NetworkServicesController) setupHandlers(config *options.KubeRouterCo } nsc.ipSetHandlers[v1.IPv4Protocol] = ipset } - if node == nil || len(nsc.nodeIPv6Addrs) > 0 { + if node == nil || nsc.krNode == nil || nsc.krNode.IsIPv6Capable() { iptHandler, err := iptables.NewWithProtocol(iptables.ProtocolIPv6) if err != nil { klog.Fatalf("Failed to allocate IPv6 iptables handler: %v", err) @@ -2084,10 +2053,7 @@ func NewNetworkServicesController(clientset kubernetes.Interface, return nil, err } - nsc.nodeHostName = node.Name - // We preserve the old logic here for getting the primary IP which is set on nrc.primaryIP. This can be either IPv4 - // or IPv6 - nsc.primaryIP, err = utils.GetPrimaryNodeIP(node) + nsc.krNode, err = utils.NewKRNode(node, nil, config.EnableIPv4, config.EnableIPv6) if err != nil { return nil, err } @@ -2097,12 +2063,12 @@ func NewNetworkServicesController(clientset kubernetes.Interface, // * Sets nsc.nodeIPv6Addr & nsc.isIPv6Capable // * Creates the iptables handlers for ipv4 & ipv6 // * Creates the ipset handlers for ipv4 & ipv6 - err = nsc.setupHandlers(config, node) + err = nsc.setupHandlers(node) if err != nil { return nil, err } - automtu, err := utils.GetMTUFromNodeIP(nsc.primaryIP) + automtu, err := nsc.krNode.GetNodeMTU() if err != nil { return nil, err } diff --git a/pkg/controllers/proxy/network_services_controller_test.go b/pkg/controllers/proxy/network_services_controller_test.go index 7ae6486d3b..fc9b528485 100644 --- a/pkg/controllers/proxy/network_services_controller_test.go +++ b/pkg/controllers/proxy/network_services_controller_test.go @@ -6,6 +6,7 @@ import ( "net" "time" + "github.com/cloudnativelabs/kube-router/v2/pkg/utils" "github.com/moby/ipvs" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -140,10 +141,13 @@ var _ = Describe("NetworkServicesController", func() { fatalf("failed to create existing services: %v", err) } + krNode := &utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP("10.0.0.0"), + } nsc = &NetworkServicesController{ - primaryIP: net.ParseIP("10.0.0.0"), - nodeHostName: "node-1", - ln: mockedLinuxNetworking, + krNode: krNode, + ln: mockedLinuxNetworking, } startInformersForServiceProxy(nsc, clientset) diff --git a/pkg/controllers/proxy/service_endpoints_sync.go b/pkg/controllers/proxy/service_endpoints_sync.go index bfa839d4ea..107be86c97 100644 --- a/pkg/controllers/proxy/service_endpoints_sync.go +++ b/pkg/controllers/proxy/service_endpoints_sync.go @@ -129,8 +129,8 @@ func (nsc *NetworkServicesController) setupClusterIPServices(serviceInfoMap serv protocol := convertSvcProtoToSysCallProto(svc.protocol) clusterIPs := getAllClusterIPs(svc) - ipv4NodeIP := utils.FindBestIPv4NodeAddress(nsc.primaryIP, nsc.nodeIPv4Addrs) - ipv6NodeIP := utils.FindBestIPv6NodeAddress(nsc.primaryIP, nsc.nodeIPv6Addrs) + ipv4NodeIP := nsc.krNode.FindBestIPv4NodeAddress() + ipv6NodeIP := nsc.krNode.FindBestIPv6NodeAddress() dummyVipInterface, err := nsc.ln.getKubeDummyInterface() if err != nil { return fmt.Errorf("failed creating dummy interface: %v", err) @@ -321,14 +321,14 @@ func (nsc *NetworkServicesController) setupNodePortServices(serviceInfoMap servi } } } else { - ipvsSvcs, svcID, ipvsSvc = nsc.addIPVSService(ipvsSvcs, activeServiceEndpointMap, svc, nsc.primaryIP, - protocol, uint16(svc.nodePort)) + ipvsSvcs, svcID, ipvsSvc = nsc.addIPVSService(ipvsSvcs, activeServiceEndpointMap, svc, + nsc.krNode.GetPrimaryNodeIP(), protocol, uint16(svc.nodePort)) // We weren't able to create the IPVS service, so we won't be able to add endpoints to it if svcID == "" { continue } - nsc.addEndpointsToIPVSService(endpoints, activeServiceEndpointMap, svc, svcID, ipvsSvc, nsc.primaryIP, - false) + nsc.addEndpointsToIPVSService(endpoints, activeServiceEndpointMap, svc, svcID, ipvsSvc, + nsc.krNode.GetPrimaryNodeIP(), false) } } @@ -401,9 +401,9 @@ func (nsc *NetworkServicesController) setupExternalIPForService(svc *serviceInfo var ipvsExternalIPSvc *ipvs.Service if externalIP.To4() != nil { - nodeIP = utils.FindBestIPv4NodeAddress(nsc.primaryIP, nsc.nodeIPv4Addrs) + nodeIP = nsc.krNode.FindBestIPv4NodeAddress() } else { - nodeIP = utils.FindBestIPv6NodeAddress(nsc.primaryIP, nsc.nodeIPv6Addrs) + nodeIP = nsc.krNode.FindBestIPv6NodeAddress() } dummyVipInterface, err := nsc.ln.getKubeDummyInterface() @@ -486,11 +486,11 @@ func (nsc *NetworkServicesController) setupExternalIPForDSRService(svcIn *servic var family v1.IPFamily var sysFamily uint16 if externalIP.To4() != nil { - nodeIP = utils.FindBestIPv4NodeAddress(nsc.primaryIP, nsc.nodeIPv4Addrs) + nodeIP = nsc.krNode.FindBestIPv4NodeAddress() family = v1.IPv4Protocol sysFamily = syscall.AF_INET } else { - nodeIP = utils.FindBestIPv6NodeAddress(nsc.primaryIP, nsc.nodeIPv6Addrs) + nodeIP = nsc.krNode.FindBestIPv6NodeAddress() family = v1.IPv6Protocol sysFamily = syscall.AF_INET6 } @@ -603,7 +603,7 @@ func (nsc *NetworkServicesController) setupExternalIPForDSRService(svcIn *servic func (nsc *NetworkServicesController) setupForDSR(serviceInfoMap serviceInfoMap) error { klog.V(1).Infof("Setting up policy routing required for Direct Server Return functionality.") - err := nsc.ln.setupPolicyRoutingForDSR(nsc.isIPv4Capable, nsc.isIPv6Capable) + err := nsc.ln.setupPolicyRoutingForDSR(nsc.krNode.IsIPv4Capable(), nsc.krNode.IsIPv6Capable()) if err != nil { return errors.New("Failed setup PBR for DSR due to: " + err.Error()) } @@ -611,7 +611,7 @@ func (nsc *NetworkServicesController) setupForDSR(serviceInfoMap serviceInfoMap) customDSRRouteTableName) klog.V(1).Infof("Setting up custom route table required to add routes for external IP's.") - err = nsc.ln.setupRoutesForExternalIPForDSR(serviceInfoMap, nsc.isIPv4Capable, nsc.isIPv6Capable) + err = nsc.ln.setupRoutesForExternalIPForDSR(serviceInfoMap, nsc.krNode.IsIPv4Capable(), nsc.krNode.IsIPv6Capable()) if err != nil { klog.Errorf("failed setup custom routing table required to add routes for external IP's due to: %v", err) @@ -649,9 +649,9 @@ func (nsc *NetworkServicesController) cleanupStaleVIPs(activeServiceEndpointMap klog.V(1).Infof("Found an IP %s which is no longer needed so cleaning up", addr.IP.String()) var nodeIPForFamily net.IP if addr.IP.To4() != nil { - nodeIPForFamily = utils.FindBestIPv4NodeAddress(nsc.primaryIP, nsc.nodeIPv4Addrs) + nodeIPForFamily = nsc.krNode.FindBestIPv4NodeAddress() } else { - nodeIPForFamily = utils.FindBestIPv6NodeAddress(nsc.primaryIP, nsc.nodeIPv6Addrs) + nodeIPForFamily = nsc.krNode.FindBestIPv6NodeAddress() } err := nsc.ln.ipAddrDel(intfc, addr.IP.String(), nodeIPForFamily.String()) @@ -821,10 +821,10 @@ func (nsc *NetworkServicesController) cleanupDSRService(fwMark uint32) error { } } - if nsc.isIPv4Capable { + if nsc.krNode.IsIPv4Capable() { cleanupTables("iptables-save") } - if nsc.isIPv6Capable { + if nsc.krNode.IsIPv6Capable() { cleanupTables("ip6tables-save") } diff --git a/pkg/controllers/proxy/utils.go b/pkg/controllers/proxy/utils.go index c0e3e35d73..eef5f2ea98 100644 --- a/pkg/controllers/proxy/utils.go +++ b/pkg/controllers/proxy/utils.go @@ -158,7 +158,7 @@ func (nsc *NetworkServicesController) isValidKubeRouterServiceArtifact(address n return true, nil } } - } else if address.Equal(nsc.primaryIP) { + } else if address.Equal(nsc.krNode.GetPrimaryNodeIP()) { return true, nil } } @@ -252,7 +252,7 @@ func (nsc *NetworkServicesController) findContainerRuntimeReferences(endpointIP } // we are only concerned with endpoint pod running on current node - if strings.Compare(podObj.Status.HostIP, nsc.primaryIP.String()) != 0 { + if strings.Compare(podObj.Status.HostIP, nsc.krNode.GetPrimaryNodeIP().String()) != 0 { return "", "", nil } @@ -318,11 +318,11 @@ func (nsc *NetworkServicesController) getPrimaryAndCIDRsByFamily(ipFamily v1.IPF switch ipFamily { case v1.IPv4Protocol: // If we're not detected to be IPv4 capable break early - if !nsc.isIPv4Capable { + if !nsc.krNode.IsIPv4Capable() { return "", nil } - primaryIP = utils.FindBestIPv4NodeAddress(nsc.primaryIP, nsc.nodeIPv4Addrs).String() + primaryIP = nsc.krNode.FindBestIPv4NodeAddress().String() if len(nsc.podCidr) > 0 && netutils.IsIPv4CIDRString(nsc.podCidr) { cidrMap[nsc.podCidr] = true } @@ -335,11 +335,11 @@ func (nsc *NetworkServicesController) getPrimaryAndCIDRsByFamily(ipFamily v1.IPF } case v1.IPv6Protocol: // If we're not detected to be IPv6 capable break early - if !nsc.isIPv6Capable { + if !nsc.krNode.IsIPv6Capable() { return "", nil } - primaryIP = utils.FindBestIPv6NodeAddress(nsc.primaryIP, nsc.nodeIPv6Addrs).String() + primaryIP = nsc.krNode.FindBestIPv6NodeAddress().String() if len(nsc.podCidr) > 0 && netutils.IsIPv6CIDRString(nsc.podCidr) { cidrMap[nsc.podCidr] = true } diff --git a/pkg/controllers/proxy/utils_test.go b/pkg/controllers/proxy/utils_test.go index a313ed6fc8..7d5f3b58c1 100644 --- a/pkg/controllers/proxy/utils_test.go +++ b/pkg/controllers/proxy/utils_test.go @@ -6,6 +6,7 @@ import ( "strconv" "testing" + "github.com/cloudnativelabs/kube-router/v2/pkg/utils" "github.com/stretchr/testify/assert" ) @@ -22,11 +23,15 @@ func getMoqNSC() *NetworkServicesController { setupPolicyRoutingForDSRFunc: lnm.setupPolicyRoutingForDSR, setupRoutesForExternalIPForDSRFunc: lnm.setupRoutesForExternalIPForDSR, } + + krNode := &utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP("10.0.0.0"), + } return &NetworkServicesController{ - primaryIP: net.ParseIP("10.0.0.0"), - nodeHostName: "node-1", - ln: mockedLinuxNetworking, - fwMarkMap: map[uint32]string{}, + krNode: krNode, + ln: mockedLinuxNetworking, + fwMarkMap: map[uint32]string{}, } } @@ -186,14 +191,19 @@ func TestIsValidKubeRouterServiceArtifact(t *testing.T) { loadBalancerIPs: []string{"172.16.0.3"}, } + krNode := &utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP("192.168.1.10"), + } + nsc := &NetworkServicesController{ + krNode: krNode, serviceMap: map[string]*serviceInfo{ "service1": service1, "service2": service2, "service3": service3, }, nodeportBindOnAllIP: false, - primaryIP: net.ParseIP("192.168.1.10"), } tests := []struct { diff --git a/pkg/controllers/routing/bgp_peers.go b/pkg/controllers/routing/bgp_peers.go index ca78775c95..34de74bfe1 100644 --- a/pkg/controllers/routing/bgp_peers.go +++ b/pkg/controllers/routing/bgp_peers.go @@ -46,14 +46,14 @@ func (nrc *NetworkRoutingController) syncInternalPeers() { currentNodes := make([]string, 0) for _, obj := range nodes { node := obj.(*v1core.Node) - nodeIP, err := utils.GetPrimaryNodeIP(node) + targetNode, err := utils.NewRemoteKRNode(node) if err != nil { - klog.Errorf("Failed to find a node IP and therefore cannot sync internal BGP Peer: %v", err) + klog.Errorf("failed to create KRNode from node object: %v", err) continue } // skip self - if nodeIP.String() == nrc.primaryIP.String() { + if targetNode.GetPrimaryNodeIP().Equal(nrc.krNode.GetPrimaryNodeIP()) { continue } @@ -70,50 +70,48 @@ func (nrc *NetworkRoutingController) syncInternalPeers() { nodeasn, ok := node.ObjectMeta.Annotations[nodeASNAnnotation] if !ok { klog.Infof("Not peering with the Node %s as ASN number of the node is unknown.", - nodeIP.String()) + targetNode.GetPrimaryNodeIP().String()) continue } asnNo, err := strconv.ParseUint(nodeasn, 0, asnMaxBitSize) if err != nil { klog.Infof("Not peering with the Node %s as ASN number of the node is invalid.", - nodeIP.String()) + targetNode.GetPrimaryNodeIP().String()) continue } // if the nodes ASN number is different from ASN number of current node skip peering if nrc.nodeAsnNumber != uint32(asnNo) { klog.Infof("Not peering with the Node %s as ASN number of the node is different.", - nodeIP.String()) + targetNode.GetPrimaryNodeIP().String()) continue } } - targetNodeIsIPv4 := nodeIP.To4() != nil - sourceNodeIsIPv4 := nrc.primaryIP.To4() != nil - ipv4NodeIPs, ipv6NodeIPs := utils.GetAllNodeIPs(node) - targetNodeIsIPv4Capable := utils.FindBestIPv4NodeAddress(nodeIP, ipv4NodeIPs) != nil - targetNodeIsIPv6Capable := utils.FindBestIPv6NodeAddress(nodeIP, ipv6NodeIPs) != nil + targetNodeIsIPv4 := targetNode.GetPrimaryNodeIP().To4() != nil + sourceNodeIsIPv4 := nrc.krNode.GetPrimaryNodeIP().To4() != nil if targetNodeIsIPv4 != sourceNodeIsIPv4 { klog.Warningf("Not peering with Node %s as it's primary IP (%s) uses a different protocol than "+ - "our primary IP (%s)", node.ObjectMeta.Name, nodeIP, nrc.primaryIP) + "our primary IP (%s)", node.ObjectMeta.Name, targetNode.GetPrimaryNodeIP(), + nrc.krNode.GetPrimaryNodeIP()) continue } - currentNodes = append(currentNodes, nodeIP.String()) - nrc.activeNodes[nodeIP.String()] = true + currentNodes = append(currentNodes, targetNode.GetPrimaryNodeIP().String()) + nrc.activeNodes[targetNode.GetPrimaryNodeIP().String()] = true // explicitly set neighbors.transport.config.local-address with primaryIP which is configured // as their neighbor address at the remote peers. // this prevents the controller from initiating connection to its peers with a different IP address // when multiple L3 interfaces are active. n := &gobgpapi.Peer{ Conf: &gobgpapi.PeerConf{ - NeighborAddress: nodeIP.String(), + NeighborAddress: targetNode.GetPrimaryNodeIP().String(), PeerAsn: nrc.nodeAsnNumber, }, Transport: &gobgpapi.Transport{ - LocalAddress: nrc.primaryIP.String(), + LocalAddress: nrc.krNode.GetPrimaryNodeIP().String(), RemotePort: nrc.bgpPort, }, } @@ -127,7 +125,7 @@ func (nrc *NetworkRoutingController) syncInternalPeers() { } // We choose to only peer using the protocol of the node's primary IP - if targetNodeIsIPv4Capable { + if targetNode.IsIPv4Capable() { afiSafi := gobgpapi.AfiSafi{ Config: &gobgpapi.AfiSafiConfig{ Family: &gobgpapi.Family{Afi: gobgpapi.Family_AFI_IP, Safi: gobgpapi.Family_SAFI_UNICAST}, @@ -142,7 +140,7 @@ func (nrc *NetworkRoutingController) syncInternalPeers() { } n.AfiSafis = append(n.AfiSafis, &afiSafi) } - if targetNodeIsIPv6Capable { + if targetNode.IsIPv6Capable() { afiSafi := gobgpapi.AfiSafi{ Config: &gobgpapi.AfiSafiConfig{ Family: &gobgpapi.Family{Afi: gobgpapi.Family_AFI_IP6, Safi: gobgpapi.Family_SAFI_UNICAST}, @@ -175,7 +173,7 @@ func (nrc *NetworkRoutingController) syncInternalPeers() { Peer: n, }); err != nil { if !strings.Contains(err.Error(), "can't overwrite the existing peer") { - klog.Errorf("Failed to add node %s as peer due to %s", nodeIP.String(), err) + klog.Errorf("Failed to add node %s as peer due to %s", targetNode.GetPrimaryNodeIP(), err) } } } @@ -239,7 +237,7 @@ func (nrc *NetworkRoutingController) connectToExternalBGPPeers(server *gobgp.Bgp LocalRestarting: true, } - if nrc.isIPv4Capable { + if nrc.krNode.IsIPv4Capable() { n.AfiSafis = []*gobgpapi.AfiSafi{ { Config: &gobgpapi.AfiSafiConfig{ @@ -254,7 +252,7 @@ func (nrc *NetworkRoutingController) connectToExternalBGPPeers(server *gobgp.Bgp }, } } - if nrc.isIPv6Capable { + if nrc.krNode.IsIPv6Capable() { afiSafi := gobgpapi.AfiSafi{ Config: &gobgpapi.AfiSafiConfig{ Family: &gobgpapi.Family{Afi: gobgpapi.Family_AFI_IP6, Safi: gobgpapi.Family_SAFI_UNICAST}, @@ -365,14 +363,14 @@ func (nrc *NetworkRoutingController) newNodeEventHandler() cache.ResourceEventHa return cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { node := obj.(*v1core.Node) - nodeIP, err := utils.GetPrimaryNodeIP(node) + targetNode, err := utils.NewRemoteKRNode(node) if err != nil { - klog.Errorf( - "New node received, but we were unable to add it as we were couldn't find its node IP: %v", err) + klog.Errorf("failed to create KRNode from node object: %v", err) return } - klog.V(2).Infof("Received node %s added update from watch API so peer with new node", nodeIP) + klog.V(2).Infof("Received node %s added update from watch API so peer with new node", + targetNode.GetPrimaryNodeIP()) nrc.OnNodeUpdate(obj) }, UpdateFunc: func(oldObj, newObj interface{}) { @@ -391,11 +389,12 @@ func (nrc *NetworkRoutingController) newNodeEventHandler() cache.ResourceEventHa return } } - nodeIP, err := utils.GetPrimaryNodeIP(node) + targetNode, err := utils.NewRemoteKRNode(node) // In this case even if we can't get the NodeIP that's alright as the node is being removed anyway and // future node lister operations that happen in OnNodeUpdate won't be affected as the node won't be returned - if err == nil { - klog.Infof("Received node %s removed update from watch API, so remove node from peer", nodeIP) + if err == nil && targetNode != nil { + klog.Infof("Received node %s removed update from watch API, so remove node from peer", + targetNode.GetPrimaryNodeIP()) } else { klog.Infof("Received node (IP unavailable) removed update from watch API, so remove node " + "from peer") diff --git a/pkg/controllers/routing/bgp_policies.go b/pkg/controllers/routing/bgp_policies.go index 311ae961f8..890b429400 100644 --- a/pkg/controllers/routing/bgp_policies.go +++ b/pkg/controllers/routing/bgp_policies.go @@ -343,15 +343,17 @@ func (nrc *NetworkRoutingController) addiBGPPeersDefinedSet() (map[v1core.IPFami nodes := nrc.nodeLister.List() for _, node := range nodes { nodeObj := node.(*v1core.Node) - nodeIP, err := utils.GetPrimaryNodeIP(nodeObj) + targetNode, err := utils.NewRemoteKRNode(nodeObj) if err != nil { klog.Errorf("Failed to find a node IP and therefore cannot add internal BGP Peer: %v", err) continue } - if nodeIP.To4() != nil { - iBGPPeerCIDRs[v1core.IPv4Protocol] = append(iBGPPeerCIDRs[v1core.IPv4Protocol], nodeIP.String()+"/32") + if targetNode.GetPrimaryNodeIP().To4() != nil { + iBGPPeerCIDRs[v1core.IPv4Protocol] = append(iBGPPeerCIDRs[v1core.IPv4Protocol], + targetNode.GetPrimaryNodeIP().String()+"/32") } else { - iBGPPeerCIDRs[v1core.IPv6Protocol] = append(iBGPPeerCIDRs[v1core.IPv6Protocol], nodeIP.String()+"/128") + iBGPPeerCIDRs[v1core.IPv6Protocol] = append(iBGPPeerCIDRs[v1core.IPv6Protocol], + targetNode.GetPrimaryNodeIP().String()+"/128") } } diff --git a/pkg/controllers/routing/bgp_policies_test.go b/pkg/controllers/routing/bgp_policies_test.go index 5da30c47cb..7e1fc0c156 100644 --- a/pkg/controllers/routing/bgp_policies_test.go +++ b/pkg/controllers/routing/bgp_policies_test.go @@ -3,6 +3,7 @@ package routing import ( "context" "fmt" + "net" "reflect" "testing" "time" @@ -12,6 +13,7 @@ import ( "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" + "github.com/cloudnativelabs/kube-router/v2/pkg/utils" gobgpapi "github.com/osrg/gobgp/v3/api" gobgp "github.com/osrg/gobgp/v3/pkg/server" ) @@ -34,6 +36,9 @@ type PolicyTestCase struct { } func Test_AddPolicies(t *testing.T) { + ipv4CapableKRNode := &utils.KRNode{ + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.IPv4(10, 10, 10, 10)}}, + } testcases := []PolicyTestCase{ { "has nodes and services", @@ -49,7 +54,7 @@ func Test_AddPolicies(t *testing.T) { activeNodes: make(map[string]bool), nodeAsnNumber: 100, podCidr: "172.20.0.0/24", - isIPv4Capable: true, + krNode: ipv4CapableKRNode, podIPv4CIDRs: []string{"172.20.0.0/24"}, }, []*v1core.Node{ @@ -180,7 +185,7 @@ func Test_AddPolicies(t *testing.T) { activeNodes: make(map[string]bool), nodeAsnNumber: 100, podCidr: "172.21.0.0/24", - isIPv4Capable: true, + krNode: ipv4CapableKRNode, podIPv4CIDRs: []string{"172.21.0.0/24"}, globalPeerRouters: []*gobgpapi.Peer{ { @@ -423,7 +428,7 @@ func Test_AddPolicies(t *testing.T) { bgpServer: gobgp.NewBgpServer(), activeNodes: make(map[string]bool), podCidr: "172.22.0.0/24", - isIPv4Capable: true, + krNode: ipv4CapableKRNode, podIPv4CIDRs: []string{"172.22.0.0/24"}, globalPeerRouters: []*gobgpapi.Peer{ { @@ -628,7 +633,7 @@ func Test_AddPolicies(t *testing.T) { bgpServer: gobgp.NewBgpServer(), activeNodes: make(map[string]bool), podCidr: "172.23.0.0/24", - isIPv4Capable: true, + krNode: ipv4CapableKRNode, podIPv4CIDRs: []string{"172.23.0.0/24"}, globalPeerRouters: []*gobgpapi.Peer{ { @@ -816,7 +821,7 @@ func Test_AddPolicies(t *testing.T) { bgpServer: gobgp.NewBgpServer(), activeNodes: make(map[string]bool), podCidr: "172.24.0.0/24", - isIPv4Capable: true, + krNode: ipv4CapableKRNode, podIPv4CIDRs: []string{"172.24.0.0/24"}, globalPeerRouters: []*gobgpapi.Peer{ { @@ -1027,7 +1032,7 @@ func Test_AddPolicies(t *testing.T) { bgpServer: gobgp.NewBgpServer(), activeNodes: make(map[string]bool), podCidr: "172.25.0.0/24", - isIPv4Capable: true, + krNode: ipv4CapableKRNode, podIPv4CIDRs: []string{"172.25.0.0/24"}, globalPeerRouters: []*gobgpapi.Peer{ { @@ -1238,7 +1243,7 @@ func Test_AddPolicies(t *testing.T) { advertisePodCidr: true, activeNodes: make(map[string]bool), podCidr: "172.26.0.0/24", - isIPv4Capable: true, + krNode: ipv4CapableKRNode, podIPv4CIDRs: []string{"172.26.0.0/24"}, globalPeerRouters: []*gobgpapi.Peer{ { diff --git a/pkg/controllers/routing/ecmp_vip.go b/pkg/controllers/routing/ecmp_vip.go index ff8eb49a58..1b6db07fa5 100644 --- a/pkg/controllers/routing/ecmp_vip.go +++ b/pkg/controllers/routing/ecmp_vip.go @@ -6,8 +6,6 @@ import ( "fmt" "strconv" - "golang.org/x/exp/maps" - "google.golang.org/protobuf/types/known/anypb" "github.com/cloudnativelabs/kube-router/v2/pkg/metrics" @@ -594,28 +592,15 @@ func (nrc *NetworkRoutingController) nodeHasEndpointsForService(svc *v1core.Serv return false, errors.New("failed to convert cache item to Endpoints type") } - // Find all the IPs that this node has on it so that we can use it to compare it against endpoint IPs - allNodeIPs := make([]string, 0) - for _, ips := range maps.Values(nrc.nodeIPv4Addrs) { - for _, ip := range ips { - allNodeIPs = append(allNodeIPs, ip.String()) - } - } - for _, ips := range maps.Values(nrc.nodeIPv6Addrs) { - for _, ip := range ips { - allNodeIPs = append(allNodeIPs, ip.String()) - } - } - for _, subset := range ep.Subsets { for _, address := range subset.Addresses { if address.NodeName != nil { - if *address.NodeName == nrc.nodeName { + if *address.NodeName == nrc.krNode.GetNodeName() { return true, nil } } else { - for _, nodeIP := range allNodeIPs { - if address.IP == nodeIP { + for _, nodeIP := range nrc.krNode.GetNodeIPAddrs() { + if address.IP == nodeIP.String() { return true, nil } } diff --git a/pkg/controllers/routing/ecmp_vip_test.go b/pkg/controllers/routing/ecmp_vip_test.go index e53da6e5bd..e935edd1c0 100644 --- a/pkg/controllers/routing/ecmp_vip_test.go +++ b/pkg/controllers/routing/ecmp_vip_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/cloudnativelabs/kube-router/v2/pkg/utils" v1core "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" @@ -55,9 +56,9 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: true, advertiseExternalIP: true, advertiseLoadBalancerIP: true, - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -97,9 +98,9 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: false, advertiseExternalIP: false, advertiseLoadBalancerIP: false, - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -139,9 +140,9 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: true, advertiseExternalIP: false, advertiseLoadBalancerIP: false, - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -181,9 +182,9 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: false, advertiseExternalIP: true, advertiseLoadBalancerIP: false, - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -223,9 +224,9 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: false, advertiseExternalIP: false, advertiseLoadBalancerIP: true, - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -265,9 +266,9 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: false, advertiseExternalIP: false, advertiseLoadBalancerIP: false, - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -336,9 +337,9 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: true, advertiseExternalIP: true, advertiseLoadBalancerIP: true, - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -394,9 +395,9 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: true, advertiseExternalIP: true, advertiseLoadBalancerIP: true, - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -444,9 +445,9 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: true, advertiseExternalIP: true, advertiseLoadBalancerIP: true, - primaryIP: net.ParseIP(testNodeIPv6), - nodeIPv6Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv6)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv6), + NodeIPv6Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv6)}}, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -494,9 +495,9 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: true, advertiseExternalIP: true, advertiseLoadBalancerIP: true, - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -540,9 +541,9 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: true, advertiseExternalIP: true, advertiseLoadBalancerIP: true, - primaryIP: net.ParseIP(testNodeIPv6), - nodeIPv6Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv6)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv6), + NodeIPv6Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv6)}}, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -586,9 +587,9 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: true, advertiseExternalIP: true, advertiseLoadBalancerIP: true, - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv6Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv6)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv6Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv6)}}, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -636,9 +637,9 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: true, advertiseExternalIP: true, advertiseLoadBalancerIP: true, - primaryIP: net.ParseIP(testNodeIPv6), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv6), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -686,9 +687,9 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: true, advertiseExternalIP: true, advertiseLoadBalancerIP: true, - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -732,9 +733,9 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: true, advertiseExternalIP: true, advertiseLoadBalancerIP: true, - primaryIP: net.ParseIP(testNodeIPv6), - nodeIPv6Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv6)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv6), + NodeIPv6Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv6)}}, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -778,9 +779,9 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: true, advertiseExternalIP: true, advertiseLoadBalancerIP: true, - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -824,9 +825,9 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: true, advertiseExternalIP: true, advertiseLoadBalancerIP: true, - primaryIP: net.ParseIP(testNodeIPv6), - nodeIPv6Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv6)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv6), + NodeIPv6Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv6)}}, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ diff --git a/pkg/controllers/routing/network_routes_controller.go b/pkg/controllers/routing/network_routes_controller.go index d6450d994e..9f7fcdc724 100644 --- a/pkg/controllers/routing/network_routes_controller.go +++ b/pkg/controllers/routing/network_routes_controller.go @@ -2,10 +2,8 @@ package routing import ( "context" - "encoding/binary" "errors" "fmt" - "hash/fnv" "net" "os" "os/exec" @@ -85,15 +83,8 @@ const ( // NetworkRoutingController is struct to hold necessary information required by controller type NetworkRoutingController struct { - primaryIP net.IP - nodeIPv4Addrs map[v1core.NodeAddressType][]net.IP - nodeIPv6Addrs map[v1core.NodeAddressType][]net.IP - nodeName string - nodeSubnet net.IPNet - nodeInterface string + krNode utils.NodeAware routerID string - isIPv4Capable bool - isIPv6Capable bool activeNodes map[string]bool mu sync.Mutex clientset kubernetes.Interface @@ -167,7 +158,7 @@ func (nrc *NetworkRoutingController) Run(healthChan chan<- *healthcheck.Controll } klog.V(1).Info("Populating ipsets.") - err = nrc.syncNodeIPSets() + err = nrc.syncNodeIPSets(nrc.krNode) if err != nil { klog.Errorf("Failed initial ipset setup: %s", err) } @@ -258,10 +249,10 @@ func (nrc *NetworkRoutingController) Run(healthChan chan<- *healthcheck.Controll } if nrc.autoMTU { - mtu, err := utils.GetMTUFromNodeIP(nrc.primaryIP) + mtu, err := nrc.krNode.GetNodeMTU() if err != nil { klog.Errorf("Failed to find MTU for node IP: %s for intelligently setting the kube-bridge MTU "+ - "due to %s.", nrc.primaryIP, err.Error()) + "due to %s.", nrc.krNode.GetPrimaryNodeIP(), err.Error()) } if mtu > 0 { klog.Infof("Setting MTU of kube-bridge interface to: %d", mtu) @@ -300,7 +291,7 @@ func (nrc *NetworkRoutingController) Run(healthChan chan<- *healthcheck.Controll klog.Errorf("Failed to enable iptables for bridge. Network policies and service proxy may "+ "not work: %s", sysctlErr.Error()) } - if nrc.isIPv6Capable { + if nrc.krNode.IsIPv6Capable() { sysctlErr = utils.SetSysctl(utils.BridgeNFCallIP6Tables, 1) if sysctlErr != nil { klog.Errorf("Failed to enable ip6tables for bridge. Network policies and service proxy may "+ @@ -358,7 +349,7 @@ func (nrc *NetworkRoutingController) Run(healthChan chan<- *healthcheck.Controll // Update ipset entries if nrc.enablePodEgress || nrc.enableOverlays { klog.V(1).Info("Syncing ipsets") - err = nrc.syncNodeIPSets() + err = nrc.syncNodeIPSets(nrc.krNode) if err != nil { klog.Errorf("Error synchronizing ipsets: %s", err.Error()) } @@ -436,7 +427,7 @@ func (nrc *NetworkRoutingController) updateCNIConfig() { if nrc.autoMTU { // Get the MTU by looking at the node's interface that is associated with the primary IP of the cluster - mtu, err := utils.GetMTUFromNodeIP(nrc.primaryIP) + mtu, err := nrc.krNode.GetNodeMTU() if err != nil { klog.Fatalf("failed to generate MTU: %v", err) } @@ -491,8 +482,8 @@ func (nrc *NetworkRoutingController) advertisePodRoute() error { } // Advertise IPv4 CIDRs - nodePrimaryIPv4IP := utils.FindBestIPv4NodeAddress(nrc.primaryIP, nrc.nodeIPv4Addrs) - if nrc.isIPv4Capable && nodePrimaryIPv4IP == nil { + nodePrimaryIPv4IP := nrc.krNode.FindBestIPv4NodeAddress() + if nrc.krNode.IsIPv4Capable() && nodePrimaryIPv4IP == nil { return fmt.Errorf("previous logic marked this node as IPv4 capable, but we couldn't find any " + "available IPv4 node IPs, this shouldn't happen") } @@ -502,7 +493,8 @@ func (nrc *NetworkRoutingController) advertisePodRoute() error { if err != nil || cidrLen < 0 || cidrLen > 32 { return fmt.Errorf("the pod CIDR IP given is not a proper mask: %d", cidrLen) } - klog.V(2).Infof("Advertising route: '%s/%d via %s' to peers", ip, cidrLen, nrc.primaryIP.String()) + klog.V(2).Infof("Advertising route: '%s/%d via %s' to peers", + ip, cidrLen, nrc.krNode.GetPrimaryNodeIP().String()) nlri, _ := anypb.New(&gobgpapi.IPAddressPrefix{ PrefixLen: uint32(cidrLen), Prefix: ip.String(), @@ -530,8 +522,8 @@ func (nrc *NetworkRoutingController) advertisePodRoute() error { } // Advertise IPv6 CIDRs - if nrc.isIPv6Capable { - nodePrimaryIPv6IP := utils.FindBestIPv6NodeAddress(nrc.primaryIP, nrc.nodeIPv6Addrs) + if nrc.krNode.IsIPv6Capable() { + nodePrimaryIPv6IP := nrc.krNode.FindBestIPv6NodeAddress() if nodePrimaryIPv6IP == nil { return fmt.Errorf("previous logic marked this node as IPv6 capable, but we couldn't find any " + "available IPv6 node IPs, this shouldn't happen") @@ -590,18 +582,16 @@ func (nrc *NetworkRoutingController) injectRoute(path *gobgpapi.Path) error { } tunnelName := generateTunnelName(nextHop.String()) - checkNHSameSubnet := func(protoIPSets map[v1core.NodeAddressType][]net.IP) bool { - for _, nodeIPSet := range protoIPSets { - for _, nodeIP := range nodeIPSet { - nodeSubnet, _, err := getNodeSubnet(nodeIP) - if err != nil { - klog.Warningf("unable to get subnet for node IP: %s, err: %v... skipping", nodeIP, err) - continue - } - // If we've found a subnet that contains our nextHop then we're done here - if nodeSubnet.Contains(nextHop) { - return true - } + checkNHSameSubnet := func(needle net.IP, haystack []net.IP) bool { + for _, nodeIP := range haystack { + nodeSubnet, _, err := utils.GetNodeSubnet(nodeIP, nil) + if err != nil { + klog.Warningf("unable to get subnet for node IP: %s, err: %v... skipping", nodeIP, err) + continue + } + // If we've found a subnet that contains our nextHop then we're done here + if nodeSubnet.Contains(needle) { + return true } } return false @@ -609,9 +599,9 @@ func (nrc *NetworkRoutingController) injectRoute(path *gobgpapi.Path) error { var sameSubnet bool if nextHop.To4() != nil { - sameSubnet = checkNHSameSubnet(nrc.nodeIPv4Addrs) + sameSubnet = checkNHSameSubnet(nextHop, nrc.krNode.GetNodeIPv4Addrs()) } else if nextHop.To16() != nil { - sameSubnet = checkNHSameSubnet(nrc.nodeIPv6Addrs) + sameSubnet = checkNHSameSubnet(nextHop, nrc.krNode.GetNodeIPv6Addrs()) } // If we've made it this far, then it is likely that the node is holding a destination route for this path already. @@ -673,10 +663,10 @@ func (nrc *NetworkRoutingController) injectRoute(path *gobgpapi.Path) error { // if we set up an overlay tunnel link, then use it for destination routing var bestIPForFamily net.IP if dst.IP.To4() != nil { - bestIPForFamily = utils.FindBestIPv4NodeAddress(nrc.primaryIP, nrc.nodeIPv4Addrs) + bestIPForFamily = nrc.krNode.FindBestIPv4NodeAddress() } else { // Need to activate the ip command in IPv6 mode - bestIPForFamily = utils.FindBestIPv6NodeAddress(nrc.primaryIP, nrc.nodeIPv6Addrs) + bestIPForFamily = nrc.krNode.FindBestIPv6NodeAddress() } if bestIPForFamily == nil { return fmt.Errorf("not able to find an appropriate configured IP address on node for destination "+ @@ -763,13 +753,13 @@ func (nrc *NetworkRoutingController) setupOverlayTunnel(tunnelName string, nextH strFormattedEncapPort := strconv.FormatInt(int64(nrc.overlayEncapPort), 10) if nextHop.To4() != nil { - bestIPForFamily = utils.FindBestIPv4NodeAddress(nrc.primaryIP, nrc.nodeIPv4Addrs) + bestIPForFamily = nrc.krNode.FindBestIPv4NodeAddress() ipipMode = encapTypeIPIP fouLinkType = ipipModev4 } else { // Need to activate the ip command in IPv6 mode ipBase = append(ipBase, "-6") - bestIPForFamily = utils.FindBestIPv6NodeAddress(nrc.primaryIP, nrc.nodeIPv6Addrs) + bestIPForFamily = nrc.krNode.FindBestIPv6NodeAddress() ipipMode = ipipModev6 fouLinkType = "ip6tnl" isIPv6 = true @@ -951,7 +941,7 @@ func (nrc *NetworkRoutingController) Cleanup() { klog.Infof("Successfully cleaned the NetworkRoutesController configuration done by kube-router") } -func (nrc *NetworkRoutingController) syncNodeIPSets() error { +func (nrc *NetworkRoutingController) syncNodeIPSets(nodeIPAware utils.NodeIPAware) error { var err error start := time.Now() defer func() { @@ -994,16 +984,11 @@ func (nrc *NetworkRoutingController) syncNodeIPSets() error { } var ipv4Addrs, ipv6Addrs [][]string - intExtNodeIPsv4, intExtNodeIPsv6 := utils.GetAllNodeIPs(node) - for _, nodeIPv4s := range intExtNodeIPsv4 { - for _, nodeIPv4 := range nodeIPv4s { - ipv4Addrs = append(ipv4Addrs, []string{nodeIPv4.String(), utils.OptionTimeout, "0"}) - } + for _, nodeIPv4 := range nodeIPAware.GetNodeIPv6Addrs() { + ipv4Addrs = append(ipv4Addrs, []string{nodeIPv4.String(), utils.OptionTimeout, "0"}) } - for _, nodeIPv6s := range intExtNodeIPsv6 { - for _, nodeIPv6 := range nodeIPv6s { - ipv6Addrs = append(ipv6Addrs, []string{nodeIPv6.String(), utils.OptionTimeout, "0"}) - } + for _, nodeIPv6 := range nodeIPAware.GetNodeIPv6Addrs() { + ipv6Addrs = append(ipv6Addrs, []string{nodeIPv6.String(), utils.OptionTimeout, "0"}) } currentNodeIPs[v1core.IPv4Protocol] = append(currentNodeIPs[v1core.IPv4Protocol], ipv4Addrs...) currentNodeIPs[v1core.IPv6Protocol] = append(currentNodeIPs[v1core.IPv6Protocol], ipv6Addrs...) @@ -1055,7 +1040,7 @@ func (nrc *NetworkRoutingController) enableForwarding() error { } comment = "allow outbound node port traffic on node interface with which node ip is associated" - args = []string{"-m", "comment", "--comment", comment, "-o", nrc.nodeInterface, "-j", "ACCEPT"} + args = []string{"-m", "comment", "--comment", comment, "-o", nrc.krNode.GetNodeInterfaceName(), "-j", "ACCEPT"} exists, err = iptablesCmdHandler.Exists("filter", "FORWARD", args...) if err != nil { return fmt.Errorf("failed to run iptables command: %s", err.Error()) @@ -1178,7 +1163,8 @@ func (nrc *NetworkRoutingController) startBgpServer(grpcServer bool) error { if grpcServer { nrc.bgpServer = gobgp.NewBgpServer( - gobgp.GrpcListenAddress(net.JoinHostPort(nrc.primaryIP.String(), "50051") + "," + "127.0.0.1:50051")) + gobgp.GrpcListenAddress(net.JoinHostPort(nrc.krNode.GetPrimaryNodeIP().String(), + "50051") + "," + "127.0.0.1:50051")) } else { nrc.bgpServer = gobgp.NewBgpServer() } @@ -1186,12 +1172,12 @@ func (nrc *NetworkRoutingController) startBgpServer(grpcServer bool) error { var localAddressList []string - if nrc.isIPv4Capable && !utils.ContainsIPv4Address(nrc.localAddressList) { + if nrc.krNode.IsIPv4Capable() && !utils.ContainsIPv4Address(nrc.localAddressList) { klog.Warningf("List of local addresses did not contain a valid IPv4 address, but IPv4 was " + "enabled in kube-router's CLI options. BGP may not work as expected!") } - if nrc.isIPv6Capable && !utils.ContainsIPv6Address(nrc.localAddressList) { + if nrc.krNode.IsIPv6Capable() && !utils.ContainsIPv6Address(nrc.localAddressList) { klog.Warningf("List of local addresses did not contain a valid IPv6 address, but IPv6 was " + "enabled in kube-router's CLI options. BGP may not work as expected!") } @@ -1207,13 +1193,13 @@ func (nrc *NetworkRoutingController) startBgpServer(grpcServer bool) error { // Make sure that the address type matches what we're capable of before listening if ip.To4() != nil { - if !nrc.isIPv4Capable { + if !nrc.krNode.IsIPv4Capable() { klog.Warningf("was configured to listen on %s, but node is not enabled for IPv4 or does not "+ "have any IPv4 addresses configured for it, skipping", addr) continue } } else { - if !nrc.isIPv6Capable { + if !nrc.krNode.IsIPv6Capable() { klog.Warningf("was configured to listen on %s, but node is not enabled for IPv6 or does not "+ "have any IPv6 addresses configured for it, skipping", addr) continue @@ -1338,7 +1324,7 @@ func (nrc *NetworkRoutingController) startBgpServer(grpcServer bool) error { // Create and set Global Peer Router complete configs nrc.globalPeerRouters, err = newGlobalPeers(peerIPs, peerPorts, peerASNs, peerPasswords, peerLocalIPs, - nrc.bgpHoldtime, nrc.primaryIP.String()) + nrc.bgpHoldtime, nrc.krNode.GetPrimaryNodeIP().String()) if err != nil { err2 := nrc.bgpServer.StopBgp(context.Background(), &gobgpapi.StopBgpRequest{}) if err2 != nil { @@ -1386,7 +1372,7 @@ func (nrc *NetworkRoutingController) setupHandlers(node *v1core.Node) error { nrc.iptablesCmdHandlers = make(map[v1core.IPFamily]utils.IPTablesHandler) nrc.ipSetHandlers = make(map[v1core.IPFamily]utils.IPSetHandler) - if node == nil || len(nrc.nodeIPv4Addrs) > 0 { + if node == nil || nrc.krNode.IsIPv4Capable() { iptHandler, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) if err != nil { klog.Fatalf("Failed to allocate IPv4 iptables handler: %v", err) @@ -1401,7 +1387,7 @@ func (nrc *NetworkRoutingController) setupHandlers(node *v1core.Node) error { } nrc.ipSetHandlers[v1core.IPv4Protocol] = ipset } - if node == nil || len(nrc.nodeIPv6Addrs) > 0 { + if node == nil || nrc.krNode.IsIPv6Capable() { iptHandler, err := iptables.NewWithProtocol(iptables.ProtocolIPv6) if err != nil { klog.Fatalf("Failed to allocate IPv6 iptables handler: %v", err) @@ -1470,29 +1456,10 @@ func NewNetworkRoutingController(clientset kubernetes.Interface, return nil, errors.New("failed getting node object from API server: " + err.Error()) } - nrc.nodeName = node.Name - - // We preserve the old logic here for getting the primary IP which is set on nrc.primaryIP. This can be either IPv4 - // or IPv6 - nodeIP, err := utils.GetPrimaryNodeIP(node) + nrc.krNode, err = utils.NewKRNode(node, nil, kubeRouterConfig.EnableIPv4, kubeRouterConfig.EnableIPv6) if err != nil { - return nil, errors.New("failed getting IP address from node object: " + err.Error()) - } - nrc.primaryIP = nodeIP - - // Here we test to see whether the node is IPv6 capable, if the user has enabled IPv6 (via command-line options) - // and the node has an IPv6 address, the following method will return an IPv6 address - nrc.nodeIPv4Addrs, nrc.nodeIPv6Addrs = utils.GetAllNodeIPs(node) - if kubeRouterConfig.EnableIPv4 && len(nrc.nodeIPv4Addrs[v1core.NodeInternalIP]) < 1 && - len(nrc.nodeIPv4Addrs[v1core.NodeExternalIP]) < 1 { - return nil, fmt.Errorf("IPv4 was enabled, but no IPv4 address was found on the node") - } - nrc.isIPv4Capable = len(nrc.nodeIPv4Addrs) > 0 - if kubeRouterConfig.EnableIPv6 && len(nrc.nodeIPv6Addrs[v1core.NodeInternalIP]) < 1 && - len(nrc.nodeIPv6Addrs[v1core.NodeExternalIP]) < 1 { - return nil, fmt.Errorf("IPv6 was enabled, but no IPv6 address was found on the node") + return nil, err } - nrc.isIPv6Capable = len(nrc.nodeIPv6Addrs) > 0 if kubeRouterConfig.EnableIPv6 { sysctlErr := utils.SetSysctl(utils.IPv6ConfAllDisableIPv6, 0) @@ -1502,21 +1469,9 @@ func NewNetworkRoutingController(clientset kubernetes.Interface, } } - switch { - case kubeRouterConfig.RouterID == "generate": - h := fnv.New32a() - h.Write(nrc.primaryIP) - hs := h.Sum32() - gip := make(net.IP, 4) - binary.BigEndian.PutUint32(gip, hs) - nrc.routerID = gip.String() - case kubeRouterConfig.RouterID != "": - nrc.routerID = kubeRouterConfig.RouterID - default: - if nrc.primaryIP.To4() == nil { - return nil, errors.New("router-id must be specified when primary node IP is an IPv6 address") - } - nrc.routerID = nrc.primaryIP.String() + nrc.routerID, err = generateRouterID(nrc.krNode, kubeRouterConfig.RouterID) + if err != nil { + return nil, err } // let's start with assumption we have necessary IAM creds to access EC2 api @@ -1627,26 +1582,18 @@ func NewNetworkRoutingController(clientset kubernetes.Interface, } nrc.globalPeerRouters, err = newGlobalPeers(kubeRouterConfig.PeerRouters, peerPorts, - peerASNs, peerPasswords, nil, nrc.bgpHoldtime, nrc.primaryIP.String()) + peerASNs, peerPasswords, nil, nrc.bgpHoldtime, nrc.krNode.GetPrimaryNodeIP().String()) if err != nil { return nil, fmt.Errorf("error processing Global Peer Router configs: %s", err) } - nrc.nodeSubnet, nrc.nodeInterface, err = getNodeSubnet(nodeIP) - if err != nil { - return nil, errors.New("failed find the subnet of the node IP and interface on" + - "which its configured: " + err.Error()) - } - bgpLocalAddressListAnnotation, ok := node.ObjectMeta.Annotations[bgpLocalAddressAnnotation] if !ok { - if nrc.isIPv4Capable { - nrc.localAddressList = append(nrc.localAddressList, - utils.FindBestIPv4NodeAddress(nrc.primaryIP, nrc.nodeIPv4Addrs).String()) + if nrc.krNode.IsIPv4Capable() { + nrc.localAddressList = append(nrc.localAddressList, nrc.krNode.FindBestIPv4NodeAddress().String()) } - if nrc.isIPv6Capable { - nrc.localAddressList = append(nrc.localAddressList, - utils.FindBestIPv6NodeAddress(nrc.primaryIP, nrc.nodeIPv6Addrs).String()) + if nrc.krNode.IsIPv6Capable() { + nrc.localAddressList = append(nrc.localAddressList, nrc.krNode.FindBestIPv6NodeAddress().String()) } klog.Infof("Could not find annotation `kube-router.io/bgp-local-addresses` on node object so BGP "+ "will listen on node IP: %s addresses.", nrc.localAddressList) diff --git a/pkg/controllers/routing/network_routes_controller_test.go b/pkg/controllers/routing/network_routes_controller_test.go index 6847ec0df2..acd72aad22 100644 --- a/pkg/controllers/routing/network_routes_controller_test.go +++ b/pkg/controllers/routing/network_routes_controller_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/cloudnativelabs/kube-router/v2/pkg/utils" "github.com/stretchr/testify/assert" v1core "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -36,9 +37,9 @@ func Test_advertiseClusterIPs(t *testing.T) { "add bgp path for service with ClusterIP", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, []*v1core.Service{ @@ -80,9 +81,9 @@ func Test_advertiseClusterIPs(t *testing.T) { "add bgp path for service with ClusterIP/NodePort/LoadBalancer", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, []*v1core.Service{ @@ -180,9 +181,9 @@ func Test_advertiseClusterIPs(t *testing.T) { "add bgp path for invalid service type", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, []*v1core.Service{ @@ -250,9 +251,9 @@ func Test_advertiseClusterIPs(t *testing.T) { "add bgp path for headless service", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, []*v1core.Service{ @@ -465,9 +466,9 @@ func Test_advertiseExternalIPs(t *testing.T) { "add bgp path for service with external IPs", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - primaryIP: net.ParseIP("10.0.0.1"), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP("10.0.0.1"), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, []*v1core.Service{ @@ -511,9 +512,9 @@ func Test_advertiseExternalIPs(t *testing.T) { "add bgp path for services with external IPs of type ClusterIP/NodePort/LoadBalancer", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, []*v1core.Service{ @@ -615,9 +616,9 @@ func Test_advertiseExternalIPs(t *testing.T) { "add bgp path for invalid service type", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, []*v1core.Service{ @@ -688,9 +689,9 @@ func Test_advertiseExternalIPs(t *testing.T) { "add bgp path for headless service", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, []*v1core.Service{ @@ -789,9 +790,9 @@ func Test_advertiseExternalIPs(t *testing.T) { "skip bgp path to loadbalancerIP for service without LoadBalancer IP", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, []*v1core.Service{ @@ -840,9 +841,9 @@ func Test_advertiseExternalIPs(t *testing.T) { "add bgp path to loadbalancerIP for service with LoadBalancer IP", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, []*v1core.Service{ @@ -897,9 +898,9 @@ func Test_advertiseExternalIPs(t *testing.T) { "no bgp path to nil loadbalancerIPs for service with LoadBalancer", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, []*v1core.Service{ @@ -944,9 +945,9 @@ func Test_advertiseExternalIPs(t *testing.T) { "no bgp path to loadbalancerIPs for service with LoadBalancer and skiplbips annotation", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, []*v1core.Service{ @@ -1118,9 +1119,9 @@ func Test_advertiseAnnotationOptOut(t *testing.T) { "add bgp paths for all service IPs", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, []*v1core.Service{ @@ -1239,9 +1240,9 @@ func Test_advertiseAnnotationOptOut(t *testing.T) { "opt out to advertise any IPs via annotations", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, []*v1core.Service{ @@ -1417,9 +1418,9 @@ func Test_advertiseAnnotationOptIn(t *testing.T) { "no bgp paths for any service IPs", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, []*v1core.Service{ @@ -1530,9 +1531,9 @@ func Test_advertiseAnnotationOptIn(t *testing.T) { "opt in to advertise all IPs via annotations", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, []*v1core.Service{ @@ -1784,10 +1785,10 @@ func Test_nodeHasEndpointsForService(t *testing.T) { { "node has endpoints for service", &NetworkRoutingController{ - nodeName: "node-1", - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, &v1core.Service{ @@ -1829,10 +1830,10 @@ func Test_nodeHasEndpointsForService(t *testing.T) { { "node has no endpoints for service", &NetworkRoutingController{ - nodeName: "node-1", - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, &v1core.Service{ @@ -1920,13 +1921,13 @@ func Test_advertisePodRoute(t *testing.T) { { "add bgp path for pod cidr using NODE_NAME", &NetworkRoutingController{ - bgpServer: gobgp.NewBgpServer(), - podCidr: "172.20.0.0/24", - isIPv4Capable: true, - podIPv4CIDRs: []string{"172.20.0.0/24"}, - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + bgpServer: gobgp.NewBgpServer(), + podCidr: "172.20.0.0/24", + podIPv4CIDRs: []string{"172.20.0.0/24"}, + krNode: &utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, "node-1", @@ -1952,11 +1953,11 @@ func Test_advertisePodRoute(t *testing.T) { bgpServer: gobgp.NewBgpServer(), hostnameOverride: "node-1", podCidr: "172.20.0.0/24", - isIPv4Capable: true, podIPv4CIDRs: []string{"172.20.0.0/24"}, - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, }, "", @@ -1983,13 +1984,13 @@ func Test_advertisePodRoute(t *testing.T) { hostnameOverride: "node-1", podCidr: "2001:db8:42:2::/64", podIPv6CIDRs: []string{"2001:db8:42:2::/64"}, - nodeIPv6Addrs: map[v1core.NodeAddressType][]net.IP{ - v1core.NodeInternalIP: {net.IPv6loopback}, - }, - isIPv6Capable: true, - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + NodeIPv6Addrs: map[v1core.NodeAddressType][]net.IP{ + v1core.NodeInternalIP: {net.IPv6loopback}, + }, }, }, "", @@ -2161,9 +2162,10 @@ func Test_syncInternalPeers(t *testing.T) { &NetworkRoutingController{ bgpFullMeshMode: true, clientset: fake.NewSimpleClientset(), - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, bgpServer: gobgp.NewBgpServer(), activeNodes: make(map[string]bool), @@ -2192,9 +2194,10 @@ func Test_syncInternalPeers(t *testing.T) { &NetworkRoutingController{ bgpFullMeshMode: true, clientset: fake.NewSimpleClientset(), - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, bgpServer: gobgp.NewBgpServer(), activeNodes: make(map[string]bool), @@ -2237,9 +2240,10 @@ func Test_syncInternalPeers(t *testing.T) { &NetworkRoutingController{ bgpFullMeshMode: true, clientset: fake.NewSimpleClientset(), - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, bgpServer: gobgp.NewBgpServer(), activeNodes: map[string]bool{ @@ -2270,9 +2274,10 @@ func Test_syncInternalPeers(t *testing.T) { &NetworkRoutingController{ bgpFullMeshMode: false, clientset: fake.NewSimpleClientset(), - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, bgpServer: gobgp.NewBgpServer(), activeNodes: make(map[string]bool), @@ -2382,9 +2387,10 @@ func Test_routeReflectorConfiguration(t *testing.T) { bgpFullMeshMode: false, bgpPort: 10000, clientset: fake.NewSimpleClientset(), - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, routerID: testNodeIPv4, bgpServer: gobgp.NewBgpServer(), @@ -2412,9 +2418,10 @@ func Test_routeReflectorConfiguration(t *testing.T) { bgpFullMeshMode: false, bgpPort: 10000, clientset: fake.NewSimpleClientset(), - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, routerID: testNodeIPv4, bgpServer: gobgp.NewBgpServer(), @@ -2442,9 +2449,10 @@ func Test_routeReflectorConfiguration(t *testing.T) { bgpFullMeshMode: false, bgpPort: 10000, clientset: fake.NewSimpleClientset(), - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, routerID: testNodeIPv4, bgpServer: gobgp.NewBgpServer(), @@ -2472,9 +2480,10 @@ func Test_routeReflectorConfiguration(t *testing.T) { bgpFullMeshMode: false, bgpPort: 10000, clientset: fake.NewSimpleClientset(), - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, routerID: testNodeIPv4, bgpServer: gobgp.NewBgpServer(), @@ -2502,9 +2511,10 @@ func Test_routeReflectorConfiguration(t *testing.T) { bgpFullMeshMode: false, bgpPort: 10000, clientset: fake.NewSimpleClientset(), - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, bgpServer: gobgp.NewBgpServer(), activeNodes: make(map[string]bool), @@ -2531,9 +2541,10 @@ func Test_routeReflectorConfiguration(t *testing.T) { bgpFullMeshMode: false, bgpPort: 10000, clientset: fake.NewSimpleClientset(), - primaryIP: net.ParseIP(testNodeIPv4), - nodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: { - net.ParseIP(testNodeIPv4)}, + krNode: &utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, }, bgpServer: gobgp.NewBgpServer(), activeNodes: make(map[string]bool), diff --git a/pkg/controllers/routing/pbr.go b/pkg/controllers/routing/pbr.go index cd93a27edb..d345ccd336 100644 --- a/pkg/controllers/routing/pbr.go +++ b/pkg/controllers/routing/pbr.go @@ -41,14 +41,14 @@ func (nrc *NetworkRoutingController) enablePolicyBasedRouting() error { return fmt.Errorf("failed to update rt_tables file: %s", err) } - if nrc.isIPv4Capable { + if nrc.krNode.IsIPv4Capable() { for _, ipv4CIDR := range nrc.podIPv4CIDRs { if err := ipRuleAbstraction("-4", "add", ipv4CIDR); err != nil { return err } } } - if nrc.isIPv6Capable { + if nrc.krNode.IsIPv6Capable() { for _, ipv6CIDR := range nrc.podIPv6CIDRs { if err := ipRuleAbstraction("-6", "add", ipv6CIDR); err != nil { return err @@ -65,14 +65,14 @@ func (nrc *NetworkRoutingController) disablePolicyBasedRouting() error { return fmt.Errorf("failed to update rt_tables file: %s", err) } - if nrc.isIPv4Capable { + if nrc.krNode.IsIPv4Capable() { for _, ipv4CIDR := range nrc.podIPv4CIDRs { if err := ipRuleAbstraction("-4", "del", ipv4CIDR); err != nil { return err } } } - if nrc.isIPv6Capable { + if nrc.krNode.IsIPv6Capable() { for _, ipv6CIDR := range nrc.podIPv6CIDRs { if err := ipRuleAbstraction("-6", "del", ipv6CIDR); err != nil { return err diff --git a/pkg/controllers/routing/utils.go b/pkg/controllers/routing/utils.go index 455240e9fa..08d99e1825 100644 --- a/pkg/controllers/routing/utils.go +++ b/pkg/controllers/routing/utils.go @@ -4,8 +4,10 @@ import ( "bufio" "crypto/sha256" "encoding/base64" + "encoding/binary" "errors" "fmt" + "hash/fnv" "net" "os/exec" "regexp" @@ -119,23 +121,26 @@ func statementsEqualByName(a, b []*gobgpapi.Statement) bool { return true } -func getNodeSubnet(nodeIP net.IP) (net.IPNet, string, error) { - links, err := netlink.LinkList() - if err != nil { - return net.IPNet{}, "", errors.New("failed to get list of links") +// generateRouterID will generate a router ID based upon the user's configuration (or lack there of) and the node's +// primary IP address if the user has not specified. If the user has configured the router ID as "generate" then we +// will generate a router ID based upon fnv hashing the node's primary IP address. +func generateRouterID(nodeIPAware utils.NodeIPAware, configRouterID string) (string, error) { + switch { + case configRouterID == "generate": + h := fnv.New32a() + h.Write(nodeIPAware.GetPrimaryNodeIP()) + hs := h.Sum32() + gip := make(net.IP, 4) + binary.BigEndian.PutUint32(gip, hs) + return gip.String(), nil + case configRouterID != "": + return configRouterID, nil } - for _, link := range links { - addresses, err := netlink.AddrList(link, netlink.FAMILY_ALL) - if err != nil { - return net.IPNet{}, "", errors.New("failed to get list of addr") - } - for _, addr := range addresses { - if addr.IPNet.IP.Equal(nodeIP) { - return *addr.IPNet, link.Attrs().Name, nil - } - } + + if nodeIPAware.GetPrimaryNodeIP().To4() == nil { + return "", errors.New("router-id must be specified when primary node IP is an IPv6 address") } - return net.IPNet{}, "", errors.New("failed to find interface with specified node ip") + return configRouterID, nil } // generateTunnelName will generate a name for a tunnel interface given a node IP @@ -306,7 +311,7 @@ func (nrc *NetworkRoutingController) getBGPRouteInfoForVIP(vip string) (subnet u if ip.To4() != nil { subnet = 32 afiFamily = gobgpapi.Family_AFI_IP - nhIP := utils.FindBestIPv4NodeAddress(nrc.primaryIP, nrc.nodeIPv4Addrs) + nhIP := nrc.krNode.FindBestIPv4NodeAddress() if nhIP == nil { err = fmt.Errorf("could not find an IPv4 address on node to set as nexthop for vip: %s", vip) } @@ -316,7 +321,7 @@ func (nrc *NetworkRoutingController) getBGPRouteInfoForVIP(vip string) (subnet u if ip.To16() != nil { subnet = 128 afiFamily = gobgpapi.Family_AFI_IP6 - nhIP := utils.FindBestIPv6NodeAddress(nrc.primaryIP, nrc.nodeIPv6Addrs) + nhIP := nrc.krNode.FindBestIPv6NodeAddress() if nhIP == nil { err = fmt.Errorf("could not find an IPv6 address on node to set as nexthop for vip: %s", vip) } diff --git a/pkg/utils/linux_routing.go b/pkg/utils/linux_routing.go index 78f2f3b527..fa6cf99bdb 100644 --- a/pkg/utils/linux_routing.go +++ b/pkg/utils/linux_routing.go @@ -5,6 +5,7 @@ import ( "os" "strings" + "github.com/vishvananda/netlink" "k8s.io/klog/v2" ) @@ -21,6 +22,11 @@ var ( } ) +type LocalLinkQuerier interface { + LinkList() ([]netlink.Link, error) + AddrList(link netlink.Link, family int) ([]netlink.Addr, error) +} + // RouteTableAdd adds a new named table to iproute's rt_tables configuration file func RouteTableAdd(tableNumber, tableName string) error { var rtTablesLoc string diff --git a/pkg/utils/node.go b/pkg/utils/node.go index 3ce03153c1..ce33cb1599 100644 --- a/pkg/utils/node.go +++ b/pkg/utils/node.go @@ -15,6 +15,291 @@ import ( netutils "k8s.io/utils/net" ) +// nodeAddressMap is a mapping of address types to a list of addresses of that type. +// It preallocates the slices of addresses. +type nodeAddressMap map[apiv1.NodeAddressType][]apiv1.NodeAddress + +// add adds an address of the given type to the address map. If the given type +// was not already in the map, it creates a new preallocated entry for it. +func (m nodeAddressMap) add(address apiv1.NodeAddress) { + if _, ok := m[address.Type]; ok { + m[address.Type] = append(m[address.Type], address) + } else { + m[address.Type] = make([]apiv1.NodeAddress, 0) + m[address.Type] = append(m[address.Type], address) + } +} + +// nodeAddressMap is a mapping of address types to a list of addresses of that type. +// It preallocates the slices of addresses. +type addressMap map[apiv1.NodeAddressType][]net.IP + +// KRNode is a struct that holds information about a node that is used by kube-router. +type KRNode struct { + NodeInterfaceName string + NodeIPv4Addrs addressMap + NodeIPv6Addrs addressMap + NodeName string + linkQ LocalLinkQuerier + PrimaryNodeSubnet net.IPNet + PrimaryIP net.IP +} + +// NodeIPAware is an interface that provides methods to get the node's IP addresses in various data structures. +type NodeIPAware interface { + FindBestIPv4NodeAddress() net.IP + FindBestIPv6NodeAddress() net.IP + GetNodeIPv4AddrsTypeMap() addressMap + GetNodeIPv6AddrsTypeMap() addressMap + GetNodeIPv4Addrs() []net.IP + GetNodeIPv6Addrs() []net.IP + GetNodeIPAddrs() []net.IP + GetPrimaryNodeIP() net.IP +} + +// NodeInterfaceAware is an interface that provides methods to get the node's interface name, MTU, and subnet. This +// interface is a collection of functions that are only available if you are running on the node itself, as kube-router +// determines this by looking at the node's interfaces and parsing the address data there. If you attempt to call these +// functions on a remote node, they will return nil or an error. +type NodeInterfaceAware interface { + GetNodeInterfaceName() string + GetPrimaryNodeSubnet() net.IPNet + GetNodeMTU() (int, error) +} + +// NodeFamilyAware is an interface that provides methods to check if a node is IPv4 or IPv6 capable. +type NodeFamilyAware interface { + IsIPv4Capable() bool + IsIPv6Capable() bool +} + +// NodeNameAware is an interface that provides a method to get the node's name. +type NodeNameAware interface { + GetNodeName() string +} + +// NodeIPAndFamilyAware is an interface that combines the NodeIPAware and NodeFamilyAware interfaces. +type NodeIPAndFamilyAware interface { + NodeIPAware + NodeFamilyAware +} + +// NodeAware is an interface that combines the NodeIPAware, NodeInterfaceAware, NodeFamilyAware, and NodeNameAware +// interfaces. +type NodeAware interface { + NodeIPAware + NodeInterfaceAware + NodeFamilyAware + NodeNameAware +} + +// GetNodeIPv4AddrsTypeMap returns the node's IPv4 addresses grouped by Kubernetes Node Object address type (internal / +// external). +func (n *KRNode) GetNodeIPv4AddrsTypeMap() addressMap { + return n.NodeIPv4Addrs +} + +// GetNodeIPv4Addrs returns the node's IPv4 addresses as defined by the Kubernetes Node Object. +func (n *KRNode) GetNodeIPv4Addrs() []net.IP { + var nodeIPs []net.IP + for _, ip := range n.NodeIPv4Addrs { + nodeIPs = append(nodeIPs, ip...) + } + return nodeIPs +} + +// GetNodeIPv6AddrsTypeMap returns the node's IPv6 addresses grouped by Kubernetes Node Object address type (internal / +// external). +func (n *KRNode) GetNodeIPv6AddrsTypeMap() addressMap { + return n.NodeIPv6Addrs +} + +// GetNodeIPv6Addrs returns the node's IPv6 addresses as defined by the Kubernetes Node Object. +func (n *KRNode) GetNodeIPv6Addrs() []net.IP { + var nodeIPs []net.IP + for _, ip := range n.NodeIPv6Addrs { + nodeIPs = append(nodeIPs, ip...) + } + return nodeIPs +} + +// GetPrimaryNodeIP returns the node's primary IP address which for the purposes of kube-router is defined as the first +// internal address defined on the Kubernetes node object. If no internal address is defined, the first external address +// is used. +func (n *KRNode) GetPrimaryNodeIP() net.IP { + return n.PrimaryIP +} + +// GetPrimaryNodeSubnet returns the node's primary subnet as defined by the primary IP address. This function is only +// available if you are running on the node itself, as kube-router determines this by looking at the node's interfaces +// and parsing the address data there. If you attempt to call this function on a remote node, it will return nil. +func (n *KRNode) GetPrimaryNodeSubnet() net.IPNet { + return n.PrimaryNodeSubnet +} + +// GetNodeInterfaceName returns the node's interface name as defined by the primary IP address. This function is only +// available if you are running on the node itself, as kube-router determines this by looking at the node's interfaces +// and parsing the address data there. If you attempt to call this function on a remote node, it will return nil. +func (n *KRNode) GetNodeInterfaceName() string { + return n.NodeInterfaceName +} + +// IsIPv4Capable returns true if the node has at least one IPv4 address defined in the Kubernetes Node Object. +func (n *KRNode) IsIPv4Capable() bool { + return len(n.NodeIPv4Addrs[apiv1.NodeInternalIP]) > 0 || len(n.NodeIPv4Addrs[apiv1.NodeExternalIP]) > 0 +} + +// IsIPv6Capable returns true if the node has at least one IPv6 address defined in the Kubernetes Node Object. +func (n *KRNode) IsIPv6Capable() bool { + return len(n.NodeIPv6Addrs[apiv1.NodeInternalIP]) > 0 || len(n.NodeIPv6Addrs[apiv1.NodeExternalIP]) > 0 +} + +// GetNodeName returns the node's name as defined by the Kubernetes Node Object. +func (n *KRNode) GetNodeName() string { + return n.NodeName +} + +// FindBestIPv6NodeAddress returns the best available IPv6 address for the node. If the primary IP is already an IPv6 +// address, it will return that. Otherwise, it will return the first internal or external IPv6 address defined in the +// Kubernetes Node Object. +func (n KRNode) FindBestIPv6NodeAddress() net.IP { + if n.PrimaryIP != nil && n.PrimaryIP.To4() == nil && n.PrimaryIP.To16() != nil { + // the NRC's primary IP is already an IPv6 address, so we'll use that + return n.PrimaryIP + } + // the NRC's primary IP is not an IPv6, let's try to find the best available IPv6 address out of our + // available node addresses to use as the nextHop for our route + if n.NodeIPv6Addrs != nil { + if len(n.NodeIPv6Addrs[apiv1.NodeInternalIP]) > 0 { + return n.NodeIPv6Addrs[apiv1.NodeInternalIP][0] + } else if len(n.NodeIPv6Addrs[apiv1.NodeExternalIP]) > 0 { + return n.NodeIPv6Addrs[apiv1.NodeExternalIP][0] + } + } + return nil +} + +// FindBestIPv4NodeAddress returns the best available IPv4 address for the node. If the primary IP is already an IPv4 +// address, it will return that. Otherwise, it will return the first internal or external IPv4 address defined in the +// Kubernetes Node Object. +func (n KRNode) FindBestIPv4NodeAddress() net.IP { + if n.PrimaryIP != nil && n.PrimaryIP.To4() != nil { + // the NRC's primary IP is already an IPv6 address, so we'll use that + return n.PrimaryIP + } + // the NRC's primary IP is not an IPv6, let's try to find the best available IPv6 address out of our + // available node addresses to use as the nextHop for our route + if n.NodeIPv4Addrs != nil { + if len(n.NodeIPv4Addrs[apiv1.NodeInternalIP]) > 0 { + return n.NodeIPv4Addrs[apiv1.NodeInternalIP][0] + } else if len(n.NodeIPv4Addrs[apiv1.NodeExternalIP]) > 0 { + return n.NodeIPv4Addrs[apiv1.NodeExternalIP][0] + } + } + return nil +} + +// GetNodeMTU returns the MTU of the interface that the node's primary IP address is assigned to. This function is only +// available if you are running on the node itself, as kube-router determines this by looking at the node's interfaces +// and parsing the address data there. If you attempt to call this function on a remote node, it will return an error. +func (n KRNode) GetNodeMTU() (int, error) { + links, err := n.linkQ.LinkList() + if err != nil { + return 0, errors.New("failed to get list of links") + } + for _, link := range links { + addresses, err := n.linkQ.AddrList(link, netlink.FAMILY_ALL) + if err != nil { + return 0, errors.New("failed to get list of addr") + } + for _, addr := range addresses { + if addr.IPNet.IP.Equal(n.PrimaryIP) { + linkMTU := link.Attrs().MTU + return linkMTU, nil + } + } + } + return 0, errors.New("failed to find interface with specified node IP") +} + +// GetNodeIPAddrs returns all of the node's IP addresses (whether internal or external) as defined by the Kubernetes +// Node Object. +func (n KRNode) GetNodeIPAddrs() []net.IP { + var nodeIPs []net.IP + for _, ip := range n.NodeIPv4Addrs { + nodeIPs = append(nodeIPs, ip...) + } + for _, ip := range n.NodeIPv6Addrs { + nodeIPs = append(nodeIPs, ip...) + } + return nodeIPs +} + +// NewKRNode creates a new KRNode object from a Kubernetes Node Object. This function is used when kube-router is +// running on the node itself and has access to the node's interfaces and address data. If you attempt to run this on +// a remote node, it will result in an error as it will not be able to find the correct subnet / interface information. +// For this use-case use NewRemoteKRNode instead. It will also return an error if the node does not have any IPv4 or +// IPv6 addresses defined in the Kubernetes Node Object. +func NewKRNode(node *apiv1.Node, linkQ LocalLinkQuerier, enableIPv4, enableIPv6 bool) (NodeAware, error) { + if linkQ == nil { + linkQ = &netlink.Handle{} + } + + primaryNodeIP, err := getPrimaryNodeIP(node) + if err != nil { + return nil, fmt.Errorf("error getting primary NodeIP: %w", err) + } + + ipv4Addrs, ipv6Addrs := getAllNodeIPs(node) + if enableIPv4 && len(ipv4Addrs[apiv1.NodeInternalIP]) < 1 && + len(ipv4Addrs[apiv1.NodeExternalIP]) < 1 { + return nil, fmt.Errorf("IPv4 was enabled, but no IPv4 address was found on the node") + } + + if enableIPv6 && len(ipv6Addrs[apiv1.NodeInternalIP]) < 1 && + len(ipv6Addrs[apiv1.NodeExternalIP]) < 1 { + return nil, fmt.Errorf("IPv6 was enabled, but no IPv6 address was found on the node") + } + + primaryNodeSubnet, nodeInterfaceName, err := GetNodeSubnet(primaryNodeIP, linkQ) + if err != nil { + return nil, fmt.Errorf("error getting node subnet: %w", err) + } + + krNode := &KRNode{ + linkQ: linkQ, + NodeName: node.Name, + PrimaryIP: primaryNodeIP, + NodeIPv4Addrs: ipv4Addrs, + NodeIPv6Addrs: ipv6Addrs, + PrimaryNodeSubnet: primaryNodeSubnet, + NodeInterfaceName: nodeInterfaceName, + } + + return krNode, nil +} + +// NewRemoteKRNode creates a new KRNode object from a Kubernetes Node Object. This function is used when kube-router is +// attempting to parse a remote node and does not have access to the node's interfaces and address data. It will return +// an error if the node does not have any IPv4 or IPv6 addresses defined in the Kubernetes Node Object. +func NewRemoteKRNode(node *apiv1.Node) (NodeIPAndFamilyAware, error) { + primaryNodeIP, err := getPrimaryNodeIP(node) + if err != nil { + return nil, fmt.Errorf("error getting primary NodeIP: %w", err) + } + + ipv4Addrs, ipv6Addrs := getAllNodeIPs(node) + + krNode := &KRNode{ + NodeName: node.Name, + PrimaryIP: primaryNodeIP, + NodeIPv4Addrs: ipv4Addrs, + NodeIPv6Addrs: ipv6Addrs, + } + + return krNode, nil +} + // GetNodeObject returns the node API object for the node func GetNodeObject(clientset kubernetes.Interface, hostnameOverride string) (*apiv1.Node, error) { // if env NODE_NAME is not set and node is not registered with hostname, then use host name override @@ -46,11 +331,11 @@ func GetNodeObject(clientset kubernetes.Interface, hostnameOverride string) (*ap return node, nil } -// GetPrimaryNodeIP returns the most valid external facing IP address for a node. +// getPrimaryNodeIP returns the most valid external facing IP address for a node. // Order of preference: // 1. NodeInternalIP -// 2. NodeExternalIP (Only set on cloud providers usually) -func GetPrimaryNodeIP(node *apiv1.Node) (net.IP, error) { +// 2. NodeExternalIP (usually only set on cloud providers usually) +func getPrimaryNodeIP(node *apiv1.Node) (net.IP, error) { addresses := node.Status.Addresses addressMap := make(map[apiv1.NodeAddressType][]apiv1.NodeAddress) for i := range addresses { @@ -65,28 +350,13 @@ func GetPrimaryNodeIP(node *apiv1.Node) (net.IP, error) { return nil, errors.New("host IP unknown") } -// addressMap is a mapping of address types to a list of addresses of that type. -// It preallocates the slices of addresses. -type addressMap map[apiv1.NodeAddressType][]apiv1.NodeAddress - -// add adds an address of the given type to the address map. If the given type -// was not already in the map, it creates a new preallocated entry for it. -func (m addressMap) add(address apiv1.NodeAddress) { - if _, ok := m[address.Type]; ok { - m[address.Type] = append(m[address.Type], address) - } else { - // There can be at most 2 addresses of the same type. - m[address.Type] = make([]apiv1.NodeAddress, 2) - m[address.Type] = append(m[address.Type], address) - } -} - -// GetAllNodeIPs returns all internal and external IP addresses grouped as IPv4 and IPv6 -func GetAllNodeIPs(node *apiv1.Node) (map[apiv1.NodeAddressType][]net.IP, map[apiv1.NodeAddressType][]net.IP) { - ipAddrv4 := make(map[apiv1.NodeAddressType][]net.IP) - ipAddrv6 := make(map[apiv1.NodeAddressType][]net.IP) +// getAllNodeIPs returns all internal and external IP addresses grouped as IPv4 and IPv6 in a map that is indexed by +// the Kubernetes Node Object address type (internal / external). +func getAllNodeIPs(node *apiv1.Node) (map[apiv1.NodeAddressType][]net.IP, map[apiv1.NodeAddressType][]net.IP) { + ipAddrv4 := make(addressMap) + ipAddrv6 := make(addressMap) addresses := node.Status.Addresses - addressesPerType := make(addressMap) + addressesPerType := make(nodeAddressMap) for _, address := range addresses { addressesPerType.add(address) } @@ -114,57 +384,28 @@ func GetAllNodeIPs(node *apiv1.Node) (map[apiv1.NodeAddressType][]net.IP, map[ap return ipAddrv4, ipAddrv6 } -func FindBestIPv6NodeAddress(priIP net.IP, intExtIPv6Addresses map[apiv1.NodeAddressType][]net.IP) net.IP { - if priIP != nil && priIP.To4() == nil && priIP.To16() != nil { - // the NRC's primary IP is already an IPv6 address, so we'll use that - return priIP - } - // the NRC's primary IP is not an IPv6, let's try to find the best available IPv6 address out of our - // available node addresses to use as the nextHop for our route - if intExtIPv6Addresses != nil { - if len(intExtIPv6Addresses[apiv1.NodeInternalIP]) > 0 { - return intExtIPv6Addresses[apiv1.NodeInternalIP][0] - } else if len(intExtIPv6Addresses[apiv1.NodeExternalIP]) > 0 { - return intExtIPv6Addresses[apiv1.NodeExternalIP][0] - } - } - return nil -} - -func FindBestIPv4NodeAddress(priIP net.IP, intExtIPv4Addresses map[apiv1.NodeAddressType][]net.IP) net.IP { - if priIP != nil && priIP.To4() != nil { - // the NRC's primary IP is already an IPv6 address, so we'll use that - return priIP - } - // the NRC's primary IP is not an IPv6, let's try to find the best available IPv6 address out of our - // available node addresses to use as the nextHop for our route - if intExtIPv4Addresses != nil { - if len(intExtIPv4Addresses[apiv1.NodeInternalIP]) > 0 { - return intExtIPv4Addresses[apiv1.NodeInternalIP][0] - } else if len(intExtIPv4Addresses[apiv1.NodeExternalIP]) > 0 { - return intExtIPv4Addresses[apiv1.NodeExternalIP][0] - } +// GetNodeSubnet returns the subnet and interface name for a given node IP +func GetNodeSubnet(nodeIP net.IP, linkQ LocalLinkQuerier) (net.IPNet, string, error) { + if linkQ == nil { + linkQ = &netlink.Handle{} } - return nil -} -// GetMTUFromNodeIP returns the MTU by detecting it from the IP on the node and figuring in tunneling configurations -func GetMTUFromNodeIP(nodeIP net.IP) (int, error) { - links, err := netlink.LinkList() + links, err := linkQ.LinkList() if err != nil { - return 0, errors.New("failed to get list of links") + return net.IPNet{}, "", errors.New("failed to get list of links") } + for _, link := range links { - addresses, err := netlink.AddrList(link, netlink.FAMILY_ALL) + addresses, err := linkQ.AddrList(link, netlink.FAMILY_ALL) if err != nil { - return 0, errors.New("failed to get list of addr") + return net.IPNet{}, "", errors.New("failed to get list of addr") } for _, addr := range addresses { if addr.IPNet.IP.Equal(nodeIP) { - linkMTU := link.Attrs().MTU - return linkMTU, nil + return *addr.IPNet, link.Attrs().Name, nil } } } - return 0, errors.New("failed to find interface with specified node IP") + + return net.IPNet{}, "", errors.New("failed to find interface with specified node ip") } diff --git a/pkg/utils/node_test.go b/pkg/utils/node_test.go index bc464ba01a..da3fe01a68 100644 --- a/pkg/utils/node_test.go +++ b/pkg/utils/node_test.go @@ -8,6 +8,7 @@ import ( "reflect" "testing" + "github.com/stretchr/testify/assert" apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" @@ -169,24 +170,26 @@ func Test_GetNodeIP(t *testing.T) { }, }, nil, - errors.New("host IP unknown"), + errors.New("error getting primary NodeIP: host IP unknown"), }, } for _, testcase := range testcases { t.Run(testcase.name, func(t *testing.T) { - ip, err := GetPrimaryNodeIP(testcase.node) - if !reflect.DeepEqual(err, testcase.err) { - t.Logf("actual error: %v", err) - t.Logf("expected error: %v", testcase.err) - t.Error("did not get expected error") + krNode, err := NewRemoteKRNode(testcase.node) + if err != nil { + assert.EqualError(t, err, testcase.err.Error()) + return } + ip := krNode.GetPrimaryNodeIP() - if !reflect.DeepEqual(ip, testcase.ip) { - t.Logf("actual ip: %v", ip) - t.Logf("expected ip: %v", testcase.ip) - t.Error("did not get expected node ip") + if testcase.err == nil { + assert.NoError(t, err) + } else { + assert.EqualError(t, err, testcase.err.Error()) } + + assert.Equal(t, testcase.ip, ip) }) } } From f6ff559f97d16d62805c711501641c5a1829b304 Mon Sep 17 00:00:00 2001 From: Aaron U'Ren Date: Sun, 22 Sep 2024 12:46:40 -0500 Subject: [PATCH 2/4] fact(network_routes_controller_test.go): use better error condition handling --- pkg/controllers/routing/network_routes_controller_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/controllers/routing/network_routes_controller_test.go b/pkg/controllers/routing/network_routes_controller_test.go index acd72aad22..316c0b6b29 100644 --- a/pkg/controllers/routing/network_routes_controller_test.go +++ b/pkg/controllers/routing/network_routes_controller_test.go @@ -2,6 +2,7 @@ package routing import ( "context" + "errors" "fmt" "net" "os" @@ -1893,7 +1894,7 @@ func Test_nodeHasEndpointsForService(t *testing.T) { waitForListerWithTimeout(testcase.nrc.epLister, time.Second*10, t) nodeHasEndpoints, err := testcase.nrc.nodeHasEndpointsForService(testcase.existingService) - if !reflect.DeepEqual(err, testcase.err) { + if !errors.Is(err, testcase.err) { t.Logf("actual err: %v", err) t.Logf("expected err: %v", testcase.err) t.Error("unexpected error") From f4766d6be77a1a8629fb90ac4c2c96d073d87157 Mon Sep 17 00:00:00 2001 From: Aaron U'Ren Date: Mon, 23 Sep 2024 09:46:34 -0500 Subject: [PATCH 3/4] fix(krnode): apply suggestions from code review Co-authored-by: Tom Wieczorek --- .../netpol/network_policy_controller.go | 2 +- .../proxy/network_services_controller_test.go | 8 +- pkg/controllers/proxy/utils_test.go | 16 +- pkg/controllers/routing/bgp_policies_test.go | 6 +- pkg/controllers/routing/ecmp_vip_test.go | 136 +++++---- .../routing/network_routes_controller.go | 2 +- .../routing/network_routes_controller_test.go | 282 +++++++++++------- pkg/utils/node.go | 89 ++---- 8 files changed, 308 insertions(+), 233 deletions(-) diff --git a/pkg/controllers/netpol/network_policy_controller.go b/pkg/controllers/netpol/network_policy_controller.go index 19b07ea7a3..0d48a41c84 100644 --- a/pkg/controllers/netpol/network_policy_controller.go +++ b/pkg/controllers/netpol/network_policy_controller.go @@ -63,7 +63,7 @@ var ( // NetworkPolicyController struct to hold information required by NetworkPolicyController type NetworkPolicyController struct { - krNode utils.NodeAware + krNode utils.NodeIPAndFamilyAware serviceClusterIPRanges []net.IPNet serviceExternalIPRanges []net.IPNet serviceLoadBalancerIPRanges []net.IPNet diff --git a/pkg/controllers/proxy/network_services_controller_test.go b/pkg/controllers/proxy/network_services_controller_test.go index fc9b528485..83d9f2d0b8 100644 --- a/pkg/controllers/proxy/network_services_controller_test.go +++ b/pkg/controllers/proxy/network_services_controller_test.go @@ -141,9 +141,11 @@ var _ = Describe("NetworkServicesController", func() { fatalf("failed to create existing services: %v", err) } - krNode := &utils.KRNode{ - NodeName: "node-1", - PrimaryIP: net.ParseIP("10.0.0.0"), + krNode := &utils.LocalKRNode{ + KRNode: utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP("10.0.0.0"), + }, } nsc = &NetworkServicesController{ krNode: krNode, diff --git a/pkg/controllers/proxy/utils_test.go b/pkg/controllers/proxy/utils_test.go index 7d5f3b58c1..b5f9b156bb 100644 --- a/pkg/controllers/proxy/utils_test.go +++ b/pkg/controllers/proxy/utils_test.go @@ -24,9 +24,11 @@ func getMoqNSC() *NetworkServicesController { setupRoutesForExternalIPForDSRFunc: lnm.setupRoutesForExternalIPForDSR, } - krNode := &utils.KRNode{ - NodeName: "node-1", - PrimaryIP: net.ParseIP("10.0.0.0"), + krNode := &utils.LocalKRNode{ + KRNode: utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP("10.0.0.0"), + }, } return &NetworkServicesController{ krNode: krNode, @@ -191,9 +193,11 @@ func TestIsValidKubeRouterServiceArtifact(t *testing.T) { loadBalancerIPs: []string{"172.16.0.3"}, } - krNode := &utils.KRNode{ - NodeName: "node-1", - PrimaryIP: net.ParseIP("192.168.1.10"), + krNode := &utils.LocalKRNode{ + KRNode: utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP("192.168.1.10"), + }, } nsc := &NetworkServicesController{ diff --git a/pkg/controllers/routing/bgp_policies_test.go b/pkg/controllers/routing/bgp_policies_test.go index 7e1fc0c156..c727cafb19 100644 --- a/pkg/controllers/routing/bgp_policies_test.go +++ b/pkg/controllers/routing/bgp_policies_test.go @@ -36,8 +36,10 @@ type PolicyTestCase struct { } func Test_AddPolicies(t *testing.T) { - ipv4CapableKRNode := &utils.KRNode{ - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.IPv4(10, 10, 10, 10)}}, + ipv4CapableKRNode := &utils.LocalKRNode{ + KRNode: utils.KRNode{ + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.IPv4(10, 10, 10, 10)}}, + }, } testcases := []PolicyTestCase{ { diff --git a/pkg/controllers/routing/ecmp_vip_test.go b/pkg/controllers/routing/ecmp_vip_test.go index e935edd1c0..9ea00e1dc7 100644 --- a/pkg/controllers/routing/ecmp_vip_test.go +++ b/pkg/controllers/routing/ecmp_vip_test.go @@ -56,9 +56,11 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: true, advertiseExternalIP: true, advertiseLoadBalancerIP: true, - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -98,9 +100,11 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: false, advertiseExternalIP: false, advertiseLoadBalancerIP: false, - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -140,9 +144,11 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: true, advertiseExternalIP: false, advertiseLoadBalancerIP: false, - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -182,9 +188,11 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: false, advertiseExternalIP: true, advertiseLoadBalancerIP: false, - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -224,9 +232,11 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: false, advertiseExternalIP: false, advertiseLoadBalancerIP: true, - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -266,9 +276,11 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: false, advertiseExternalIP: false, advertiseLoadBalancerIP: false, - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -337,9 +349,11 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: true, advertiseExternalIP: true, advertiseLoadBalancerIP: true, - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -395,9 +409,11 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: true, advertiseExternalIP: true, advertiseLoadBalancerIP: true, - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -445,9 +461,11 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: true, advertiseExternalIP: true, advertiseLoadBalancerIP: true, - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv6), - NodeIPv6Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv6)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv6), + NodeIPv6Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv6)}}, + }, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -495,9 +513,11 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: true, advertiseExternalIP: true, advertiseLoadBalancerIP: true, - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -541,9 +561,11 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: true, advertiseExternalIP: true, advertiseLoadBalancerIP: true, - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv6), - NodeIPv6Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv6)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv6), + NodeIPv6Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv6)}}, + }, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -587,9 +609,11 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: true, advertiseExternalIP: true, advertiseLoadBalancerIP: true, - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv6Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv6)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv6Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv6)}}, + }, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -637,9 +661,11 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: true, advertiseExternalIP: true, advertiseLoadBalancerIP: true, - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv6), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv6), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -687,9 +713,11 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: true, advertiseExternalIP: true, advertiseLoadBalancerIP: true, - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -733,9 +761,11 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: true, advertiseExternalIP: true, advertiseLoadBalancerIP: true, - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv6), - NodeIPv6Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv6)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv6), + NodeIPv6Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv6)}}, + }, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -779,9 +809,11 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: true, advertiseExternalIP: true, advertiseLoadBalancerIP: true, - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ @@ -825,9 +857,11 @@ func Test_getVIPsForService(t *testing.T) { advertiseClusterIP: true, advertiseExternalIP: true, advertiseLoadBalancerIP: true, - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv6), - NodeIPv6Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv6)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv6), + NodeIPv6Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv6)}}, + }, }, }, serviceAdvertisedIPs: []*ServiceAdvertisedIPs{ diff --git a/pkg/controllers/routing/network_routes_controller.go b/pkg/controllers/routing/network_routes_controller.go index 9f7fcdc724..77f1eec5da 100644 --- a/pkg/controllers/routing/network_routes_controller.go +++ b/pkg/controllers/routing/network_routes_controller.go @@ -984,7 +984,7 @@ func (nrc *NetworkRoutingController) syncNodeIPSets(nodeIPAware utils.NodeIPAwar } var ipv4Addrs, ipv6Addrs [][]string - for _, nodeIPv4 := range nodeIPAware.GetNodeIPv6Addrs() { + for _, nodeIPv4 := range nodeIPAware.GetNodeIPv4Addrs() { ipv4Addrs = append(ipv4Addrs, []string{nodeIPv4.String(), utils.OptionTimeout, "0"}) } for _, nodeIPv6 := range nodeIPAware.GetNodeIPv6Addrs() { diff --git a/pkg/controllers/routing/network_routes_controller_test.go b/pkg/controllers/routing/network_routes_controller_test.go index 316c0b6b29..46aac623df 100644 --- a/pkg/controllers/routing/network_routes_controller_test.go +++ b/pkg/controllers/routing/network_routes_controller_test.go @@ -38,9 +38,11 @@ func Test_advertiseClusterIPs(t *testing.T) { "add bgp path for service with ClusterIP", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, []*v1core.Service{ @@ -82,9 +84,11 @@ func Test_advertiseClusterIPs(t *testing.T) { "add bgp path for service with ClusterIP/NodePort/LoadBalancer", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, []*v1core.Service{ @@ -182,9 +186,11 @@ func Test_advertiseClusterIPs(t *testing.T) { "add bgp path for invalid service type", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, []*v1core.Service{ @@ -252,9 +258,11 @@ func Test_advertiseClusterIPs(t *testing.T) { "add bgp path for headless service", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, []*v1core.Service{ @@ -467,9 +475,11 @@ func Test_advertiseExternalIPs(t *testing.T) { "add bgp path for service with external IPs", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP("10.0.0.1"), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP("10.0.0.1"), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, []*v1core.Service{ @@ -513,9 +523,11 @@ func Test_advertiseExternalIPs(t *testing.T) { "add bgp path for services with external IPs of type ClusterIP/NodePort/LoadBalancer", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, []*v1core.Service{ @@ -617,9 +629,11 @@ func Test_advertiseExternalIPs(t *testing.T) { "add bgp path for invalid service type", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, []*v1core.Service{ @@ -690,9 +704,11 @@ func Test_advertiseExternalIPs(t *testing.T) { "add bgp path for headless service", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, []*v1core.Service{ @@ -791,9 +807,11 @@ func Test_advertiseExternalIPs(t *testing.T) { "skip bgp path to loadbalancerIP for service without LoadBalancer IP", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, []*v1core.Service{ @@ -842,9 +860,11 @@ func Test_advertiseExternalIPs(t *testing.T) { "add bgp path to loadbalancerIP for service with LoadBalancer IP", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, []*v1core.Service{ @@ -899,9 +919,11 @@ func Test_advertiseExternalIPs(t *testing.T) { "no bgp path to nil loadbalancerIPs for service with LoadBalancer", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, []*v1core.Service{ @@ -946,9 +968,11 @@ func Test_advertiseExternalIPs(t *testing.T) { "no bgp path to loadbalancerIPs for service with LoadBalancer and skiplbips annotation", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, []*v1core.Service{ @@ -1120,9 +1144,11 @@ func Test_advertiseAnnotationOptOut(t *testing.T) { "add bgp paths for all service IPs", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, []*v1core.Service{ @@ -1241,9 +1267,11 @@ func Test_advertiseAnnotationOptOut(t *testing.T) { "opt out to advertise any IPs via annotations", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, []*v1core.Service{ @@ -1419,9 +1447,11 @@ func Test_advertiseAnnotationOptIn(t *testing.T) { "no bgp paths for any service IPs", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, []*v1core.Service{ @@ -1532,9 +1562,11 @@ func Test_advertiseAnnotationOptIn(t *testing.T) { "opt in to advertise all IPs via annotations", &NetworkRoutingController{ bgpServer: gobgp.NewBgpServer(), - krNode: &utils.KRNode{ - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, []*v1core.Service{ @@ -1786,10 +1818,12 @@ func Test_nodeHasEndpointsForService(t *testing.T) { { "node has endpoints for service", &NetworkRoutingController{ - krNode: &utils.KRNode{ - NodeName: "node-1", - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, &v1core.Service{ @@ -1831,10 +1865,12 @@ func Test_nodeHasEndpointsForService(t *testing.T) { { "node has no endpoints for service", &NetworkRoutingController{ - krNode: &utils.KRNode{ - NodeName: "node-1", - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, &v1core.Service{ @@ -1925,10 +1961,12 @@ func Test_advertisePodRoute(t *testing.T) { bgpServer: gobgp.NewBgpServer(), podCidr: "172.20.0.0/24", podIPv4CIDRs: []string{"172.20.0.0/24"}, - krNode: &utils.KRNode{ - NodeName: "node-1", - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, "node-1", @@ -1955,10 +1993,12 @@ func Test_advertisePodRoute(t *testing.T) { hostnameOverride: "node-1", podCidr: "172.20.0.0/24", podIPv4CIDRs: []string{"172.20.0.0/24"}, - krNode: &utils.KRNode{ - NodeName: "node-1", - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, }, "", @@ -1985,12 +2025,14 @@ func Test_advertisePodRoute(t *testing.T) { hostnameOverride: "node-1", podCidr: "2001:db8:42:2::/64", podIPv6CIDRs: []string{"2001:db8:42:2::/64"}, - krNode: &utils.KRNode{ - NodeName: "node-1", - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, - NodeIPv6Addrs: map[v1core.NodeAddressType][]net.IP{ - v1core.NodeInternalIP: {net.IPv6loopback}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + NodeIPv6Addrs: map[v1core.NodeAddressType][]net.IP{ + v1core.NodeInternalIP: {net.IPv6loopback}, + }, }, }, }, @@ -2163,10 +2205,12 @@ func Test_syncInternalPeers(t *testing.T) { &NetworkRoutingController{ bgpFullMeshMode: true, clientset: fake.NewSimpleClientset(), - krNode: &utils.KRNode{ - NodeName: "node-1", - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, bgpServer: gobgp.NewBgpServer(), activeNodes: make(map[string]bool), @@ -2195,10 +2239,12 @@ func Test_syncInternalPeers(t *testing.T) { &NetworkRoutingController{ bgpFullMeshMode: true, clientset: fake.NewSimpleClientset(), - krNode: &utils.KRNode{ - NodeName: "node-1", - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, bgpServer: gobgp.NewBgpServer(), activeNodes: make(map[string]bool), @@ -2241,10 +2287,12 @@ func Test_syncInternalPeers(t *testing.T) { &NetworkRoutingController{ bgpFullMeshMode: true, clientset: fake.NewSimpleClientset(), - krNode: &utils.KRNode{ - NodeName: "node-1", - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, bgpServer: gobgp.NewBgpServer(), activeNodes: map[string]bool{ @@ -2275,10 +2323,12 @@ func Test_syncInternalPeers(t *testing.T) { &NetworkRoutingController{ bgpFullMeshMode: false, clientset: fake.NewSimpleClientset(), - krNode: &utils.KRNode{ - NodeName: "node-1", - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, bgpServer: gobgp.NewBgpServer(), activeNodes: make(map[string]bool), @@ -2388,10 +2438,12 @@ func Test_routeReflectorConfiguration(t *testing.T) { bgpFullMeshMode: false, bgpPort: 10000, clientset: fake.NewSimpleClientset(), - krNode: &utils.KRNode{ - NodeName: "node-1", - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, routerID: testNodeIPv4, bgpServer: gobgp.NewBgpServer(), @@ -2419,10 +2471,12 @@ func Test_routeReflectorConfiguration(t *testing.T) { bgpFullMeshMode: false, bgpPort: 10000, clientset: fake.NewSimpleClientset(), - krNode: &utils.KRNode{ - NodeName: "node-1", - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, routerID: testNodeIPv4, bgpServer: gobgp.NewBgpServer(), @@ -2450,10 +2504,12 @@ func Test_routeReflectorConfiguration(t *testing.T) { bgpFullMeshMode: false, bgpPort: 10000, clientset: fake.NewSimpleClientset(), - krNode: &utils.KRNode{ - NodeName: "node-1", - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, routerID: testNodeIPv4, bgpServer: gobgp.NewBgpServer(), @@ -2481,10 +2537,12 @@ func Test_routeReflectorConfiguration(t *testing.T) { bgpFullMeshMode: false, bgpPort: 10000, clientset: fake.NewSimpleClientset(), - krNode: &utils.KRNode{ - NodeName: "node-1", - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, routerID: testNodeIPv4, bgpServer: gobgp.NewBgpServer(), @@ -2512,10 +2570,12 @@ func Test_routeReflectorConfiguration(t *testing.T) { bgpFullMeshMode: false, bgpPort: 10000, clientset: fake.NewSimpleClientset(), - krNode: &utils.KRNode{ - NodeName: "node-1", - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, bgpServer: gobgp.NewBgpServer(), activeNodes: make(map[string]bool), @@ -2542,10 +2602,12 @@ func Test_routeReflectorConfiguration(t *testing.T) { bgpFullMeshMode: false, bgpPort: 10000, clientset: fake.NewSimpleClientset(), - krNode: &utils.KRNode{ - NodeName: "node-1", - PrimaryIP: net.ParseIP(testNodeIPv4), - NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + krNode: &utils.LocalKRNode{ + KRNode: utils.KRNode{ + NodeName: "node-1", + PrimaryIP: net.ParseIP(testNodeIPv4), + NodeIPv4Addrs: map[v1core.NodeAddressType][]net.IP{v1core.NodeInternalIP: {net.ParseIP(testNodeIPv4)}}, + }, }, bgpServer: gobgp.NewBgpServer(), activeNodes: make(map[string]bool), diff --git a/pkg/utils/node.go b/pkg/utils/node.go index ce33cb1599..3f30cc354a 100644 --- a/pkg/utils/node.go +++ b/pkg/utils/node.go @@ -15,42 +15,32 @@ import ( netutils "k8s.io/utils/net" ) -// nodeAddressMap is a mapping of address types to a list of addresses of that type. -// It preallocates the slices of addresses. +// nodeAddressMap contains Kubernetes node address types (apiv1.NodeAddressType) grouped by Kubernetes Node Object +// address type (internal / external). type nodeAddressMap map[apiv1.NodeAddressType][]apiv1.NodeAddress -// add adds an address of the given type to the address map. If the given type -// was not already in the map, it creates a new preallocated entry for it. -func (m nodeAddressMap) add(address apiv1.NodeAddress) { - if _, ok := m[address.Type]; ok { - m[address.Type] = append(m[address.Type], address) - } else { - m[address.Type] = make([]apiv1.NodeAddress, 0) - m[address.Type] = append(m[address.Type], address) - } -} - -// nodeAddressMap is a mapping of address types to a list of addresses of that type. -// It preallocates the slices of addresses. +// addressMap contains net.IP addresses grouped by Kubernetes Node Object address type (internal / external). type addressMap map[apiv1.NodeAddressType][]net.IP // KRNode is a struct that holds information about a node that is used by kube-router. type KRNode struct { + NodeIPv4Addrs addressMap + NodeIPv6Addrs addressMap + NodeName string + PrimaryIP net.IP +} + +// LocalKRNode is a struct that holds information about this kube-router node. +type LocalKRNode struct { + KRNode NodeInterfaceName string - NodeIPv4Addrs addressMap - NodeIPv6Addrs addressMap - NodeName string linkQ LocalLinkQuerier - PrimaryNodeSubnet net.IPNet - PrimaryIP net.IP } // NodeIPAware is an interface that provides methods to get the node's IP addresses in various data structures. type NodeIPAware interface { FindBestIPv4NodeAddress() net.IP FindBestIPv6NodeAddress() net.IP - GetNodeIPv4AddrsTypeMap() addressMap - GetNodeIPv6AddrsTypeMap() addressMap GetNodeIPv4Addrs() []net.IP GetNodeIPv6Addrs() []net.IP GetNodeIPAddrs() []net.IP @@ -63,7 +53,6 @@ type NodeIPAware interface { // functions on a remote node, they will return nil or an error. type NodeInterfaceAware interface { GetNodeInterfaceName() string - GetPrimaryNodeSubnet() net.IPNet GetNodeMTU() (int, error) } @@ -93,12 +82,6 @@ type NodeAware interface { NodeNameAware } -// GetNodeIPv4AddrsTypeMap returns the node's IPv4 addresses grouped by Kubernetes Node Object address type (internal / -// external). -func (n *KRNode) GetNodeIPv4AddrsTypeMap() addressMap { - return n.NodeIPv4Addrs -} - // GetNodeIPv4Addrs returns the node's IPv4 addresses as defined by the Kubernetes Node Object. func (n *KRNode) GetNodeIPv4Addrs() []net.IP { var nodeIPs []net.IP @@ -108,12 +91,6 @@ func (n *KRNode) GetNodeIPv4Addrs() []net.IP { return nodeIPs } -// GetNodeIPv6AddrsTypeMap returns the node's IPv6 addresses grouped by Kubernetes Node Object address type (internal / -// external). -func (n *KRNode) GetNodeIPv6AddrsTypeMap() addressMap { - return n.NodeIPv6Addrs -} - // GetNodeIPv6Addrs returns the node's IPv6 addresses as defined by the Kubernetes Node Object. func (n *KRNode) GetNodeIPv6Addrs() []net.IP { var nodeIPs []net.IP @@ -130,17 +107,10 @@ func (n *KRNode) GetPrimaryNodeIP() net.IP { return n.PrimaryIP } -// GetPrimaryNodeSubnet returns the node's primary subnet as defined by the primary IP address. This function is only -// available if you are running on the node itself, as kube-router determines this by looking at the node's interfaces -// and parsing the address data there. If you attempt to call this function on a remote node, it will return nil. -func (n *KRNode) GetPrimaryNodeSubnet() net.IPNet { - return n.PrimaryNodeSubnet -} - // GetNodeInterfaceName returns the node's interface name as defined by the primary IP address. This function is only // available if you are running on the node itself, as kube-router determines this by looking at the node's interfaces // and parsing the address data there. If you attempt to call this function on a remote node, it will return nil. -func (n *KRNode) GetNodeInterfaceName() string { +func (n *LocalKRNode) GetNodeInterfaceName() string { return n.NodeInterfaceName } @@ -162,7 +132,7 @@ func (n *KRNode) GetNodeName() string { // FindBestIPv6NodeAddress returns the best available IPv6 address for the node. If the primary IP is already an IPv6 // address, it will return that. Otherwise, it will return the first internal or external IPv6 address defined in the // Kubernetes Node Object. -func (n KRNode) FindBestIPv6NodeAddress() net.IP { +func (n *KRNode) FindBestIPv6NodeAddress() net.IP { if n.PrimaryIP != nil && n.PrimaryIP.To4() == nil && n.PrimaryIP.To16() != nil { // the NRC's primary IP is already an IPv6 address, so we'll use that return n.PrimaryIP @@ -182,7 +152,7 @@ func (n KRNode) FindBestIPv6NodeAddress() net.IP { // FindBestIPv4NodeAddress returns the best available IPv4 address for the node. If the primary IP is already an IPv4 // address, it will return that. Otherwise, it will return the first internal or external IPv4 address defined in the // Kubernetes Node Object. -func (n KRNode) FindBestIPv4NodeAddress() net.IP { +func (n *KRNode) FindBestIPv4NodeAddress() net.IP { if n.PrimaryIP != nil && n.PrimaryIP.To4() != nil { // the NRC's primary IP is already an IPv6 address, so we'll use that return n.PrimaryIP @@ -202,7 +172,7 @@ func (n KRNode) FindBestIPv4NodeAddress() net.IP { // GetNodeMTU returns the MTU of the interface that the node's primary IP address is assigned to. This function is only // available if you are running on the node itself, as kube-router determines this by looking at the node's interfaces // and parsing the address data there. If you attempt to call this function on a remote node, it will return an error. -func (n KRNode) GetNodeMTU() (int, error) { +func (n *LocalKRNode) GetNodeMTU() (int, error) { links, err := n.linkQ.LinkList() if err != nil { return 0, errors.New("failed to get list of links") @@ -224,7 +194,7 @@ func (n KRNode) GetNodeMTU() (int, error) { // GetNodeIPAddrs returns all of the node's IP addresses (whether internal or external) as defined by the Kubernetes // Node Object. -func (n KRNode) GetNodeIPAddrs() []net.IP { +func (n *KRNode) GetNodeIPAddrs() []net.IP { var nodeIPs []net.IP for _, ip := range n.NodeIPv4Addrs { nodeIPs = append(nodeIPs, ip...) @@ -240,7 +210,7 @@ func (n KRNode) GetNodeIPAddrs() []net.IP { // a remote node, it will result in an error as it will not be able to find the correct subnet / interface information. // For this use-case use NewRemoteKRNode instead. It will also return an error if the node does not have any IPv4 or // IPv6 addresses defined in the Kubernetes Node Object. -func NewKRNode(node *apiv1.Node, linkQ LocalLinkQuerier, enableIPv4, enableIPv6 bool) (NodeAware, error) { +func NewKRNode(node *apiv1.Node, linkQ LocalLinkQuerier, enableIPv4, enableIPv6 bool) (*LocalKRNode, error) { if linkQ == nil { linkQ = &netlink.Handle{} } @@ -261,18 +231,19 @@ func NewKRNode(node *apiv1.Node, linkQ LocalLinkQuerier, enableIPv4, enableIPv6 return nil, fmt.Errorf("IPv6 was enabled, but no IPv6 address was found on the node") } - primaryNodeSubnet, nodeInterfaceName, err := GetNodeSubnet(primaryNodeIP, linkQ) + _, nodeInterfaceName, err := GetNodeSubnet(primaryNodeIP, linkQ) if err != nil { return nil, fmt.Errorf("error getting node subnet: %w", err) } - krNode := &KRNode{ + krNode := &LocalKRNode{ + KRNode: KRNode{ + NodeName: node.Name, + PrimaryIP: primaryNodeIP, + NodeIPv4Addrs: ipv4Addrs, + NodeIPv6Addrs: ipv6Addrs, + }, linkQ: linkQ, - NodeName: node.Name, - PrimaryIP: primaryNodeIP, - NodeIPv4Addrs: ipv4Addrs, - NodeIPv6Addrs: ipv6Addrs, - PrimaryNodeSubnet: primaryNodeSubnet, NodeInterfaceName: nodeInterfaceName, } @@ -282,7 +253,7 @@ func NewKRNode(node *apiv1.Node, linkQ LocalLinkQuerier, enableIPv4, enableIPv6 // NewRemoteKRNode creates a new KRNode object from a Kubernetes Node Object. This function is used when kube-router is // attempting to parse a remote node and does not have access to the node's interfaces and address data. It will return // an error if the node does not have any IPv4 or IPv6 addresses defined in the Kubernetes Node Object. -func NewRemoteKRNode(node *apiv1.Node) (NodeIPAndFamilyAware, error) { +func NewRemoteKRNode(node *apiv1.Node) (*KRNode, error) { primaryNodeIP, err := getPrimaryNodeIP(node) if err != nil { return nil, fmt.Errorf("error getting primary NodeIP: %w", err) @@ -337,7 +308,7 @@ func GetNodeObject(clientset kubernetes.Interface, hostnameOverride string) (*ap // 2. NodeExternalIP (usually only set on cloud providers usually) func getPrimaryNodeIP(node *apiv1.Node) (net.IP, error) { addresses := node.Status.Addresses - addressMap := make(map[apiv1.NodeAddressType][]apiv1.NodeAddress) + addressMap := make(nodeAddressMap) for i := range addresses { addressMap[addresses[i].Type] = append(addressMap[addresses[i].Type], addresses[i]) } @@ -352,13 +323,13 @@ func getPrimaryNodeIP(node *apiv1.Node) (net.IP, error) { // getAllNodeIPs returns all internal and external IP addresses grouped as IPv4 and IPv6 in a map that is indexed by // the Kubernetes Node Object address type (internal / external). -func getAllNodeIPs(node *apiv1.Node) (map[apiv1.NodeAddressType][]net.IP, map[apiv1.NodeAddressType][]net.IP) { +func getAllNodeIPs(node *apiv1.Node) (addressMap, addressMap) { ipAddrv4 := make(addressMap) ipAddrv6 := make(addressMap) addresses := node.Status.Addresses addressesPerType := make(nodeAddressMap) for _, address := range addresses { - addressesPerType.add(address) + addressesPerType[address.Type] = append(addressesPerType[address.Type], address) } if internalAddresses, ok := addressesPerType[apiv1.NodeInternalIP]; ok { for _, address := range internalAddresses { From 9e9fb733967cec515efaefa35372296ee1ad8145 Mon Sep 17 00:00:00 2001 From: Aaron U'Ren Date: Sun, 29 Sep 2024 17:24:03 -0500 Subject: [PATCH 4/4] test(krnode): add unit tests for new functionality --- go.mod | 1 + go.sum | 1 + .../netpol/network_policy_controller_test.go | 48 +- pkg/utils/linux_routingtest.go | 78 ++ pkg/utils/node.go | 2 +- pkg/utils/node_test.go | 687 ++++++++++++++++++ 6 files changed, 769 insertions(+), 48 deletions(-) create mode 100644 pkg/utils/linux_routingtest.go diff --git a/go.mod b/go.mod index cd84f91018..96a2a3d091 100644 --- a/go.mod +++ b/go.mod @@ -86,6 +86,7 @@ require ( github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spf13/viper v1.19.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.6.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect go.opentelemetry.io/otel v1.28.0 // indirect diff --git a/go.sum b/go.sum index d5c5eb3c8f..34280963ab 100644 --- a/go.sum +++ b/go.sum @@ -195,6 +195,7 @@ github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= diff --git a/pkg/controllers/netpol/network_policy_controller_test.go b/pkg/controllers/netpol/network_policy_controller_test.go index cd187ee230..614f6d2232 100644 --- a/pkg/controllers/netpol/network_policy_controller_test.go +++ b/pkg/controllers/netpol/network_policy_controller_test.go @@ -12,7 +12,6 @@ import ( "time" "github.com/coreos/go-iptables/iptables" - "github.com/vishvananda/netlink" netv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -758,51 +757,6 @@ func (ips *fakeIPSet) Name(name string) string { return name } -type fakeLocalLinkQuerier struct { - links []netlink.Link - addrs []*net.IPNet -} - -func newFakeLocalLinkQuerier(addrStrings []string) *fakeLocalLinkQuerier { - links := make([]netlink.Link, len(addrStrings)) - for idx := range addrStrings { - linkAttrs := netlink.LinkAttrs{ - Index: idx, - } - linkDevice := netlink.Device{LinkAttrs: linkAttrs} - links[idx] = &linkDevice - } - addrs := make([]*net.IPNet, len(addrStrings)) - for idx, addr := range addrStrings { - ip := net.ParseIP(addr) - var netMask net.IPMask - if ip.To4() != nil { - netMask = net.CIDRMask(24, 32) - } else { - netMask = net.CIDRMask(64, 128) - } - ipNet := &net.IPNet{ - IP: ip, - Mask: netMask, - } - addrs[idx] = ipNet - } - return &fakeLocalLinkQuerier{ - links: links, - addrs: addrs, - } -} - -func (f *fakeLocalLinkQuerier) LinkList() ([]netlink.Link, error) { - return f.links, nil -} - -func (f *fakeLocalLinkQuerier) AddrList(link netlink.Link, family int) ([]netlink.Addr, error) { - addrs := make([]netlink.Addr, 1) - addrs[0] = netlink.Addr{IPNet: f.addrs[link.Attrs().Index]} - return addrs, nil -} - func TestNetworkPolicyController(t *testing.T) { curHostname, _ := os.Hostname() testCases := []tNetPolConfigTestCase{ @@ -989,7 +943,7 @@ func TestNetworkPolicyController(t *testing.T) { }, } fakeNodeIPs := []string{"10.10.10.10", "2001:0db8:0042:0001:0000:0000:0000:0000"} - fakeLinkQuerier := newFakeLocalLinkQuerier(fakeNodeIPs) + fakeLinkQuerier := utils.NewFakeLocalLinkQuerier(fakeNodeIPs, nil) client := fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{*newFakeNode("node", fakeNodeIPs)}}) _, podInformer, nsInformer, netpolInformer := newFakeInformersFromClient(client) for _, test := range testCases { diff --git a/pkg/utils/linux_routingtest.go b/pkg/utils/linux_routingtest.go new file mode 100644 index 0000000000..1c83ce6572 --- /dev/null +++ b/pkg/utils/linux_routingtest.go @@ -0,0 +1,78 @@ +package utils + +import ( + "fmt" + "net" + + "github.com/stretchr/testify/mock" + "github.com/vishvananda/netlink" +) + +type FakeLocalLinkQuerier struct { + links []netlink.Link + addrs []*net.IPNet +} + +func NewFakeLocalLinkQuerier(addrStrings []string, mtus []int) *FakeLocalLinkQuerier { + links := make([]netlink.Link, len(addrStrings)) + for idx := range addrStrings { + mtu := 1 + if idx < len(mtus) { + mtu = mtus[idx] + } + linkAttrs := netlink.LinkAttrs{ + Index: idx, + MTU: mtu, + } + linkDevice := netlink.Device{LinkAttrs: linkAttrs} + links[idx] = &linkDevice + } + addrs := make([]*net.IPNet, len(addrStrings)) + for idx, addr := range addrStrings { + ip := net.ParseIP(addr) + var netMask net.IPMask + if ip.To4() != nil { + //nolint:gomnd // Hardcoded value is used for testing purposes + netMask = net.CIDRMask(24, 32) + } else { + //nolint:gomnd // Hardcoded value is used for testing purposes + netMask = net.CIDRMask(64, 128) + } + ipNet := &net.IPNet{ + IP: ip, + Mask: netMask, + } + addrs[idx] = ipNet + } + return &FakeLocalLinkQuerier{ + links: links, + addrs: addrs, + } +} + +func (f *FakeLocalLinkQuerier) LinkList() ([]netlink.Link, error) { + return f.links, nil +} + +func (f *FakeLocalLinkQuerier) AddrList(link netlink.Link, family int) ([]netlink.Addr, error) { + addrs := make([]netlink.Addr, 1) + addrs[0] = netlink.Addr{IPNet: f.addrs[link.Attrs().Index]} + if link.Attrs().MTU == 0 { + return nil, fmt.Errorf("MTU was set to 0 to simulate an error") + } + return addrs, nil +} + +type MockLocalLinkQuerier struct { + mock.Mock +} + +func (m *MockLocalLinkQuerier) LinkList() ([]netlink.Link, error) { + args := m.Called() + return args.Get(0).([]netlink.Link), args.Error(1) +} + +func (m *MockLocalLinkQuerier) AddrList(link netlink.Link, family int) ([]netlink.Addr, error) { + args := m.Called(link, family) + return args.Get(0).([]netlink.Addr), args.Error(1) +} diff --git a/pkg/utils/node.go b/pkg/utils/node.go index 3f30cc354a..b580de1376 100644 --- a/pkg/utils/node.go +++ b/pkg/utils/node.go @@ -369,7 +369,7 @@ func GetNodeSubnet(nodeIP net.IP, linkQ LocalLinkQuerier) (net.IPNet, string, er for _, link := range links { addresses, err := linkQ.AddrList(link, netlink.FAMILY_ALL) if err != nil { - return net.IPNet{}, "", errors.New("failed to get list of addr") + return net.IPNet{}, "", errors.New("failed to get list of addrs") } for _, addr := range addresses { if addr.IPNet.IP.Equal(nodeIP) { diff --git a/pkg/utils/node_test.go b/pkg/utils/node_test.go index da3fe01a68..1fa9c8a783 100644 --- a/pkg/utils/node_test.go +++ b/pkg/utils/node_test.go @@ -9,6 +9,8 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/vishvananda/netlink" apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" @@ -193,3 +195,688 @@ func Test_GetNodeIP(t *testing.T) { }) } } + +func Test_GetNodeIPv4Addrs(t *testing.T) { + testcases := []struct { + name string + node *apiv1.Node + expected []net.IP + err error + }{ + { + "node with internal and external IPv4 addresses", + &apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + }, + Status: apiv1.NodeStatus{ + Addresses: []apiv1.NodeAddress{ + { + Type: apiv1.NodeInternalIP, + Address: "10.0.0.1", + }, + { + Type: apiv1.NodeExternalIP, + Address: "192.168.1.1", + }, + { + Type: apiv1.NodeExternalIP, + Address: "2001:db8::1", + }, + }, + }, + }, + []net.IP{net.ParseIP("10.0.0.1"), net.ParseIP("192.168.1.1")}, + nil, + }, + { + "node with only internal IPv4 address", + &apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + }, + Status: apiv1.NodeStatus{ + Addresses: []apiv1.NodeAddress{ + { + Type: apiv1.NodeInternalIP, + Address: "10.0.0.1", + }, + { + Type: apiv1.NodeInternalIP, + Address: "2001:db8::1", + }, + { + Type: apiv1.NodeExternalIP, + Address: "2001:db8::2", + }, + }, + }, + }, + []net.IP{net.ParseIP("10.0.0.1")}, + nil, + }, + { + "node with only external IPv4 address", + &apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + }, + Status: apiv1.NodeStatus{ + Addresses: []apiv1.NodeAddress{ + { + Type: apiv1.NodeExternalIP, + Address: "192.168.1.1", + }, + { + Type: apiv1.NodeInternalIP, + Address: "2001:db8::1", + }, + }, + }, + }, + []net.IP{net.ParseIP("192.168.1.1")}, + nil, + }, + { + "node with no IPv4 addresses", + &apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + }, + Status: apiv1.NodeStatus{ + Addresses: []apiv1.NodeAddress{}, + }, + }, + []net.IP{}, + errors.New("error getting primary NodeIP: host IP unknown"), + }, + } + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + krNode, err := NewRemoteKRNode(testcase.node) + if err != nil { + if testcase.err == nil { + t.Fatalf("failed to create KRNode: %v", err) + } + assert.EqualError(t, err, testcase.err.Error()) + return + } + ipv4Addrs := krNode.GetNodeIPv4Addrs() + assert.Equal(t, testcase.expected, ipv4Addrs) + }) + } +} + +func Test_GetNodeIPv6Addrs(t *testing.T) { + testcases := []struct { + name string + node *apiv1.Node + expected []net.IP + err error + }{ + { + "node with internal and external IPv4 addresses", + &apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + }, + Status: apiv1.NodeStatus{ + Addresses: []apiv1.NodeAddress{ + { + Type: apiv1.NodeInternalIP, + Address: "10.0.0.1", + }, + { + Type: apiv1.NodeExternalIP, + Address: "192.168.1.1", + }, + { + Type: apiv1.NodeExternalIP, + Address: "2001:db8::1", + }, + }, + }, + }, + []net.IP{net.ParseIP("2001:db8::1")}, + nil, + }, + { + "node with only internal IPv4 address", + &apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + }, + Status: apiv1.NodeStatus{ + Addresses: []apiv1.NodeAddress{ + { + Type: apiv1.NodeInternalIP, + Address: "10.0.0.1", + }, + { + Type: apiv1.NodeInternalIP, + Address: "2001:db8::1", + }, + { + Type: apiv1.NodeExternalIP, + Address: "2001:db8::2", + }, + }, + }, + }, + []net.IP{net.ParseIP("2001:db8::1"), net.ParseIP("2001:db8::2")}, + nil, + }, + { + "node with only external IPv4 address", + &apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + }, + Status: apiv1.NodeStatus{ + Addresses: []apiv1.NodeAddress{ + { + Type: apiv1.NodeExternalIP, + Address: "192.168.1.1", + }, + { + Type: apiv1.NodeInternalIP, + Address: "2001:db8::1", + }, + }, + }, + }, + []net.IP{net.ParseIP("2001:db8::1")}, + nil, + }, + { + "node with no IPv4 addresses", + &apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + }, + Status: apiv1.NodeStatus{ + Addresses: []apiv1.NodeAddress{}, + }, + }, + []net.IP{}, + errors.New("error getting primary NodeIP: host IP unknown"), + }, + } + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + krNode, err := NewRemoteKRNode(testcase.node) + if err != nil { + if testcase.err == nil { + t.Fatalf("failed to create KRNode: %v", err) + } + assert.EqualError(t, err, testcase.err.Error()) + return + } + ipv4Addrs := krNode.GetNodeIPv6Addrs() + assert.Equal(t, testcase.expected, ipv4Addrs) + }) + } +} + +func Test_FindBestIPv6NodeAddress(t *testing.T) { + testcases := []struct { + name string + node *apiv1.Node + expected net.IP + }{ + { + "primary IP is already IPv6", + &apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + }, + Status: apiv1.NodeStatus{ + Addresses: []apiv1.NodeAddress{ + { + Type: apiv1.NodeInternalIP, + Address: "2001:db8::1", + }, + }, + }, + }, + net.ParseIP("2001:db8::1"), + }, + { + "internal IPv6 address available", + &apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + }, + Status: apiv1.NodeStatus{ + Addresses: []apiv1.NodeAddress{ + { + Type: apiv1.NodeInternalIP, + Address: "2001:db8::1", + }, + { + Type: apiv1.NodeExternalIP, + Address: "2001:db8::2", + }, + }, + }, + }, + net.ParseIP("2001:db8::1"), + }, + { + "external IPv6 address available", + &apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + }, + Status: apiv1.NodeStatus{ + Addresses: []apiv1.NodeAddress{ + { + Type: apiv1.NodeExternalIP, + Address: "2001:db8::2", + }, + }, + }, + }, + net.ParseIP("2001:db8::2"), + }, + { + "no IPv6 address available", + &apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + }, + Status: apiv1.NodeStatus{ + Addresses: []apiv1.NodeAddress{ + { + Type: apiv1.NodeInternalIP, + Address: "10.0.0.1", + }, + }, + }, + }, + nil, + }, + } + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + krNode, err := NewRemoteKRNode(testcase.node) + if err != nil { + t.Fatalf("failed to create KRNode: %v", err) + } + ip := krNode.FindBestIPv6NodeAddress() + assert.Equal(t, testcase.expected, ip) + }) + } +} + +func Test_FindBestIPv4NodeAddress(t *testing.T) { + testcases := []struct { + name string + node *apiv1.Node + expected net.IP + }{ + { + "primary IP is already IPv4", + &apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + }, + Status: apiv1.NodeStatus{ + Addresses: []apiv1.NodeAddress{ + { + Type: apiv1.NodeInternalIP, + Address: "10.0.0.1", + }, + { + Type: apiv1.NodeInternalIP, + Address: "2001:db8::1", + }, + }, + }, + }, + net.ParseIP("10.0.0.1"), + }, + { + "internal IPv4 address available", + &apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + }, + Status: apiv1.NodeStatus{ + Addresses: []apiv1.NodeAddress{ + { + Type: apiv1.NodeInternalIP, + Address: "10.0.0.1", + }, + { + Type: apiv1.NodeExternalIP, + Address: "192.168.1.1", + }, + { + Type: apiv1.NodeInternalIP, + Address: "2001:db8::1", + }, + }, + }, + }, + net.ParseIP("10.0.0.1"), + }, + { + "external IPv4 address available", + &apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + }, + Status: apiv1.NodeStatus{ + Addresses: []apiv1.NodeAddress{ + { + Type: apiv1.NodeExternalIP, + Address: "192.168.1.1", + }, + { + Type: apiv1.NodeInternalIP, + Address: "2001:db8::1", + }, + }, + }, + }, + net.ParseIP("192.168.1.1"), + }, + { + "no IPv4 address available", + &apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + }, + Status: apiv1.NodeStatus{ + Addresses: []apiv1.NodeAddress{ + { + Type: apiv1.NodeInternalIP, + Address: "2001:db8::1", + }, + }, + }, + }, + nil, + }, + } + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + krNode, err := NewRemoteKRNode(testcase.node) + if err != nil { + t.Fatalf("failed to create KRNode: %v", err) + } + ip := krNode.FindBestIPv4NodeAddress() + assert.Equal(t, testcase.expected, ip) + }) + } +} + +func Test_NewKRNode(t *testing.T) { + testcases := []struct { + name string + node *apiv1.Node + linkQ LocalLinkQuerier + enableIPv4 bool + enableIPv6 bool + expectedErr error + }{ + { + "valid node with IPv4 and IPv6", + &apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + }, + Status: apiv1.NodeStatus{ + Addresses: []apiv1.NodeAddress{ + { + Type: apiv1.NodeInternalIP, + Address: "10.0.0.1", + }, + { + Type: apiv1.NodeInternalIP, + Address: "2001:db8::1", + }, + }, + }, + }, + NewFakeLocalLinkQuerier([]string{"10.0.0.1", "2001:db8::1"}, nil), + true, + true, + nil, + }, + { + "node with no IPv4 address", + &apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + }, + Status: apiv1.NodeStatus{ + Addresses: []apiv1.NodeAddress{ + { + Type: apiv1.NodeInternalIP, + Address: "2001:db8::1", + }, + }, + }, + }, + NewFakeLocalLinkQuerier([]string{"2001:db8::1"}, nil), + true, + true, + errors.New("IPv4 was enabled, but no IPv4 address was found on the node"), + }, + { + "node with no IPv6 address", + &apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + }, + Status: apiv1.NodeStatus{ + Addresses: []apiv1.NodeAddress{ + { + Type: apiv1.NodeInternalIP, + Address: "10.0.0.1", + }, + }, + }, + }, + NewFakeLocalLinkQuerier([]string{"10.0.0.1"}, nil), + true, + true, + errors.New("IPv6 was enabled, but no IPv6 address was found on the node"), + }, + } + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + _, err := NewKRNode(testcase.node, testcase.linkQ, testcase.enableIPv4, testcase.enableIPv6) + if testcase.expectedErr != nil { + assert.EqualError(t, err, testcase.expectedErr.Error()) + } else { + assert.NoError(t, err) + } + }) + } +} + +func Test_NewRemoteKRNode(t *testing.T) { + testcases := []struct { + name string + node *apiv1.Node + expectedErr error + }{ + { + "valid node with IPv4 and IPv6", + &apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + }, + Status: apiv1.NodeStatus{ + Addresses: []apiv1.NodeAddress{ + { + Type: apiv1.NodeInternalIP, + Address: "10.0.0.1", + }, + { + Type: apiv1.NodeInternalIP, + Address: "2001:db8::1", + }, + }, + }, + }, + nil, + }, + { + "node with no addresses", + &apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + }, + Status: apiv1.NodeStatus{ + Addresses: []apiv1.NodeAddress{}, + }, + }, + errors.New("error getting primary NodeIP: host IP unknown"), + }, + } + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + _, err := NewRemoteKRNode(testcase.node) + if testcase.expectedErr != nil { + assert.EqualError(t, err, testcase.expectedErr.Error()) + } else { + assert.NoError(t, err) + } + }) + } +} + +func Test_GetNodeMTU(t *testing.T) { + testcases := []struct { + name string + node *apiv1.Node + linkQ LocalLinkQuerier + expectedMTU int + expectedErr error + }{ + { + "valid node with MTU", + &apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + }, + Status: apiv1.NodeStatus{ + Addresses: []apiv1.NodeAddress{ + { + Type: apiv1.NodeInternalIP, + Address: "10.0.0.1", + }, + { + Type: apiv1.NodeInternalIP, + Address: "2001:db8::1", + }, + }, + }, + }, + NewFakeLocalLinkQuerier([]string{"10.0.0.1", "2001:db8::1"}, []int{1480, 1500}), + 1480, + nil, + }, + } + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + krNode, err := NewKRNode(testcase.node, testcase.linkQ, true, true) + if err != nil { + t.Fatalf("failed to create KRNode: %v", err) + } + mtu, err := krNode.GetNodeMTU() + if testcase.expectedErr != nil { + assert.EqualError(t, err, testcase.expectedErr.Error()) + } else { + assert.NoError(t, err) + assert.Equal(t, testcase.expectedMTU, mtu) + } + }) + } +} +func Test_GetNodeSubnet(t *testing.T) { + testcases := []struct { + name string + nodeIP net.IP + setupMock func(*MockLocalLinkQuerier) + expectedNet net.IPNet + expectedInt string + expectedErr error + }{ + { + "valid node with subnet", + net.ParseIP("10.0.0.1"), + func(myMock *MockLocalLinkQuerier) { + myMock.On("LinkList").Return( + []netlink.Link{&netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: "eth0"}}}, nil) + myMock.On("AddrList", mock.Anything, mock.Anything).Return( + []netlink.Addr{{IPNet: &net.IPNet{IP: net.ParseIP("10.0.0.1"), Mask: net.CIDRMask(24, 32)}}}, nil) + }, + net.IPNet{IP: net.ParseIP("10.0.0.1"), Mask: net.CIDRMask(24, 32)}, + "eth0", + nil, + }, + { + "node with no matching IP", + net.ParseIP("10.0.0.2"), + func(myMock *MockLocalLinkQuerier) { + myMock.On("LinkList").Return( + []netlink.Link{&netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: "eth0"}}}, nil) + myMock.On("AddrList", mock.Anything, mock.Anything).Return( + []netlink.Addr{{IPNet: &net.IPNet{IP: net.ParseIP("10.0.0.1"), Mask: net.CIDRMask(24, 32)}}}, nil) + }, + net.IPNet{}, + "", + errors.New("failed to find interface with specified node ip"), + }, + { + "error getting list of links", + net.ParseIP("10.0.0.1"), + func(myMock *MockLocalLinkQuerier) { + myMock.On("LinkList").Return([]netlink.Link{}, errors.New("failed to get list of links")) + }, + net.IPNet{}, + "", + errors.New("failed to get list of links"), + }, + { + "error getting addrs", + net.ParseIP("10.0.0.1"), + func(myMock *MockLocalLinkQuerier) { + myMock.On("LinkList").Return( + []netlink.Link{&netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: "eth0"}}}, nil) + myMock.On("AddrList", mock.Anything, mock.Anything).Return( + []netlink.Addr{}, errors.New("failed to get list of addrs")) + }, + net.IPNet{}, + "", + errors.New("failed to get list of addrs"), + }, + } + + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { + mockLinkQ := &MockLocalLinkQuerier{} + testcase.setupMock(mockLinkQ) + subnet, iface, err := GetNodeSubnet(testcase.nodeIP, mockLinkQ) + if testcase.expectedErr != nil { + assert.EqualError(t, err, testcase.expectedErr.Error()) + } else { + assert.NoError(t, err) + assert.Equal(t, testcase.expectedNet, subnet) + assert.Equal(t, testcase.expectedInt, iface) + } + }) + } +}