diff --git a/Makefile b/Makefile index 016c97734c..3545f903d4 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,10 @@ COREFILES = \ $(wildcard log/*.go) \ $(wildcard netlink/*.go) \ $(wildcard network/*.go) \ + $(wildcard network/epcommon/*.go) \ + $(wildcard network/ovsnfravnet/*.go) \ + $(wildcard network/ovssnat/*.go) \ + $(wildcard network/policy/*.go) \ $(wildcard platform/*.go) \ $(wildcard store/*.go) diff --git a/cni/netconfig.go b/cni/netconfig.go index 7ce5057bcd..db335372e0 100644 --- a/cni/netconfig.go +++ b/cni/netconfig.go @@ -24,17 +24,20 @@ type KVPair struct { // NetworkConfig represents Azure CNI plugin network configuration. type NetworkConfig struct { - CNIVersion string `json:"cniVersion"` - Name string `json:"name"` - Type string `json:"type"` - Mode string `json:"mode"` - Master string `json:"master"` - Bridge string `json:"bridge,omitempty"` - LogLevel string `json:"logLevel,omitempty"` - LogTarget string `json:"logTarget,omitempty"` - MultiTenancy bool `json:"multiTenancy,omitempty"` - EnableSnatOnHost bool `json:"enableSnatOnHost,omitempty"` - Ipam struct { + CNIVersion string `json:"cniVersion"` + Name string `json:"name"` + Type string `json:"type"` + Mode string `json:"mode"` + Master string `json:"master"` + Bridge string `json:"bridge,omitempty"` + LogLevel string `json:"logLevel,omitempty"` + LogTarget string `json:"logTarget,omitempty"` + InfraVnetAddressSpace string `json:"infraVnetAddressSpace,omitempty"` + PodNamespaceForDualNetwork []string `json:"podNamespaceForDualNetwork,omitempty"` + MultiTenancy bool `json:"multiTenancy,omitempty"` + EnableSnatOnHost bool `json:"enableSnatOnHost,omitempty"` + EnableExactMatchForPodName bool `json:"enableExactMatchForPodName,omitempty"` + Ipam struct { Type string `json:"type"` Environment string `json:"environment,omitempty"` AddrSpace string `json:"addressSpace,omitempty"` diff --git a/cni/network/mutlitenancy.go b/cni/network/mutlitenancy.go index e3d26a2e08..36c617a1e1 100644 --- a/cni/network/mutlitenancy.go +++ b/cni/network/mutlitenancy.go @@ -2,6 +2,7 @@ package network import ( "encoding/json" + "errors" "fmt" "net" "strings" @@ -16,7 +17,12 @@ import ( cniTypesCurr "github.com/containernetworking/cni/pkg/types/current" ) -func SetupRoutingForMultitenancy(nwCfg *cni.NetworkConfig, cnsNetworkConfig *cns.GetNetworkContainerResponse, epInfo *network.EndpointInfo, result *cniTypesCurr.Result) { +func SetupRoutingForMultitenancy( + nwCfg *cni.NetworkConfig, + cnsNetworkConfig *cns.GetNetworkContainerResponse, + azIpamResult *cniTypesCurr.Result, + epInfo *network.EndpointInfo, + result *cniTypesCurr.Result) { // Adding default gateway if nwCfg.MultiTenancy { // if snat enabled, add 169.254.0.1 as default gateway @@ -30,20 +36,34 @@ func SetupRoutingForMultitenancy(nwCfg *cni.NetworkConfig, cnsNetworkConfig *cns epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: dstIP, Gw: gwIP}) result.Routes = append(result.Routes, &cniTypes.Route{Dst: dstIP, GW: gwIP}) } + + setupInfraVnetRoutingForMultitenancy(nwCfg, azIpamResult, epInfo, result) } } -func GetContainerNetworkConfiguration(multiTenancy bool, address string, podName string, podNamespace string) (*cniTypesCurr.Result, *cns.GetNetworkContainerResponse, net.IPNet, error) { - if multiTenancy { - podNameWithoutSuffix := getPodNameWithoutSuffix(podName) - log.Printf("Podname without suffix %v", podNameWithoutSuffix) - return getContainerNetworkConfiguration(address, podNamespace, podNameWithoutSuffix) +func getContainerNetworkConfiguration( + nwCfg *cni.NetworkConfig, + address string, + podName string, + podNamespace string, + ifName string) (*cniTypesCurr.Result, *cns.GetNetworkContainerResponse, net.IPNet, error) { + var podNameWithoutSuffix string + + if !nwCfg.EnableExactMatchForPodName { + podNameWithoutSuffix = getPodNameWithoutSuffix(podName) + } else { + podNameWithoutSuffix = podName } - return nil, nil, net.IPNet{}, nil + log.Printf("Podname without suffix %v", podNameWithoutSuffix) + return getContainerNetworkConfigurationInternal(address, podNamespace, podNameWithoutSuffix, ifName) } -func getContainerNetworkConfiguration(address string, namespace string, podName string) (*cniTypesCurr.Result, *cns.GetNetworkContainerResponse, net.IPNet, error) { +func getContainerNetworkConfigurationInternal( + address string, + namespace string, + podName string, + ifName string) (*cniTypesCurr.Result, *cns.GetNetworkContainerResponse, net.IPNet, error) { cnsClient, err := cnsclient.NewCnsClient(address) if err != nil { log.Printf("Initializing CNS client error %v", err) @@ -72,10 +92,10 @@ func getContainerNetworkConfiguration(address string, namespace string, podName return nil, nil, net.IPNet{}, fmt.Errorf(errBuf) } - return convertToCniResult(networkConfig), networkConfig, *subnetPrefix, nil + return convertToCniResult(networkConfig, ifName), networkConfig, *subnetPrefix, nil } -func convertToCniResult(networkConfig *cns.GetNetworkContainerResponse) *cniTypesCurr.Result { +func convertToCniResult(networkConfig *cns.GetNetworkContainerResponse, ifName string) *cniTypesCurr.Result { result := &cniTypesCurr.Result{} resultIpconfig := &cniTypesCurr.IPConfig{} @@ -109,6 +129,9 @@ func convertToCniResult(networkConfig *cns.GetNetworkContainerResponse) *cniType result.Routes = append(result.Routes, &cniTypes.Route{Dst: routeIPnet, GW: gwIP}) } + iface := &cniTypesCurr.Interface{Name: ifName} + result.Interfaces = append(result.Interfaces, iface) + return result } @@ -124,3 +147,117 @@ func getPodNameWithoutSuffix(podName string) string { log.Printf("Pod name after splitting based on - : %v", nameSplit) return strings.Join(nameSplit, "-") } + +func getInfraVnetIP( + enableInfraVnet bool, + infraSubnet string, + nwCfg *cni.NetworkConfig, + plugin *netPlugin, +) (*cniTypesCurr.Result, error) { + + if enableInfraVnet { + _, ipNet, _ := net.ParseCIDR(infraSubnet) + nwCfg.Ipam.Subnet = ipNet.String() + + log.Printf("call ipam to allocate ip from subnet %v", nwCfg.Ipam.Subnet) + azIpamResult, err := plugin.DelegateAdd(nwCfg.Ipam.Type, nwCfg) + if err != nil { + err = plugin.Errorf("Failed to allocate address: %v", err) + return nil, err + } + + return azIpamResult, nil + } + + return nil, nil +} + +func cleanupInfraVnetIP( + enableInfraVnet bool, + infraIPNet *net.IPNet, + nwCfg *cni.NetworkConfig, + plugin *netPlugin) { + + log.Printf("Cleanup infravnet ip") + + if enableInfraVnet { + _, ipNet, _ := net.ParseCIDR(infraIPNet.String()) + nwCfg.Ipam.Subnet = ipNet.String() + nwCfg.Ipam.Address = infraIPNet.IP.String() + plugin.DelegateDel(nwCfg.Ipam.Type, nwCfg) + } +} + +func checkIfSubnetOverlaps(enableInfraVnet bool, nwCfg *cni.NetworkConfig, cnsNetworkConfig *cns.GetNetworkContainerResponse) bool { + if enableInfraVnet { + if cnsNetworkConfig != nil { + _, infraNet, _ := net.ParseCIDR(nwCfg.InfraVnetAddressSpace) + for _, cnetSpace := range cnsNetworkConfig.CnetAddressSpace { + cnetSpaceIPNet := &net.IPNet{ + IP: net.ParseIP(cnetSpace.IPAddress), + Mask: net.CIDRMask(int(cnetSpace.PrefixLength), 32), + } + + return infraNet.Contains(cnetSpaceIPNet.IP) || cnetSpaceIPNet.Contains(infraNet.IP) + } + } + } + + return false +} + +func GetMultiTenancyCNIResult( + enableInfraVnet bool, + nwCfg *cni.NetworkConfig, + plugin *netPlugin, + k8sPodName string, + k8sNamespace string, + ifName string) (*cniTypesCurr.Result, *cns.GetNetworkContainerResponse, net.IPNet, *cniTypesCurr.Result, error) { + + if nwCfg.MultiTenancy { + result, cnsNetworkConfig, subnetPrefix, err := getContainerNetworkConfiguration(nwCfg, "", k8sPodName, k8sNamespace, ifName) + if err != nil { + log.Printf("GetContainerNetworkConfiguration failed for podname %v namespace %v with error %v", k8sPodName, k8sNamespace, err) + return nil, nil, net.IPNet{}, nil, err + } + + log.Printf("PrimaryInterfaceIdentifier :%v", subnetPrefix.IP.String()) + + if checkIfSubnetOverlaps(enableInfraVnet, nwCfg, cnsNetworkConfig) { + buf := fmt.Sprintf("InfraVnet %v overlaps with customerVnet %+v", nwCfg.InfraVnetAddressSpace, cnsNetworkConfig.CnetAddressSpace) + log.Printf(buf) + err = errors.New(buf) + return nil, nil, net.IPNet{}, nil, err + } + + if nwCfg.EnableSnatOnHost { + if cnsNetworkConfig.LocalIPConfiguration.IPSubnet.IPAddress == "" { + log.Printf("Snat IP is not populated. Got empty string") + return nil, nil, net.IPNet{}, nil, fmt.Errorf("Snat IP is not populated. Got empty string") + } + } + + if enableInfraVnet { + if nwCfg.InfraVnetAddressSpace == "" { + log.Printf("InfraVnetAddressSpace is not populated. Got empty string") + return nil, nil, net.IPNet{}, nil, fmt.Errorf("InfraVnetAddressSpace is not populated. Got empty string") + } + } + + azIpamResult, err := getInfraVnetIP(enableInfraVnet, subnetPrefix.String(), nwCfg, plugin) + if err != nil { + log.Printf("GetInfraVnetIP failed with error %v", err) + return nil, nil, net.IPNet{}, nil, err + } + + return result, cnsNetworkConfig, subnetPrefix, azIpamResult, nil + } + + return nil, nil, net.IPNet{}, nil, nil +} + +func CleanupMultitenancyResources(enableInfraVnet bool, nwCfg *cni.NetworkConfig, azIpamResult *cniTypesCurr.Result, plugin *netPlugin) { + if nwCfg.MultiTenancy && azIpamResult != nil && azIpamResult.IPs != nil { + cleanupInfraVnetIP(enableInfraVnet, &azIpamResult.IPs[0].Address, nwCfg, plugin) + } +} diff --git a/cni/network/network.go b/cni/network/network.go index 5d2dd15f63..7a4fb8acc3 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -139,17 +139,28 @@ func GetEndpointID(args *cniSkel.CmdArgs) string { func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { var ( result *cniTypesCurr.Result + azIpamResult *cniTypesCurr.Result err error nwCfg *cni.NetworkConfig epInfo *network.EndpointInfo iface *cniTypesCurr.Interface subnetPrefix net.IPNet cnsNetworkConfig *cns.GetNetworkContainerResponse + enableInfraVnet bool ) log.Printf("[cni-net] Processing ADD command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.", args.ContainerID, args.Netns, args.IfName, args.Args, args.Path) + // Parse network configuration from stdin. + nwCfg, err = cni.ParseNetworkConfig(args.StdinData) + if err != nil { + err = plugin.Errorf("Failed to parse network configuration: %v.", err) + return err + } + + log.Printf("[cni-net] Read network configuration %+v.", nwCfg) + defer func() { // Add Interfaces to result. if result == nil { @@ -214,26 +225,31 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { return plugin.Errorf(errMsg) } - // Parse network configuration from stdin. - nwCfg, err = cni.ParseNetworkConfig(args.StdinData) - if err != nil { - err = plugin.Errorf("Failed to parse network configuration: %v.", err) - return err + for _, ns := range nwCfg.PodNamespaceForDualNetwork { + if k8sNamespace == ns { + log.Printf("Enable infravnet for this pod %v in namespace %v", k8sPodName, k8sNamespace) + enableInfraVnet = true + break + } } - log.Printf("[cni-net] Read network configuration %+v.", nwCfg) - // Initialize values from network config. networkId := nwCfg.Name endpointId := GetEndpointID(args) - result, cnsNetworkConfig, subnetPrefix, err = GetContainerNetworkConfiguration(nwCfg.MultiTenancy, "", k8sPodName, k8sNamespace) + result, cnsNetworkConfig, subnetPrefix, azIpamResult, err = GetMultiTenancyCNIResult(enableInfraVnet, nwCfg, plugin, k8sPodName, k8sNamespace, args.IfName) if err != nil { - log.Printf("GetContainerNetworkConfiguration failed for podname %v namespace %v with error %v", k8sPodName, k8sNamespace, err) + log.Printf("GetMultiTenancyCNIResult failed with error %v", err) return err } - log.Printf("PrimaryInterfaceIdentifier :%v", subnetPrefix.IP.String()) + defer func() { + if err != nil { + CleanupMultitenancyResources(enableInfraVnet, nwCfg, azIpamResult, plugin) + } + }() + + log.Printf("Result from multitenancy %+v", result) policies := cni.GetPoliciesFromNwCfg(nwCfg.AdditionalArgs) @@ -275,6 +291,9 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { // Derive the subnet prefix from allocated IP address. subnetPrefix = result.IPs[0].Address + + iface := &cniTypesCurr.Interface{Name: args.IfName} + result.Interfaces = append(result.Interfaces, iface) } ipconfig := result.IPs[0] @@ -354,6 +373,9 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { ipconfig := result.IPs[0] + iface := &cniTypesCurr.Interface{Name: args.IfName} + result.Interfaces = append(result.Interfaces, iface) + // On failure, call into IPAM plugin to release the address. defer func() { if err != nil { @@ -370,32 +392,16 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { NetNsPath: args.Netns, IfName: args.IfName, EnableSnatOnHost: nwCfg.EnableSnatOnHost, + EnableInfraVnet: enableInfraVnet, } epInfo.Data = make(map[string]interface{}) - // A runtime must not call ADD twice (without a corresponding DEL) for the same - // (network name, container id, name of the interface inside the container) - vethName := fmt.Sprintf("%s%s%s", networkId, k8sContainerID, k8sIfName) - setEndpointOptions(cnsNetworkConfig, epInfo, vethName) - - var dns network.DNSInfo - if (len(nwCfg.DNS.Search) == 0) != (len(nwCfg.DNS.Nameservers) == 0) { - err = plugin.Errorf("Wrong DNS configuration: %+v", nwCfg.DNS) + dns, err := getDNSSettings(nwCfg, result, k8sNamespace) + if err != nil { + log.Printf("Error retrieving dns settings %v", err) return err } - if len(nwCfg.DNS.Search) > 0 { - dns = network.DNSInfo{ - Servers: nwCfg.DNS.Nameservers, - Suffix: k8sNamespace + "." + strings.Join(nwCfg.DNS.Search, ","), - } - } else { - dns = network.DNSInfo{ - Suffix: result.DNS.Domain, - Servers: result.DNS.Nameservers, - } - } - epInfo.DNS = dns epInfo.Policies = policies @@ -409,7 +415,16 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW}) } - SetupRoutingForMultitenancy(nwCfg, cnsNetworkConfig, epInfo, result) + if azIpamResult != nil && azIpamResult.IPs != nil { + epInfo.InfraVnetIP = azIpamResult.IPs[0].Address + } + + SetupRoutingForMultitenancy(nwCfg, cnsNetworkConfig, azIpamResult, epInfo, result) + + // A runtime must not call ADD twice (without a corresponding DEL) for the same + // (network name, container id, name of the interface inside the container) + vethName := fmt.Sprintf("%s%s%s", networkId, k8sContainerID, k8sIfName) + setEndpointOptions(cnsNetworkConfig, epInfo, vethName) // Create the endpoint. log.Printf("[cni-net] Creating endpoint %v.", epInfo.Id) @@ -553,10 +568,20 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error { return err } - // Call into IPAM plugin to release the endpoint's addresses. - nwCfg.Ipam.Subnet = nwInfo.Subnets[0].Prefix.String() - for _, address := range epInfo.IPAddresses { - nwCfg.Ipam.Address = address.IP.String() + if !nwCfg.MultiTenancy { + // Call into IPAM plugin to release the endpoint's addresses. + nwCfg.Ipam.Subnet = nwInfo.Subnets[0].Prefix.String() + for _, address := range epInfo.IPAddresses { + nwCfg.Ipam.Address = address.IP.String() + err = plugin.DelegateDel(nwCfg.Ipam.Type, nwCfg) + if err != nil { + err = plugin.Errorf("Failed to release address: %v", err) + return err + } + } + } else if epInfo.EnableInfraVnet { + nwCfg.Ipam.Subnet = nwInfo.Subnets[0].Prefix.String() + nwCfg.Ipam.Address = epInfo.InfraVnetIP.IP.String() err = plugin.DelegateDel(nwCfg.Ipam.Type, nwCfg) if err != nil { err = plugin.Errorf("Failed to release address: %v", err) diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index e1b9fb5af4..d103274e1a 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -13,7 +13,8 @@ import ( ) const ( - snatInterface = "eth1" + snatInterface = "eth1" + infraInterface = "eth2" ) // handleConsecutiveAdd is a dummy function for Linux platform. @@ -29,6 +30,13 @@ func addDefaultRoute(gwIPString string, epInfo *network.EndpointInfo, result *cn result.Routes = append(result.Routes, &cniTypes.Route{Dst: dstIP, GW: gwIP}) } +func addInfraRoutes(azIpamResult *cniTypesCurr.Result, result *cniTypesCurr.Result, epInfo *network.EndpointInfo) { + for _, route := range azIpamResult.Routes { + epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW, DevName: infraInterface}) + result.Routes = append(result.Routes, &cniTypes.Route{Dst: route.Dst, GW: route.GW}) + } +} + func setNetworkOptions(cnsNwConfig *cns.GetNetworkContainerResponse, nwInfo *network.NetworkInfo) { if cnsNwConfig != nil && cnsNwConfig.MultiTenancyInfo.ID != 0 { log.Printf("Setting Network Options") @@ -59,3 +67,33 @@ func addSnatInterface(nwCfg *cni.NetworkConfig, result *cniTypesCurr.Result) { result.Interfaces = append(result.Interfaces, snatIface) } } + +func setupInfraVnetRoutingForMultitenancy( + nwCfg *cni.NetworkConfig, + azIpamResult *cniTypesCurr.Result, + epInfo *network.EndpointInfo, + result *cniTypesCurr.Result) { + + if epInfo.EnableInfraVnet { + _, ipNet, _ := net.ParseCIDR(nwCfg.InfraVnetAddressSpace) + epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: *ipNet, Gw: azIpamResult.IPs[0].Gateway, DevName: infraInterface}) + } +} + +func getDNSSettings(nwCfg *cni.NetworkConfig, result *cniTypesCurr.Result, namespace string) (network.DNSInfo, error) { + var dns network.DNSInfo + + if len(nwCfg.DNS.Nameservers) > 0 { + dns = network.DNSInfo{ + Servers: nwCfg.DNS.Nameservers, + Suffix: nwCfg.DNS.Domain, + } + } else { + dns = network.DNSInfo{ + Suffix: result.DNS.Domain, + Servers: result.DNS.Nameservers, + } + } + + return dns, nil +} diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index 031064e19e..ec459d48cc 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -3,6 +3,7 @@ package network import ( "fmt" "net" + "strings" "github.com/Azure/azure-container-networking/cni" "github.com/Azure/azure-container-networking/cns" @@ -63,6 +64,9 @@ func handleConsecutiveAdd(containerId, endpointId string, nwInfo *network.Networ func addDefaultRoute(gwIPString string, epInfo *network.EndpointInfo, result *cniTypesCurr.Result) { } +func addInfraRoutes(azIpamResult *cniTypesCurr.Result, result *cniTypesCurr.Result, epInfo *network.EndpointInfo) { +} + func setNetworkOptions(cnsNwConfig *cns.GetNetworkContainerResponse, nwInfo *network.NetworkInfo) { } @@ -71,3 +75,33 @@ func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *ne func addSnatInterface(nwCfg *cni.NetworkConfig, result *cniTypesCurr.Result) { } + +func setupInfraVnetRoutingForMultitenancy( + nwCfg *cni.NetworkConfig, + azIpamResult *cniTypesCurr.Result, + epInfo *network.EndpointInfo, + result *cniTypesCurr.Result) { +} + +func getDNSSettings(nwCfg *cni.NetworkConfig, result *cniTypesCurr.Result, namespace string) (network.DNSInfo, error) { + var dns network.DNSInfo + + if (len(nwCfg.DNS.Search) == 0) != (len(nwCfg.DNS.Nameservers) == 0) { + err := fmt.Errorf("Wrong DNS configuration: %+v", nwCfg.DNS) + return dns, err + } + + if len(nwCfg.DNS.Search) > 0 { + dns = network.DNSInfo{ + Servers: nwCfg.DNS.Nameservers, + Suffix: namespace + "." + strings.Join(nwCfg.DNS.Search, ","), + } + } else { + dns = network.DNSInfo{ + Suffix: result.DNS.Domain, + Servers: result.DNS.Nameservers, + } + } + + return dns, nil +} diff --git a/network/bridge_endpointclient_linux.go b/network/bridge_endpointclient_linux.go index 9f1d95e363..662f8a89c0 100644 --- a/network/bridge_endpointclient_linux.go +++ b/network/bridge_endpointclient_linux.go @@ -6,6 +6,7 @@ import ( "github.com/Azure/azure-container-networking/ebtables" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/network/epcommon" ) type LinuxBridgeEndpointClient struct { @@ -38,7 +39,7 @@ func NewLinuxBridgeEndpointClient( } func (client *LinuxBridgeEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { - if err := createEndpoint(client.hostVethName, client.containerVethName); err != nil { + if err := epcommon.CreateEndpoint(client.hostVethName, client.containerVethName); err != nil { return err } @@ -121,7 +122,7 @@ func (client *LinuxBridgeEndpointClient) MoveEndpointsToContainerNS(epInfo *Endp } func (client *LinuxBridgeEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { - if err := setupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { + if err := epcommon.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { return err } @@ -130,7 +131,7 @@ func (client *LinuxBridgeEndpointClient) SetupContainerInterfaces(epInfo *Endpoi } func (client *LinuxBridgeEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { - if err := assignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { + if err := epcommon.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { return err } diff --git a/network/endpoint.go b/network/endpoint.go index 12cb53206c..1a10d9277b 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -10,6 +10,10 @@ import ( "github.com/Azure/azure-container-networking/network/policy" ) +const ( + InfraVnet = 0 +) + // Endpoint represents a container network interface. type endpoint struct { Id string @@ -18,12 +22,14 @@ type endpoint struct { IfName string HostIfName string MacAddress net.HardwareAddr + InfraVnetIP net.IPNet IPAddresses []net.IPNet Gateways []net.IP DNS DNSInfo Routes []RouteInfo VlanID int EnableSnatOnHost bool + EnableInfraVnet bool } // EndpointInfo contains read-only information about an endpoint. @@ -37,10 +43,12 @@ type EndpointInfo struct { MacAddress net.HardwareAddr DNS DNSInfo IPAddresses []net.IPNet + InfraVnetIP net.IPNet Routes []RouteInfo Policies []policy.Policy Gateways []net.IP EnableSnatOnHost bool + EnableInfraVnet bool Data map[string]interface{} } @@ -129,12 +137,14 @@ func (ep *endpoint) getInfo() *EndpointInfo { info := &EndpointInfo{ Id: ep.Id, IPAddresses: ep.IPAddresses, + InfraVnetIP: ep.InfraVnetIP, Data: make(map[string]interface{}), MacAddress: ep.MacAddress, SandboxKey: ep.SandboxKey, IfIndex: 0, // Azure CNI supports only one interface DNS: ep.DNS, EnableSnatOnHost: ep.EnableSnatOnHost, + EnableInfraVnet: ep.EnableInfraVnet, } for _, route := range ep.Routes { diff --git a/network/endpoint_common_linux.go b/network/endpoint_common_linux.go deleted file mode 100644 index d96934d0c5..0000000000 --- a/network/endpoint_common_linux.go +++ /dev/null @@ -1,128 +0,0 @@ -package network - -import ( - "net" - "strings" - - "github.com/Azure/azure-container-networking/log" - "github.com/Azure/azure-container-networking/netlink" -) - -func createEndpoint(hostVethName string, containerVethName string) error { - log.Printf("[net] Creating veth pair %v %v.", hostVethName, containerVethName) - - link := netlink.VEthLink{ - LinkInfo: netlink.LinkInfo{ - Type: netlink.LINK_TYPE_VETH, - Name: hostVethName, - }, - PeerName: containerVethName, - } - - err := netlink.AddLink(&link) - if err != nil { - log.Printf("[net] Failed to create veth pair, err:%v.", err) - return err - } - - log.Printf("[net] Setting link %v state up.", hostVethName) - err = netlink.SetLinkState(hostVethName, true) - if err != nil { - return err - } - - return nil -} - -func setupContainerInterface(containerVethName string, targetIfName string) error { - // Interface needs to be down before renaming. - log.Printf("[net] Setting link %v state down.", containerVethName) - if err := netlink.SetLinkState(containerVethName, false); err != nil { - return err - } - - // Rename the container interface. - log.Printf("[net] Setting link %v name %v.", containerVethName, targetIfName) - if err := netlink.SetLinkName(containerVethName, targetIfName); err != nil { - return err - } - - // Bring the interface back up. - log.Printf("[net] Setting link %v state up.", targetIfName) - return netlink.SetLinkState(targetIfName, true) -} - -func assignIPToInterface(interfaceName string, ipAddresses []net.IPNet) error { - // Assign IP address to container network interface. - for _, ipAddr := range ipAddresses { - log.Printf("[net] Adding IP address %v to link %v.", ipAddr.String(), interfaceName) - err := netlink.AddIpAddress(interfaceName, ipAddr.IP, &ipAddr) - if err != nil { - return err - } - } - - return nil -} - -func addRoutes(interfaceName string, routes []RouteInfo) error { - ifIndex := 0 - interfaceIf, _ := net.InterfaceByName(interfaceName) - - for _, route := range routes { - log.Printf("[ovs] Adding IP route %+v to link %v.", route, interfaceName) - - if route.DevName != "" { - devIf, _ := net.InterfaceByName(route.DevName) - ifIndex = devIf.Index - } else { - ifIndex = interfaceIf.Index - } - - nlRoute := &netlink.Route{ - Family: netlink.GetIpAddressFamily(route.Gw), - Dst: &route.Dst, - Gw: route.Gw, - LinkIndex: ifIndex, - } - - if err := netlink.AddIpRoute(nlRoute); err != nil { - if !strings.Contains(strings.ToLower(err.Error()), "file exists") { - return err - } else { - log.Printf("route already exists") - } - } - } - - return nil -} - -func deleteRoutes(interfaceName string, routes []RouteInfo) error { - ifIndex := 0 - interfaceIf, _ := net.InterfaceByName(interfaceName) - - for _, route := range routes { - log.Printf("[ovs] Adding IP route %+v to link %v.", route, interfaceName) - - if route.DevName != "" { - devIf, _ := net.InterfaceByName(route.DevName) - ifIndex = devIf.Index - } else { - ifIndex = interfaceIf.Index - } - - nlRoute := &netlink.Route{ - Family: netlink.GetIpAddressFamily(route.Gw), - Dst: &route.Dst, - Gw: route.Gw, - LinkIndex: ifIndex, - } - - if err := netlink.DeleteIpRoute(nlRoute); err != nil { - return err - } - } - - return nil -} diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go index 4862b5c35d..a06476671e 100644 --- a/network/endpoint_linux.go +++ b/network/endpoint_linux.go @@ -10,8 +10,10 @@ import ( "encoding/hex" "fmt" "net" + "strings" "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/netlink" ) const ( @@ -175,11 +177,13 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) { IfName: epInfo.IfName, HostIfName: hostIfName, MacAddress: containerIf.HardwareAddr, + InfraVnetIP: epInfo.InfraVnetIP, IPAddresses: epInfo.IPAddresses, Gateways: []net.IP{nw.extIf.IPv4Gateway}, DNS: epInfo.DNS, VlanID: vlanid, EnableSnatOnHost: epInfo.EnableSnatOnHost, + EnableInfraVnet: epInfo.EnableInfraVnet, } for _, route := range epInfo.Routes { @@ -212,3 +216,65 @@ func (nw *network) deleteEndpointImpl(ep *endpoint) error { // getInfoImpl returns information about the endpoint. func (ep *endpoint) getInfoImpl(epInfo *EndpointInfo) { } + +func addRoutes(interfaceName string, routes []RouteInfo) error { + ifIndex := 0 + interfaceIf, _ := net.InterfaceByName(interfaceName) + + for _, route := range routes { + log.Printf("[ovs] Adding IP route %+v to link %v.", route, interfaceName) + + if route.DevName != "" { + devIf, _ := net.InterfaceByName(route.DevName) + ifIndex = devIf.Index + } else { + ifIndex = interfaceIf.Index + } + + nlRoute := &netlink.Route{ + Family: netlink.GetIpAddressFamily(route.Gw), + Dst: &route.Dst, + Gw: route.Gw, + LinkIndex: ifIndex, + } + + if err := netlink.AddIpRoute(nlRoute); err != nil { + if !strings.Contains(strings.ToLower(err.Error()), "file exists") { + return err + } else { + log.Printf("route already exists") + } + } + } + + return nil +} + +func deleteRoutes(interfaceName string, routes []RouteInfo) error { + ifIndex := 0 + interfaceIf, _ := net.InterfaceByName(interfaceName) + + for _, route := range routes { + log.Printf("[ovs] Adding IP route %+v to link %v.", route, interfaceName) + + if route.DevName != "" { + devIf, _ := net.InterfaceByName(route.DevName) + ifIndex = devIf.Index + } else { + ifIndex = interfaceIf.Index + } + + nlRoute := &netlink.Route{ + Family: netlink.GetIpAddressFamily(route.Gw), + Dst: &route.Dst, + Gw: route.Gw, + LinkIndex: ifIndex, + } + + if err := netlink.DeleteIpRoute(nlRoute); err != nil { + return err + } + } + + return nil +} diff --git a/network/epcommon/endpoint_common_linux.go b/network/epcommon/endpoint_common_linux.go new file mode 100644 index 0000000000..3851f6de99 --- /dev/null +++ b/network/epcommon/endpoint_common_linux.go @@ -0,0 +1,147 @@ +package epcommon + +import ( + "fmt" + "net" + + "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/platform" +) + +func getPrivateIPSpace() []string { + privateIPAddresses := []string{"10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"} + return privateIPAddresses +} + +func getFilterChains() []string { + chains := []string{"FORWARD", "INPUT", "OUTPUT"} + return chains +} + +func getFilterchainTarget() []string { + actions := []string{"ACCEPT", "DROP"} + return actions +} + +func CreateEndpoint(hostVethName string, containerVethName string) error { + log.Printf("[net] Creating veth pair %v %v.", hostVethName, containerVethName) + + link := netlink.VEthLink{ + LinkInfo: netlink.LinkInfo{ + Type: netlink.LINK_TYPE_VETH, + Name: hostVethName, + }, + PeerName: containerVethName, + } + + err := netlink.AddLink(&link) + if err != nil { + log.Printf("[net] Failed to create veth pair, err:%v.", err) + return err + } + + log.Printf("[net] Setting link %v state up.", hostVethName) + err = netlink.SetLinkState(hostVethName, true) + if err != nil { + return err + } + + return nil +} + +func SetupContainerInterface(containerVethName string, targetIfName string) error { + // Interface needs to be down before renaming. + log.Printf("[net] Setting link %v state down.", containerVethName) + if err := netlink.SetLinkState(containerVethName, false); err != nil { + return err + } + + // Rename the container interface. + log.Printf("[net] Setting link %v name %v.", containerVethName, targetIfName) + if err := netlink.SetLinkName(containerVethName, targetIfName); err != nil { + return err + } + + // Bring the interface back up. + log.Printf("[net] Setting link %v state up.", targetIfName) + return netlink.SetLinkState(targetIfName, true) +} + +func AssignIPToInterface(interfaceName string, ipAddresses []net.IPNet) error { + // Assign IP address to container network interface. + for _, ipAddr := range ipAddresses { + log.Printf("[net] Adding IP address %v to link %v.", ipAddr.String(), interfaceName) + err := netlink.AddIpAddress(interfaceName, ipAddr.IP, &ipAddr) + if err != nil { + return err + } + } + + return nil +} + +func addOrDeleteFilterRule(bridgeName string, action string, ipAddress string, chainName string, target string) error { + option := "i" + + if chainName == "OUTPUT" { + option = "o" + } + + if action != "D" { + cmd := fmt.Sprintf("iptables -t filter -C %v -%v %v -d %v -j %v", chainName, option, bridgeName, ipAddress, target) + _, err := platform.ExecuteCommand(cmd) + if err == nil { + log.Printf("Iptable filter for private ipaddr %v on %v chain %v target rule already exists", ipAddress, chainName, target) + return nil + } + } + + cmd := fmt.Sprintf("iptables -t filter -%v %v -%v %v -d %v -j %v", action, chainName, option, bridgeName, ipAddress, target) + _, err := platform.ExecuteCommand(cmd) + if err != nil { + log.Printf("Iptable filter %v action for private ipaddr %v on %v chain %v target failed with %v", action, ipAddress, chainName, target, err) + return err + } + + return nil +} + +func AddOrDeletePrivateIPBlockRule(bridgeName string, skipAddresses []string, action string) error { + privateIPAddresses := getPrivateIPSpace() + chains := getFilterChains() + target := getFilterchainTarget() + + log.Printf("[net] Addresses to allow %v", skipAddresses) + + for _, address := range skipAddresses { + if err := addOrDeleteFilterRule(bridgeName, action, address, chains[0], target[0]); err != nil { + return err + } + + if err := addOrDeleteFilterRule(bridgeName, action, address, chains[1], target[0]); err != nil { + return err + } + + if err := addOrDeleteFilterRule(bridgeName, action, address, chains[2], target[0]); err != nil { + return err + } + + } + + for _, ipAddress := range privateIPAddresses { + if err := addOrDeleteFilterRule(bridgeName, action, ipAddress, chains[0], target[1]); err != nil { + return err + } + + if err := addOrDeleteFilterRule(bridgeName, action, ipAddress, chains[1], target[1]); err != nil { + return err + } + + if err := addOrDeleteFilterRule(bridgeName, action, ipAddress, chains[2], target[1]); err != nil { + return err + } + } + + return nil +} diff --git a/network/epcommon/endpoint_common_windows.go b/network/epcommon/endpoint_common_windows.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/network/network.go b/network/network.go index f2def56489..0e68c490b4 100644 --- a/network/network.go +++ b/network/network.go @@ -40,6 +40,7 @@ type network struct { Subnets []SubnetInfo Endpoints map[string]*endpoint extIf *externalInterface + DNS DNSInfo EnableSnatOnHost bool } diff --git a/network/network_linux.go b/network/network_linux.go index 5c336a2a2b..f31e7d4507 100644 --- a/network/network_linux.go +++ b/network/network_linux.go @@ -29,6 +29,8 @@ const ( LocalIPKey = "localIP" + InfraVnetIPKey = "infraVnetIP" + OptVethName = "vethname" ) @@ -66,6 +68,7 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt Endpoints: make(map[string]*endpoint), extIf: extIf, VlanId: vlanid, + DNS: nwInfo.DNS, EnableSnatOnHost: nwInfo.EnableSnatOnHost, } @@ -77,7 +80,7 @@ func (nm *networkManager) deleteNetworkImpl(nw *network) error { var networkClient NetworkClient if nw.VlanId != 0 { - networkClient = NewOVSClient(nw.extIf.BridgeName, nw.extIf.Name, "", nw.EnableSnatOnHost) + networkClient = NewOVSClient(nw.extIf.BridgeName, nw.extIf.Name, "", nw.DNS.Servers, nw.EnableSnatOnHost) } else { networkClient = NewLinuxBridgeClient(nw.extIf.BridgeName, nw.extIf.Name, nw.Mode) } @@ -201,7 +204,7 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwI snatBridgeIP, _ = opt[SnatBridgeIPKey].(string) } - networkClient = NewOVSClient(bridgeName, extIf.Name, snatBridgeIP, nwInfo.EnableSnatOnHost) + networkClient = NewOVSClient(bridgeName, extIf.Name, snatBridgeIP, nwInfo.DNS.Servers, nwInfo.EnableSnatOnHost) } else { networkClient = NewLinuxBridgeClient(bridgeName, extIf.Name, nwInfo.Mode) } @@ -325,3 +328,20 @@ func getNetworkInfoImpl(nwInfo *NetworkInfo, nw *network) { nwInfo.Options[genericData] = vlanMap } } + +func AddStaticRoute(ip string, interfaceName string) error { + log.Printf("[ovs] Adding %v static route", ip) + var routes []RouteInfo + _, ipNet, _ := net.ParseCIDR(ip) + gwIP := net.ParseIP("0.0.0.0") + route := RouteInfo{Dst: *ipNet, Gw: gwIP} + routes = append(routes, route) + if err := addRoutes(interfaceName, routes); err != nil { + if err != nil && !strings.Contains(strings.ToLower(err.Error()), "file exists") { + log.Printf("addroutes failed with error %v", err) + return err + } + } + + return nil +} diff --git a/network/ovs_endpoint_infraroute_linux.go b/network/ovs_endpoint_infraroute_linux.go new file mode 100644 index 0000000000..b411088e23 --- /dev/null +++ b/network/ovs_endpoint_infraroute_linux.go @@ -0,0 +1,71 @@ +package network + +import ( + "fmt" + "net" + + "github.com/Azure/azure-container-networking/network/ovsinfravnet" +) + +func NewInfraVnetClient(client *OVSEndpointClient, epID string) { + if client.enableInfraVnet { + hostIfName := fmt.Sprintf("%s%s", infraVethInterfacePrefix, epID) + contIfName := fmt.Sprintf("%s%s-2", infraVethInterfacePrefix, epID) + + client.infraVnetClient = ovsinfravnet.NewInfraVnetClient(hostIfName, contIfName) + } +} + +func AddInfraVnetEndpoint(client *OVSEndpointClient) error { + if client.enableInfraVnet { + return client.infraVnetClient.CreateInfraVnetEndpoint(client.bridgeName) + } + + return nil +} + +func AddInfraEndpointRules(client *OVSEndpointClient, infraIP net.IPNet, hostPort string) error { + if client.enableInfraVnet { + return client.infraVnetClient.CreateInfraVnetRules(client.bridgeName, infraIP, client.hostPrimaryMac, hostPort) + } + + return nil +} + +func DeleteInfraVnetEndpointRules(client *OVSEndpointClient, ep *endpoint, hostPort string) { + if client.enableInfraVnet { + client.infraVnetClient.DeleteInfraVnetRules(client.bridgeName, ep.InfraVnetIP, hostPort) + } +} + +func MoveInfraEndpointToContainerNS(client *OVSEndpointClient, netnsPath string, nsID uintptr) error { + if client.enableInfraVnet { + return client.infraVnetClient.MoveInfraEndpointToContainerNS(netnsPath, nsID) + } + + return nil +} + +func SetupInfraVnetContainerInterface(client *OVSEndpointClient) error { + if client.enableInfraVnet { + return client.infraVnetClient.SetupInfraVnetContainerInterface() + } + + return nil +} + +func ConfigureInfraVnetContainerInterface(client *OVSEndpointClient, infraIP net.IPNet) error { + if client.enableInfraVnet { + return client.infraVnetClient.ConfigureInfraVnetContainerInterface(infraIP) + } + + return nil +} + +func DeleteInfraVnetEndpoint(client *OVSEndpointClient, epID string) error { + if client.enableInfraVnet { + return client.infraVnetClient.DeleteInfraVnetEndpoint() + } + + return nil +} diff --git a/network/ovs_endpoint_snatroute_linux.go b/network/ovs_endpoint_snatroute_linux.go new file mode 100644 index 0000000000..47c9c8d3d5 --- /dev/null +++ b/network/ovs_endpoint_snatroute_linux.go @@ -0,0 +1,78 @@ +package network + +import ( + "fmt" + + "github.com/Azure/azure-container-networking/network/ovssnat" +) + +func NewSnatClient(client *OVSEndpointClient, epInfo *EndpointInfo) { + if client.enableSnatOnHost { + var localIP, snatBridgeIP string + + hostIfName := fmt.Sprintf("%s%s", snatVethInterfacePrefix, epInfo.Id[:7]) + contIfName := fmt.Sprintf("%s%s-2", snatVethInterfacePrefix, epInfo.Id[:7]) + + if _, ok := epInfo.Data[LocalIPKey]; ok { + localIP = epInfo.Data[LocalIPKey].(string) + } + + if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { + snatBridgeIP = epInfo.Data[SnatBridgeIPKey].(string) + } + + client.snatClient = ovssnat.NewSnatClient(hostIfName, contIfName, localIP, snatBridgeIP, epInfo.DNS.Servers) + } +} + +func AddSnatEndpoint(client *OVSEndpointClient) error { + if client.enableSnatOnHost { + return client.snatClient.CreateSnatEndpoint(client.bridgeName) + } + + return nil +} + +func AddSnatEndpointRules(client *OVSEndpointClient) error { + if client.enableSnatOnHost { + if err := client.snatClient.AddPrivateIPBlockRule(); err != nil { + return err + } + + return AddStaticRoute(ovssnat.ImdsIP, client.bridgeName) + } + + return nil +} + +func MoveSnatEndpointToContainerNS(client *OVSEndpointClient, netnsPath string, nsID uintptr) error { + if client.enableSnatOnHost { + return client.snatClient.MoveSnatEndpointToContainerNS(netnsPath, nsID) + } + + return nil +} + +func SetupSnatContainerInterface(client *OVSEndpointClient) error { + if client.enableSnatOnHost { + return client.snatClient.SetupSnatContainerInterface() + } + + return nil +} + +func ConfigureSnatContainerInterface(client *OVSEndpointClient) error { + if client.enableSnatOnHost { + return client.snatClient.ConfigureSnatContainerInterface() + } + + return nil +} + +func DeleteSnatEndpoint(client *OVSEndpointClient) error { + if client.enableSnatOnHost { + return client.snatClient.DeleteSnatEndpoint() + } + + return nil +} diff --git a/network/ovs_endpointclient_linux.go b/network/ovs_endpointclient_linux.go index 7c88e5086e..b580bbd5dd 100644 --- a/network/ovs_endpointclient_linux.go +++ b/network/ovs_endpointclient_linux.go @@ -1,11 +1,13 @@ package network import ( - "fmt" "net" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/network/epcommon" + "github.com/Azure/azure-container-networking/network/ovsinfravnet" + "github.com/Azure/azure-container-networking/network/ovssnat" "github.com/Azure/azure-container-networking/ovsctl" ) @@ -16,16 +18,16 @@ type OVSEndpointClient struct { hostPrimaryMac string containerVethName string containerMac string - snatVethName string - snatBridgeIP string - localIP string + snatClient ovssnat.OVSSnatClient + infraVnetClient ovsinfravnet.OVSInfraVnetClient vlanID int enableSnatOnHost bool + enableInfraVnet bool } const ( - snatVethInterfacePrefix = commonInterfacePrefix + "vint" - azureSnatIfName = "eth1" + snatVethInterfacePrefix = commonInterfacePrefix + "vint" + infraVethInterfacePrefix = commonInterfacePrefix + "vifv" ) func NewOVSEndpointClient( @@ -44,69 +46,34 @@ func NewOVSEndpointClient( containerVethName: containerVethName, vlanID: vlanid, enableSnatOnHost: epInfo.EnableSnatOnHost, + enableInfraVnet: epInfo.EnableInfraVnet, } - if _, ok := epInfo.Data[LocalIPKey]; ok { - client.localIP = epInfo.Data[LocalIPKey].(string) - } - - if _, ok := epInfo.Data[SnatBridgeIPKey]; ok { - client.snatBridgeIP = epInfo.Data[SnatBridgeIPKey].(string) - } + NewInfraVnetClient(client, epInfo.Id[:7]) + NewSnatClient(client, epInfo) return client } func (client *OVSEndpointClient) AddEndpoints(epInfo *EndpointInfo) error { - if err := createEndpoint(client.hostVethName, client.containerVethName); err != nil { + if err := epcommon.CreateEndpoint(client.hostVethName, client.containerVethName); err != nil { return err } containerIf, err := net.InterfaceByName(client.containerVethName) if err != nil { + log.Printf("InterfaceByName returns error for ifname %v with error %v", client.containerVethName, err) return err } client.containerMac = containerIf.HardwareAddr.String() - if client.enableSnatOnHost { - if err := createSnatBridge(client.snatBridgeIP, client.bridgeName); err != nil { - log.Printf("creating snat bridge failed with error %v", err) - return err - } - - if err := addOrDeletePrivateIPBlockRule("A"); err != nil { - log.Printf("addPrivateIPBlockRule failed with error %v", err) - return err - } - - if err := addMasqueradeRule(client.snatBridgeIP); err != nil { - log.Printf("Adding snat rule failed with error %v", err) - return err - } - - if err := addVlanDropRule(); err != nil { - log.Printf("Adding vlan drop rule failed with error %v", err) - return err - } - - if err := addStaticRoute(imdsIP, client.bridgeName); err != nil { - log.Printf("Adding imds static route failed with error %v", err) - return err - } - - hostIfName := fmt.Sprintf("%s%s", snatVethInterfacePrefix, epInfo.Id[:7]) - contIfName := fmt.Sprintf("%s%s-2", snatVethInterfacePrefix, epInfo.Id[:7]) - - if err := createEndpoint(hostIfName, contIfName); err != nil { - return err - } - - if err := netlink.SetLinkMaster(hostIfName, snatBridgeName); err != nil { - return err - } + if err := AddSnatEndpoint(client); err != nil { + return err + } - client.snatVethName = contIfName + if err := AddInfraVnetEndpoint(client); err != nil { + return err } return nil @@ -134,7 +101,7 @@ func (client *OVSEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { // IP SNAT Rule log.Printf("[ovs] Adding IP SNAT rule for egress traffic on %v.", containerPort) - if err := ovsctl.AddIpSnatRule(client.bridgeName, containerPort, client.hostPrimaryMac); err != nil { + if err := ovsctl.AddIpSnatRule(client.bridgeName, containerPort, client.hostPrimaryMac, ""); err != nil { return err } @@ -152,7 +119,11 @@ func (client *OVSEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error { } } - return nil + if err := AddInfraEndpointRules(client, epInfo.InfraVnetIP, hostPort); err != nil { + return err + } + + return AddSnatEndpointRules(client) } func (client *OVSEndpointClient) DeleteEndpointRules(ep *endpoint) { @@ -173,7 +144,7 @@ func (client *OVSEndpointClient) DeleteEndpointRules(ep *endpoint) { ovsctl.DeleteIPSnatRule(client.bridgeName, containerPort) // Delete Arp Reply Rules for container - log.Printf("[ovs] Deleting ARP reply rule for ip %v vlanid %v for container port", ep.IPAddresses[0].IP.String(), ep.VlanID, containerPort) + log.Printf("[ovs] Deleting ARP reply rule for ip %v vlanid %v for container port %v", ep.IPAddresses[0].IP.String(), ep.VlanID, containerPort) ovsctl.DeleteArpReplyRule(client.bridgeName, containerPort, ep.IPAddresses[0].IP, ep.VlanID) // Delete MAC address translation rule. @@ -183,6 +154,8 @@ func (client *OVSEndpointClient) DeleteEndpointRules(ep *endpoint) { // Delete port from ovs bridge log.Printf("[ovs] Deleting interface %v from bridge %v", client.hostVethName, client.bridgeName) ovsctl.DeletePortFromOVS(client.bridgeName, client.hostVethName) + + DeleteInfraVnetEndpointRules(client, ep, hostPort) } func (client *OVSEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error { @@ -192,52 +165,43 @@ func (client *OVSEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo return err } - if client.enableSnatOnHost { - log.Printf("[ovs] Setting link %v netns %v.", client.snatVethName, epInfo.NetNsPath) - if err := netlink.SetLinkNetNs(client.snatVethName, nsID); err != nil { - return err - } + if err := MoveSnatEndpointToContainerNS(client, epInfo.NetNsPath, nsID); err != nil { + return err } - return nil + return MoveInfraEndpointToContainerNS(client, epInfo.NetNsPath, nsID) + } func (client *OVSEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error { - if err := setupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { + if err := epcommon.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil { return err } client.containerVethName = epInfo.IfName - if client.enableSnatOnHost { - if err := setupContainerInterface(client.snatVethName, azureSnatIfName); err != nil { - return err - } - client.snatVethName = azureSnatIfName + if err := SetupSnatContainerInterface(client); err != nil { + return err } - return nil + return SetupInfraVnetContainerInterface(client) } func (client *OVSEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error { - if err := assignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { + if err := epcommon.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil { return err } - if client.enableSnatOnHost { - log.Printf("[ovs] Adding IP address %v to link %v.", client.localIP, client.snatVethName) - ip, intIpAddr, _ := net.ParseCIDR(client.localIP) - if err := netlink.AddIpAddress(client.snatVethName, ip, intIpAddr); err != nil { - return err - } + if err := ConfigureSnatContainerInterface(client); err != nil { + return err } - if err := addRoutes(client.containerVethName, epInfo.Routes); err != nil { + if err := ConfigureInfraVnetContainerInterface(client, epInfo.InfraVnetIP); err != nil { return err } - return nil + return addRoutes(client.containerVethName, epInfo.Routes) } func (client *OVSEndpointClient) DeleteEndpoints(ep *endpoint) error { @@ -248,15 +212,5 @@ func (client *OVSEndpointClient) DeleteEndpoints(ep *endpoint) error { return err } - if client.enableSnatOnHost { - hostIfName := fmt.Sprintf("%s%s", snatVethInterfacePrefix, ep.Id[:7]) - log.Printf("[ovs] Deleting snat veth pair %v.", hostIfName) - err = netlink.DeleteLink(hostIfName) - if err != nil { - log.Printf("[ovs] Failed to delete veth pair %v: %v.", hostIfName, err) - return err - } - } - - return nil + return DeleteInfraVnetEndpoint(client, ep.Id[:7]) } diff --git a/network/ovs_networkclient_linux.go b/network/ovs_networkclient_linux.go index 77d56b50ec..590abef73f 100644 --- a/network/ovs_networkclient_linux.go +++ b/network/ovs_networkclient_linux.go @@ -2,48 +2,28 @@ package network import ( "bytes" - "fmt" - "net" "os" "strings" "github.com/Azure/azure-container-networking/log" - "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/network/epcommon" + "github.com/Azure/azure-container-networking/network/ovssnat" "github.com/Azure/azure-container-networking/ovsctl" - "github.com/Azure/azure-container-networking/platform" ) type OVSNetworkClient struct { - bridgeName string - hostInterfaceName string - snatBridgeIP string - enableSnatOnHost bool + bridgeName string + hostInterfaceName string + snatBridgeIP string + skipAddressesFromBlock []string + enableSnatOnHost bool } const ( - azureSnatVeth0 = "azSnatveth0" - azureSnatVeth1 = "azSnatveth1" - snatBridgeName = "azSnatbr" - imdsIP = "169.254.169.254/32" - ovsConfigFile = "/etc/default/openvswitch-switch" - ovsOpt = "OVS_CTL_OPTS='--delete-bridges'" + ovsConfigFile = "/etc/default/openvswitch-switch" + ovsOpt = "OVS_CTL_OPTS='--delete-bridges'" ) -func getPrivateIPSpace() []string { - privateIPAddresses := []string{"10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"} - return privateIPAddresses -} - -func getFilterChains() []string { - chains := []string{"FORWARD", "INPUT", "OUTPUT"} - return chains -} - -func getFilterchainTarget() []string { - actions := []string{"ACCEPT", "DROP"} - return actions -} - func updateOVSConfig(option string) error { f, err := os.OpenFile(ovsConfigFile, os.O_APPEND|os.O_RDWR, 0666) if err != nil { @@ -75,12 +55,13 @@ func updateOVSConfig(option string) error { return nil } -func NewOVSClient(bridgeName, hostInterfaceName, snatBridgeIP string, enableSnatOnHost bool) *OVSNetworkClient { +func NewOVSClient(bridgeName, hostInterfaceName, snatBridgeIP string, skipAddressesFromBlock []string, enableSnatOnHost bool) *OVSNetworkClient { ovsClient := &OVSNetworkClient{ - bridgeName: bridgeName, - hostInterfaceName: hostInterfaceName, - snatBridgeIP: snatBridgeIP, - enableSnatOnHost: enableSnatOnHost, + bridgeName: bridgeName, + hostInterfaceName: hostInterfaceName, + snatBridgeIP: snatBridgeIP, + skipAddressesFromBlock: skipAddressesFromBlock, + enableSnatOnHost: enableSnatOnHost, } return ovsClient @@ -96,81 +77,16 @@ func (client *OVSNetworkClient) CreateBridge() error { } if client.enableSnatOnHost { - if err := createSnatBridge(client.snatBridgeIP, client.bridgeName); err != nil { + if err := ovssnat.CreateSnatBridge(client.snatBridgeIP, client.bridgeName); err != nil { log.Printf("[net] Creating snat bridge failed with erro %v", err) return err } - if err := addOrDeletePrivateIPBlockRule("A"); err != nil { - log.Printf("addPrivateIPBlockRule failed with error %v", err) - return err - } - - if err := addMasqueradeRule(client.snatBridgeIP); err != nil { + if err := ovssnat.AddMasqueradeRule(client.snatBridgeIP); err != nil { return err } - return addVlanDropRule() - } - - return nil -} - -func addVlanDropRule() error { - cmd := "ebtables -t nat -L PREROUTING" - out, err := platform.ExecuteCommand(cmd) - if err != nil { - log.Printf("Error while listing ebtable rules %v", err) - return err - } - - out = strings.TrimSpace(out) - if strings.Contains(out, "-p 802_1Q -j DROP") { - log.Printf("vlan drop rule already exists") - return nil - } - - cmd = "ebtables -t nat -A PREROUTING -p 802_1Q -j DROP" - log.Printf("Adding ebtable rule to drop vlan traffic on snat bridge %v", cmd) - _, err = platform.ExecuteCommand(cmd) - return err -} - -func addMasqueradeRule(snatBridgeIPWithPrefix string) error { - _, ipNet, _ := net.ParseCIDR(snatBridgeIPWithPrefix) - cmd := fmt.Sprintf("iptables -t nat -C POSTROUTING -s %v -j MASQUERADE", ipNet.String()) - _, err := platform.ExecuteCommand(cmd) - if err == nil { - log.Printf("iptable snat rule already exists") - return nil - } - - cmd = fmt.Sprintf("iptables -t nat -A POSTROUTING -s %v -j MASQUERADE", ipNet.String()) - log.Printf("Adding iptable snat rule %v", cmd) - _, err = platform.ExecuteCommand(cmd) - return err -} - -func deleteMasqueradeRule(interfaceName string) error { - snatIf, err := net.InterfaceByName(interfaceName) - if err != nil { - return err - } - - addrs, _ := snatIf.Addrs() - for _, addr := range addrs { - ipAddr, ipNet, err := net.ParseCIDR(addr.String()) - if err != nil { - log.Printf("error %v", err) - continue - } - - if ipAddr.To4() != nil { - cmd := fmt.Sprintf("iptables -t nat -D POSTROUTING -s %v -j MASQUERADE", ipNet.String()) - log.Printf("Deleting iptable snat rule %v", cmd) - _, err = platform.ExecuteCommand(cmd) - return err - } + return ovssnat.AddVlanDropRule() } return nil @@ -183,188 +99,18 @@ func (client *OVSNetworkClient) DeleteBridge() error { } if client.enableSnatOnHost { - deleteMasqueradeRule(snatBridgeName) - - cmd := "ebtables -t nat -D PREROUTING -p 802_1Q -j DROP" - _, err := platform.ExecuteCommand(cmd) - if err != nil { - log.Printf("Deleting ebtable vlan drop rule failed with error %v", err) - } - - if err := addOrDeletePrivateIPBlockRule("D"); err != nil { - log.Printf("Deleting PrivateIP Block rules failed with error %v", err) - } + ovssnat.DeleteMasqueradeRule() - if err := ovsctl.DeletePortFromOVS(client.bridgeName, azureSnatVeth1); err != nil { - return err - } - - if err := DeleteSnatBridge(); err != nil { + if err := ovssnat.DeleteSnatBridge(client.bridgeName); err != nil { log.Printf("Deleting snat bridge failed with error %v", err) return err } - - return netlink.DeleteLink(azureSnatVeth0) } return nil } -func createSnatBridge(snatBridgeIP string, mainInterface string) error { - _, err := net.InterfaceByName(snatBridgeName) - if err == nil { - log.Printf("Snat Bridge already exists") - return nil - } - - log.Printf("[net] Creating Snat bridge %v.", snatBridgeName) - - link := netlink.BridgeLink{ - LinkInfo: netlink.LinkInfo{ - Type: netlink.LINK_TYPE_BRIDGE, - Name: snatBridgeName, - }, - } - - if err := netlink.AddLink(&link); err != nil { - return err - } - - _, err = net.InterfaceByName(azureSnatVeth0) - if err == nil { - log.Printf("Azure snat veth already exists") - return nil - } - - vethLink := netlink.VEthLink{ - LinkInfo: netlink.LinkInfo{ - Type: netlink.LINK_TYPE_VETH, - Name: azureSnatVeth0, - }, - PeerName: azureSnatVeth1, - } - - err = netlink.AddLink(&vethLink) - if err != nil { - log.Printf("[net] Failed to create veth pair, err:%v.", err) - return err - } - - log.Printf("Assigning %v on snat bridge", snatBridgeIP) - - ip, addr, _ := net.ParseCIDR(snatBridgeIP) - err = netlink.AddIpAddress(snatBridgeName, ip, addr) - if err != nil && !strings.Contains(strings.ToLower(err.Error()), "file exists") { - log.Printf("[net] Failed to add IP address %v: %v.", addr, err) - return err - } - - if err := netlink.SetLinkState(snatBridgeName, true); err != nil { - return err - } - - if err := netlink.SetLinkState(azureSnatVeth0, true); err != nil { - return err - } - - if err := netlink.SetLinkMaster(azureSnatVeth0, snatBridgeName); err != nil { - return err - } - - if err := netlink.SetLinkState(azureSnatVeth1, true); err != nil { - return err - } - - if err := ovsctl.AddPortOnOVSBridge(azureSnatVeth1, mainInterface, 0); err != nil { - return err - } - - return nil -} - -func addOrDeleteFilterRule(action string, ipAddress string, chainName string, target string) error { - option := "i" - - if chainName == "OUTPUT" { - option = "o" - } - - if action != "D" { - cmd := fmt.Sprintf("iptables -t filter -C %v -%v %v -d %v -j %v", chainName, option, snatBridgeName, ipAddress, target) - _, err := platform.ExecuteCommand(cmd) - if err == nil { - log.Printf("Iptable filter for private ipaddr %v on %v chain %v target rule already exists", ipAddress, chainName, target) - return nil - } - } - - cmd := fmt.Sprintf("iptables -t filter -%v %v -%v %v -d %v -j %v", action, chainName, option, snatBridgeName, ipAddress, target) - _, err := platform.ExecuteCommand(cmd) - if err != nil { - log.Printf("Iptable filter %v action for private ipaddr %v on %v chain %v target failed with %v", action, ipAddress, chainName, target, err) - return err - } - - return nil -} - -func addOrDeletePrivateIPBlockRule(action string) error { - privateIPAddresses := getPrivateIPSpace() - chains := getFilterChains() - target := getFilterchainTarget() - - for _, chain := range chains { - if err := addOrDeleteFilterRule(action, "10.0.0.10", chain, target[0]); err != nil { - return err - } - } - - for _, ipAddress := range privateIPAddresses { - if err := addOrDeleteFilterRule(action, ipAddress, chains[0], target[1]); err != nil { - return err - } - - if err := addOrDeleteFilterRule(action, ipAddress, chains[1], target[1]); err != nil { - return err - } - - if err := addOrDeleteFilterRule(action, ipAddress, chains[2], target[1]); err != nil { - return err - } - } - - return nil -} - -func addStaticRoute(ip string, interfaceName string) error { - log.Printf("[ovs] Adding %v static route", ip) - var routes []RouteInfo - _, ipNet, _ := net.ParseCIDR(imdsIP) - gwIP := net.ParseIP("0.0.0.0") - route := RouteInfo{Dst: *ipNet, Gw: gwIP} - routes = append(routes, route) - if err := addRoutes(interfaceName, routes); err != nil { - if err != nil && !strings.Contains(strings.ToLower(err.Error()), "file exists") { - log.Printf("addroutes failed with error %v", err) - return err - } - } - - return nil -} - -func DeleteSnatBridge() error { - // Delete the bridge. - err := netlink.DeleteLink(snatBridgeName) - if err != nil { - log.Printf("[net] Failed to delete bridge %v, err:%v.", snatBridgeName, err) - } - - return err -} - func (client *OVSNetworkClient) AddL2Rules(extIf *externalInterface) error { - //primary := extIf.IPAddresses[0].IP.String() mac := extIf.MacAddress.String() macHex := strings.Replace(mac, ":", "", -1) @@ -385,7 +131,11 @@ func (client *OVSNetworkClient) AddL2Rules(extIf *externalInterface) error { } if client.enableSnatOnHost { - addStaticRoute(imdsIP, client.bridgeName) + if err := epcommon.AddOrDeletePrivateIPBlockRule(ovssnat.SnatBridgeName, client.skipAddressesFromBlock, "A"); err != nil { + return err + } + + return AddStaticRoute(ovssnat.ImdsIP, client.bridgeName) } return nil @@ -393,6 +143,12 @@ func (client *OVSNetworkClient) AddL2Rules(extIf *externalInterface) error { func (client *OVSNetworkClient) DeleteL2Rules(extIf *externalInterface) { ovsctl.DeletePortFromOVS(client.bridgeName, client.hostInterfaceName) + + if client.enableSnatOnHost { + if err := epcommon.AddOrDeletePrivateIPBlockRule(ovssnat.SnatBridgeName, client.skipAddressesFromBlock, "D"); err != nil { + log.Printf("Deleting PrivateIPBlock rules failed with error %v", err) + } + } } func (client *OVSNetworkClient) SetBridgeMasterToHostInterface() error { diff --git a/network/ovsinfravnet/infravnet_linux.go b/network/ovsinfravnet/infravnet_linux.go new file mode 100644 index 0000000000..ee4dfd86f3 --- /dev/null +++ b/network/ovsinfravnet/infravnet_linux.go @@ -0,0 +1,131 @@ +package ovsinfravnet + +import ( + "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/netlink" + + "net" + + "github.com/Azure/azure-container-networking/network/epcommon" + "github.com/Azure/azure-container-networking/ovsctl" +) + +const ( + azureInfraIfName = "eth2" +) + +type OVSInfraVnetClient struct { + hostInfraVethName string + ContainerInfraVethName string + containerInfraMac string +} + +func NewInfraVnetClient(hostIfName string, contIfName string) OVSInfraVnetClient { + infraVnetClient := OVSInfraVnetClient{} + infraVnetClient.hostInfraVethName = hostIfName + infraVnetClient.ContainerInfraVethName = contIfName + + log.Printf("Initialize new infravnet client %+v", infraVnetClient) + + return infraVnetClient +} + +func (client *OVSInfraVnetClient) CreateInfraVnetEndpoint(bridgeName string) error { + if err := epcommon.CreateEndpoint(client.hostInfraVethName, client.ContainerInfraVethName); err != nil { + log.Printf("Creating infraep failed with error %v", err) + return err + } + + log.Printf("[ovs] Adding port %v master %v.", client.hostInfraVethName, bridgeName) + if err := ovsctl.AddPortOnOVSBridge(client.hostInfraVethName, bridgeName, 0); err != nil { + log.Printf("Adding infraveth to ovsbr failed with error %v", err) + return err + } + + infraContainerIf, err := net.InterfaceByName(client.ContainerInfraVethName) + if err != nil { + log.Printf("InterfaceByName returns error for ifname %v with error %v", client.ContainerInfraVethName, err) + return err + } + + client.containerInfraMac = infraContainerIf.HardwareAddr.String() + + return nil +} + +func (client *OVSInfraVnetClient) CreateInfraVnetRules( + bridgeName string, + infraIP net.IPNet, + hostPrimaryMac string, + hostPort string) error { + + infraContainerPort, err := ovsctl.GetOVSPortNumber(client.hostInfraVethName) + if err != nil { + log.Printf("[ovs] Get ofport failed with error %v", err) + return err + } + + if err := ovsctl.AddIpSnatRule(bridgeName, infraContainerPort, hostPrimaryMac, hostPort); err != nil { + log.Printf("[ovs] AddIpSnatRule failed with error %v", err) + return err + } + + if err := ovsctl.AddMacDnatRule(bridgeName, hostPort, infraIP.IP, client.containerInfraMac, 0); err != nil { + log.Printf("[ovs] AddMacDnatRule failed with error %v", err) + return err + } + + return nil +} + +func (client *OVSInfraVnetClient) MoveInfraEndpointToContainerNS(netnsPath string, nsID uintptr) error { + log.Printf("[ovs] Setting link %v netns %v.", client.ContainerInfraVethName, netnsPath) + return netlink.SetLinkNetNs(client.ContainerInfraVethName, nsID) +} + +func (client *OVSInfraVnetClient) SetupInfraVnetContainerInterface() error { + if err := epcommon.SetupContainerInterface(client.ContainerInfraVethName, azureInfraIfName); err != nil { + return err + } + + client.ContainerInfraVethName = azureInfraIfName + + return nil +} + +func (client *OVSInfraVnetClient) ConfigureInfraVnetContainerInterface(infraIP net.IPNet) error { + log.Printf("[ovs] Adding IP address %v to link %v.", infraIP.String(), client.ContainerInfraVethName) + return netlink.AddIpAddress(client.ContainerInfraVethName, infraIP.IP, &infraIP) +} + +func (client *OVSInfraVnetClient) DeleteInfraVnetRules( + bridgeName string, + infraIP net.IPNet, + hostPort string) { + + log.Printf("[ovs] Deleting MAC DNAT rule for infravnet IP address %v", infraIP.IP.String()) + ovsctl.DeleteMacDnatRule(bridgeName, hostPort, infraIP.IP, 0) + + log.Printf("[ovs] Get ovs port for infravnet interface %v.", client.hostInfraVethName) + infraContainerPort, err := ovsctl.GetOVSPortNumber(client.hostInfraVethName) + if err != nil { + log.Printf("[ovs] Get infravnet portnum failed with error %v", err) + } + + log.Printf("[ovs] Deleting IP SNAT for infravnet port %v", infraContainerPort) + ovsctl.DeleteIPSnatRule(bridgeName, infraContainerPort) + + log.Printf("[ovs] Deleting infravnet interface %v from bridge %v", client.hostInfraVethName, bridgeName) + ovsctl.DeletePortFromOVS(bridgeName, client.hostInfraVethName) +} + +func (client *OVSInfraVnetClient) DeleteInfraVnetEndpoint() error { + log.Printf("[ovs] Deleting Infra veth pair %v.", client.hostInfraVethName) + err := netlink.DeleteLink(client.hostInfraVethName) + if err != nil { + log.Printf("[ovs] Failed to delete veth pair %v: %v.", client.hostInfraVethName, err) + return err + } + + return nil +} diff --git a/network/ovsinfravnet/infravnet_windows.go b/network/ovsinfravnet/infravnet_windows.go new file mode 100644 index 0000000000..512fc9de77 --- /dev/null +++ b/network/ovsinfravnet/infravnet_windows.go @@ -0,0 +1 @@ +package ovsinfravnet diff --git a/network/ovssnat/ovssnat_linux.go b/network/ovssnat/ovssnat_linux.go new file mode 100644 index 0000000000..a6e9ba7d42 --- /dev/null +++ b/network/ovssnat/ovssnat_linux.go @@ -0,0 +1,267 @@ +package ovssnat + +import ( + "fmt" + "net" + "strings" + + "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/netlink" + "github.com/Azure/azure-container-networking/network/epcommon" + "github.com/Azure/azure-container-networking/ovsctl" + "github.com/Azure/azure-container-networking/platform" +) + +const ( + azureSnatVeth0 = "azSnatveth0" + azureSnatVeth1 = "azSnatveth1" + azureSnatIfName = "eth1" + SnatBridgeName = "azSnatbr" + ImdsIP = "169.254.169.254/32" +) + +type OVSSnatClient struct { + hostSnatVethName string + containerSnatVethName string + localIP string + snatBridgeIP string + SkipAddressesFromBlock []string +} + +func NewSnatClient(hostIfName string, contIfName string, localIP string, snatBridgeIP string, skipAddressesFromBlock []string) OVSSnatClient { + log.Printf("Initialize new snat client") + snatClient := OVSSnatClient{} + snatClient.hostSnatVethName = hostIfName + snatClient.containerSnatVethName = contIfName + snatClient.localIP = localIP + snatClient.snatBridgeIP = snatBridgeIP + + for _, address := range skipAddressesFromBlock { + snatClient.SkipAddressesFromBlock = append(snatClient.SkipAddressesFromBlock, address) + } + + log.Printf("Initialize new snat client %+v", snatClient) + + return snatClient +} + +func (client *OVSSnatClient) CreateSnatEndpoint(bridgeName string) error { + if err := CreateSnatBridge(client.snatBridgeIP, bridgeName); err != nil { + log.Printf("creating snat bridge failed with error %v", err) + return err + } + + if err := AddMasqueradeRule(client.snatBridgeIP); err != nil { + log.Printf("Adding snat rule failed with error %v", err) + return err + } + + if err := AddVlanDropRule(); err != nil { + log.Printf("Adding vlan drop rule failed with error %v", err) + return err + } + + if err := epcommon.CreateEndpoint(client.hostSnatVethName, client.containerSnatVethName); err != nil { + log.Printf("Creating Snat Endpoint failed with error %v", err) + return err + } + + return netlink.SetLinkMaster(client.hostSnatVethName, SnatBridgeName) +} + +func (client *OVSSnatClient) AddPrivateIPBlockRule() error { + if err := epcommon.AddOrDeletePrivateIPBlockRule(SnatBridgeName, client.SkipAddressesFromBlock, "A"); err != nil { + log.Printf("AddPrivateIPBlockRule failed with error %v", err) + return err + } + + return nil +} + +func (client *OVSSnatClient) MoveSnatEndpointToContainerNS(netnsPath string, nsID uintptr) error { + log.Printf("[ovs] Setting link %v netns %v.", client.containerSnatVethName, netnsPath) + return netlink.SetLinkNetNs(client.containerSnatVethName, nsID) +} + +func (client *OVSSnatClient) SetupSnatContainerInterface() error { + if err := epcommon.SetupContainerInterface(client.containerSnatVethName, azureSnatIfName); err != nil { + return err + } + + client.containerSnatVethName = azureSnatIfName + + return nil +} + +func (client *OVSSnatClient) ConfigureSnatContainerInterface() error { + log.Printf("[ovs] Adding IP address %v to link %v.", client.localIP, client.containerSnatVethName) + ip, intIpAddr, _ := net.ParseCIDR(client.localIP) + return netlink.AddIpAddress(client.containerSnatVethName, ip, intIpAddr) +} + +func (client *OVSSnatClient) DeleteSnatEndpoint() error { + log.Printf("[ovs] Deleting snat veth pair %v.", client.hostSnatVethName) + err := netlink.DeleteLink(client.hostSnatVethName) + if err != nil { + log.Printf("[ovs] Failed to delete veth pair %v: %v.", client.hostSnatVethName, err) + return err + } + + return nil +} + +func CreateSnatBridge(snatBridgeIP string, mainInterface string) error { + _, err := net.InterfaceByName(SnatBridgeName) + if err == nil { + log.Printf("Snat Bridge already exists") + return nil + } + + log.Printf("[net] Creating Snat bridge %v.", SnatBridgeName) + + link := netlink.BridgeLink{ + LinkInfo: netlink.LinkInfo{ + Type: netlink.LINK_TYPE_BRIDGE, + Name: SnatBridgeName, + }, + } + + if err := netlink.AddLink(&link); err != nil { + return err + } + + _, err = net.InterfaceByName(azureSnatVeth0) + if err == nil { + log.Printf("Azure snat veth already exists") + return nil + } + + vethLink := netlink.VEthLink{ + LinkInfo: netlink.LinkInfo{ + Type: netlink.LINK_TYPE_VETH, + Name: azureSnatVeth0, + }, + PeerName: azureSnatVeth1, + } + + err = netlink.AddLink(&vethLink) + if err != nil { + log.Printf("[net] Failed to create veth pair, err:%v.", err) + return err + } + + log.Printf("Assigning %v on snat bridge", snatBridgeIP) + + ip, addr, _ := net.ParseCIDR(snatBridgeIP) + err = netlink.AddIpAddress(SnatBridgeName, ip, addr) + if err != nil && !strings.Contains(strings.ToLower(err.Error()), "file exists") { + log.Printf("[net] Failed to add IP address %v: %v.", addr, err) + return err + } + + if err := netlink.SetLinkState(SnatBridgeName, true); err != nil { + return err + } + + if err := netlink.SetLinkState(azureSnatVeth0, true); err != nil { + return err + } + + if err := netlink.SetLinkMaster(azureSnatVeth0, SnatBridgeName); err != nil { + return err + } + + if err := netlink.SetLinkState(azureSnatVeth1, true); err != nil { + return err + } + + if err := ovsctl.AddPortOnOVSBridge(azureSnatVeth1, mainInterface, 0); err != nil { + return err + } + + return nil +} + +func DeleteSnatBridge(bridgeName string) error { + cmd := "ebtables -t nat -D PREROUTING -p 802_1Q -j DROP" + _, err := platform.ExecuteCommand(cmd) + if err != nil { + log.Printf("Deleting ebtable vlan drop rule failed with error %v", err) + } + + if err = ovsctl.DeletePortFromOVS(bridgeName, azureSnatVeth1); err != nil { + log.Printf("Deleting snatveth from ovs failed with error %v", err) + } + + if err = netlink.DeleteLink(azureSnatVeth0); err != nil { + log.Printf("Deleting host snatveth failed with error %v", err) + } + + // Delete the bridge. + err = netlink.DeleteLink(SnatBridgeName) + if err != nil { + log.Printf("[net] Failed to delete bridge %v, err:%v.", SnatBridgeName, err) + } + + return err +} + +func AddMasqueradeRule(snatBridgeIPWithPrefix string) error { + _, ipNet, _ := net.ParseCIDR(snatBridgeIPWithPrefix) + cmd := fmt.Sprintf("iptables -t nat -C POSTROUTING -s %v -j MASQUERADE", ipNet.String()) + _, err := platform.ExecuteCommand(cmd) + if err == nil { + log.Printf("iptable snat rule already exists") + return nil + } + + cmd = fmt.Sprintf("iptables -t nat -A POSTROUTING -s %v -j MASQUERADE", ipNet.String()) + log.Printf("Adding iptable snat rule %v", cmd) + _, err = platform.ExecuteCommand(cmd) + return err +} + +func DeleteMasqueradeRule() error { + snatIf, err := net.InterfaceByName(SnatBridgeName) + if err != nil { + return err + } + + addrs, _ := snatIf.Addrs() + for _, addr := range addrs { + ipAddr, ipNet, err := net.ParseCIDR(addr.String()) + if err != nil { + log.Printf("error %v", err) + continue + } + + if ipAddr.To4() != nil { + cmd := fmt.Sprintf("iptables -t nat -D POSTROUTING -s %v -j MASQUERADE", ipNet.String()) + log.Printf("Deleting iptable snat rule %v", cmd) + _, err = platform.ExecuteCommand(cmd) + return err + } + } + + return nil +} + +func AddVlanDropRule() error { + cmd := "ebtables -t nat -L PREROUTING" + out, err := platform.ExecuteCommand(cmd) + if err != nil { + log.Printf("Error while listing ebtable rules %v", err) + return err + } + + out = strings.TrimSpace(out) + if strings.Contains(out, "-p 802_1Q -j DROP") { + log.Printf("vlan drop rule already exists") + return nil + } + + cmd = "ebtables -t nat -A PREROUTING -p 802_1Q -j DROP" + log.Printf("Adding ebtable rule to drop vlan traffic on snat bridge %v", cmd) + _, err = platform.ExecuteCommand(cmd) + return err +} diff --git a/network/ovssnat/ovssnat_windows.go b/network/ovssnat/ovssnat_windows.go new file mode 100644 index 0000000000..4978a6a744 --- /dev/null +++ b/network/ovssnat/ovssnat_windows.go @@ -0,0 +1 @@ +package ovssnat diff --git a/ovsctl/ovsctl.go b/ovsctl/ovsctl.go index 7e777121a3..5186fbd610 100644 --- a/ovsctl/ovsctl.go +++ b/ovsctl/ovsctl.go @@ -91,9 +91,13 @@ func AddArpSnatRule(bridgeName string, mac string, macHex string, ofport string) return nil } -func AddIpSnatRule(bridgeName string, port string, mac string) error { - cmd := fmt.Sprintf("ovs-ofctl add-flow %v priority=20,ip,in_port=%s,vlan_tci=0,actions=mod_dl_src:%s,strip_vlan,normal", - bridgeName, port, mac) +func AddIpSnatRule(bridgeName string, port string, mac string, outport string) error { + if outport == "" { + outport = "normal" + } + + cmd := fmt.Sprintf("ovs-ofctl add-flow %v priority=20,ip,in_port=%s,vlan_tci=0,actions=mod_dl_src:%s,strip_vlan,%v", + bridgeName, port, mac, outport) _, err := platform.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Adding IP SNAT rule failed with error %v", err) @@ -174,8 +178,15 @@ func AddArpReplyRule(bridgeName string, port string, ip net.IP, mac string, vlan } func AddMacDnatRule(bridgeName string, port string, ip net.IP, mac string, vlanid int) error { - cmd := fmt.Sprintf("ovs-ofctl add-flow %s ip,nw_dst=%s,dl_vlan=%v,in_port=%s,actions=mod_dl_dst:%s,normal", - bridgeName, ip.String(), vlanid, port, mac) + var cmd string + + if vlanid != 0 { + cmd = fmt.Sprintf("ovs-ofctl add-flow %s ip,nw_dst=%s,dl_vlan=%v,in_port=%s,actions=mod_dl_dst:%s,normal", + bridgeName, ip.String(), vlanid, port, mac) + } else { + cmd = fmt.Sprintf("ovs-ofctl add-flow %s ip,nw_dst=%s,in_port=%s,actions=mod_dl_dst:%s,normal", + bridgeName, ip.String(), port, mac) + } _, err := platform.ExecuteCommand(cmd) if err != nil { log.Printf("[ovs] Adding MAC DNAT rule failed with error %v", err) @@ -211,8 +222,16 @@ func DeleteIPSnatRule(bridgeName string, port string) { } func DeleteMacDnatRule(bridgeName string, port string, ip net.IP, vlanid int) { - cmd := fmt.Sprintf("ovs-ofctl del-flows %s ip,nw_dst=%s,dl_vlan=%v,in_port=%s", - bridgeName, ip.String(), vlanid, port) + var cmd string + + if vlanid != 0 { + cmd = fmt.Sprintf("ovs-ofctl del-flows %s ip,nw_dst=%s,dl_vlan=%v,in_port=%s", + bridgeName, ip.String(), vlanid, port) + } else { + cmd = fmt.Sprintf("ovs-ofctl del-flows %s ip,nw_dst=%s,in_port=%s", + bridgeName, ip.String(), port) + } + _, err := platform.ExecuteCommand(cmd) if err != nil { log.Printf("[net] Deleting MAC DNAT rule failed with error %v", err)