Implement multitenancy for windows (#259)

* Implement multitenancy for windows

cni/network/network.go

@@ -131,6 +131,31 @@ func GetEndpointID(args *cniSkel.CmdArgs) string {
 	return infraEpId
 }
 
+// getPodInfo returns POD info by parsing the CNI args.
+func (plugin *netPlugin) getPodInfo(args string) (string, string, error) {
+	podCfg, err := cni.ParseCniArgs(args)
+	if err != nil {
+		log.Printf("Error while parsing CNI Args %v", err)
+		return "", "", err
+	}
+
+	k8sNamespace := string(podCfg.K8S_POD_NAMESPACE)
+	if len(k8sNamespace) == 0 {
+		errMsg := "Pod Namespace not specified in CNI Args"
+		log.Printf(errMsg)
+		return "", "", plugin.Errorf(errMsg)
+	}
+
+	k8sPodName := string(podCfg.K8S_POD_NAME)
+	if len(k8sPodName) == 0 {
+		errMsg := "Pod Name not specified in CNI Args"
+		log.Printf(errMsg)
+		return "", "", plugin.Errorf(errMsg)
+	}
+
+	return k8sPodName, k8sNamespace, nil
+}
+
 //
 // CNI implementation
 // https://github.com/containernetworking/cni/blob/master/SPEC.md
@@ -192,26 +217,11 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 	}()
 
 	// Parse Pod arguments.
-	podCfg, err := cni.ParseCniArgs(args.Args)
+	k8sPodName, k8sNamespace, err := plugin.getPodInfo(args.Args)
 	if err != nil {
-		log.Printf("Error while parsing CNI Args %v", err)
 		return err
 	}
 
-	k8sNamespace := string(podCfg.K8S_POD_NAMESPACE)
-	if len(k8sNamespace) == 0 {
-		errMsg := "Pod Namespace not specified in CNI Args"
-		log.Printf(errMsg)
-		return plugin.Errorf(errMsg)
-	}
-
-	k8sPodName := string(podCfg.K8S_POD_NAME)
-	if len(k8sPodName) == 0 {
-		errMsg := "Pod Name not specified in CNI Args"
-		log.Printf(errMsg)
-		return plugin.Errorf(errMsg)
-	}
-
 	k8sContainerID := args.ContainerID
 	if len(k8sContainerID) == 0 {
 		errMsg := "Container ID not specified in CNI Args"
@@ -234,10 +244,6 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 		}
 	}
 
-	// Initialize values from network config.
-	networkId := nwCfg.Name
-	endpointId := GetEndpointID(args)
-
 	result, cnsNetworkConfig, subnetPrefix, azIpamResult, err = GetMultiTenancyCNIResult(enableInfraVnet, nwCfg, plugin, k8sPodName, k8sNamespace, args.IfName)
 	if err != nil {
 		log.Printf("GetMultiTenancyCNIResult failed with error %v", err)
@@ -252,6 +258,15 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 
 	log.Printf("Result from multitenancy %+v", result)
 
+	// Initialize values from network config.
+	networkId, err := getNetworkName(k8sPodName, k8sNamespace, args.IfName, nwCfg)
+	if err != nil {
+		log.Printf("[cni-net] Failed to extract network name from network config. error: %v", err)
+		return err
+	}
+
+	endpointId := GetEndpointID(args)
+
 	policies := cni.GetPoliciesFromNwCfg(nwCfg.AdditionalArgs)
 
 	// Check whether the network already exists.
@@ -265,13 +280,15 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 		 */
 		epInfo, _ := plugin.nm.GetEndpointInfo(networkId, endpointId)
 		if epInfo != nil {
-			result, err = handleConsecutiveAdd(args.ContainerID, endpointId, nwInfo, nwCfg)
-			if err != nil {
-				log.Printf("handleConsecutiveAdd failed with error %v", err)
-				return err
+			resultConsAdd, errConsAdd := handleConsecutiveAdd(args.ContainerID, endpointId, nwInfo, nwCfg)
+			if errConsAdd != nil {
+				log.Printf("handleConsecutiveAdd failed with error %v", errConsAdd)
+				result = resultConsAdd
+				return errConsAdd
 			}
 
-			if result != nil {
+			if resultConsAdd != nil {
+				result = resultConsAdd
 				return nil
 			}
 		}
@@ -335,11 +352,14 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 		}
 
 		log.Printf("[cni-net] nwDNSInfo: %v", nwDNSInfo)
+		// Update subnet prefix for multi-tenant scenario
+		updateSubnetPrefix(cnsNetworkConfig, &subnetPrefix)
 
 		// Create the network.
 		nwInfo := network.NetworkInfo{
-			Id:   networkId,
-			Mode: nwCfg.Mode,
+			Id:           networkId,
+			Mode:         nwCfg.Mode,
+			MasterIfName: masterIfName,
 			Subnets: []network.SubnetInfo{
 				network.SubnetInfo{
 					Family:  platform.AfINET,
@@ -492,8 +512,18 @@ func (plugin *netPlugin) Get(args *cniSkel.CmdArgs) error {
 
 	log.Printf("[cni-net] Read network configuration %+v.", nwCfg)
 
+	// Parse Pod arguments.
+	k8sPodName, k8sNamespace, err := plugin.getPodInfo(args.Args)
+	if err != nil {
+		return err
+	}
+
 	// Initialize values from network config.
-	networkId := nwCfg.Name
+	networkId, err := getNetworkName(k8sPodName, k8sNamespace, args.IfName, nwCfg)
+	if err != nil {
+		log.Printf("[cni-net] Failed to extract network name from network config. error: %v", err)
+	}
+
 	endpointId := GetEndpointID(args)
 
 	// Query the network.
@@ -552,8 +582,18 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error {
 
 	log.Printf("[cni-net] Read network configuration %+v.", nwCfg)
 
+	// Parse Pod arguments.
+	k8sPodName, k8sNamespace, err := plugin.getPodInfo(args.Args)
+	if err != nil {
+		return err
+	}
+
 	// Initialize values from network config.
-	networkId := nwCfg.Name
+	networkId, err := getNetworkName(k8sPodName, k8sNamespace, args.IfName, nwCfg)
+	if err != nil {
+		log.Printf("[cni-net] Failed to extract network name from network config. error: %v", err)
+	}
+
 	endpointId := GetEndpointID(args)
 
 	// Query the network.

cni/network/network_linux.go

@@ -108,3 +108,10 @@ func getEndpointDNSSettings(nwCfg *cni.NetworkConfig, result *cniTypesCurr.Resul
 func getPoliciesFromRuntimeCfg(nwCfg *cni.NetworkConfig) []policy.Policy {
 	return nil
 }
+
+func updateSubnetPrefix(cnsNetworkConfig *cns.GetNetworkContainerResponse, subnetPrefix *net.IPNet) {
+}
+
+func getNetworkName(podName, podNs, ifName string, nwCfg *cni.NetworkConfig) (string, error) {
+	return nwCfg.Name, nil
+}

cni/network/network_windows.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
+	"strconv"
 	"strings"
 
 	"github.com/Azure/azure-container-networking/cni"
@@ -70,14 +71,59 @@ func addInfraRoutes(azIpamResult *cniTypesCurr.Result, result *cniTypesCurr.Resu
 }
 
 func setNetworkOptions(cnsNwConfig *cns.GetNetworkContainerResponse, nwInfo *network.NetworkInfo) {
+	if cnsNwConfig != nil && cnsNwConfig.MultiTenancyInfo.ID != 0 {
+		log.Printf("Setting Network Options")
+		vlanMap := make(map[string]interface{})
+		vlanMap[network.VlanIDKey] = strconv.Itoa(cnsNwConfig.MultiTenancyInfo.ID)
+		nwInfo.Options[dockerNetworkOption] = vlanMap
+	}
 }
 
 func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *network.EndpointInfo, vethName string) {
+	if cnsNwConfig != nil && cnsNwConfig.MultiTenancyInfo.ID != 0 {
+		log.Printf("Setting Endpoint Options")
+		var cnetAddressMap []string
+		for _, ipSubnet := range cnsNwConfig.CnetAddressSpace {
+			cnetAddressMap = append(cnetAddressMap, ipSubnet.IPAddress+"/"+strconv.Itoa(int(ipSubnet.PrefixLength)))
+		}
+		epInfo.Data[network.CnetAddressSpace] = cnetAddressMap
+	}
 }
 
 func addSnatInterface(nwCfg *cni.NetworkConfig, result *cniTypesCurr.Result) {
 }
 
+func updateSubnetPrefix(cnsNwConfig *cns.GetNetworkContainerResponse, subnetPrefix *net.IPNet) {
+	if cnsNwConfig != nil && cnsNwConfig.MultiTenancyInfo.ID != 0 {
+		ipconfig := cnsNwConfig.IPConfiguration
+		ipAddr := net.ParseIP(ipconfig.IPSubnet.IPAddress)
+
+		if ipAddr.To4() != nil {
+			*subnetPrefix = net.IPNet{IP: ipAddr, Mask: net.CIDRMask(int(ipconfig.IPSubnet.PrefixLength), 32)}
+		} else {
+			*subnetPrefix = net.IPNet{IP: ipAddr, Mask: net.CIDRMask(int(ipconfig.IPSubnet.PrefixLength), 128)}
+		}
+
+		subnetPrefix.IP = subnetPrefix.IP.Mask(subnetPrefix.Mask)
+		log.Printf("Updated subnetPrefix: %s", subnetPrefix.String())
+	}
+}
+
+func getNetworkName(podName, podNs, ifName string, nwCfg *cni.NetworkConfig) (string, error) {
+	if nwCfg.MultiTenancy {
+		_, cnsNetworkConfig, _, err := getContainerNetworkConfiguration(nwCfg, "", podName, podNs, ifName)
+		if err != nil {
+			log.Printf("GetContainerNetworkConfiguration failed for podname %v namespace %v with error %v", podName, podNs, err)
+			return "", err
+		}
+
+		networkName := fmt.Sprintf("%s-vlanid%v", nwCfg.Name, cnsNetworkConfig.MultiTenancyInfo.ID)
+		return networkName, nil
+	}
+
+	return nwCfg.Name, nil
+}
+
 func setupInfraVnetRoutingForMultitenancy(
 	nwCfg *cni.NetworkConfig,
 	azIpamResult *cniTypesCurr.Result,

network/endpoint_windows.go

@@ -44,6 +44,14 @@ func ConstructEndpointID(containerID string, netNsPath string, ifName string) (s
 
 // newEndpointImpl creates a new endpoint in the network.
 func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) {
+	var vlanid int
+
+	if epInfo.Data != nil {
+		if _, ok := epInfo.Data[VlanIDKey]; ok {
+			vlanid = epInfo.Data[VlanIDKey].(int)
+		}
+	}
+
 	// Get Infrastructure containerID. Handle ADD calls for workload container.
 	var err error
 	infraEpName, _ := ConstructEndpointID(epInfo.ContainerID, epInfo.NetNsPath, epInfo.IfName)
@@ -53,7 +61,35 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) {
 		VirtualNetwork: nw.HnsId,
 		DNSSuffix:      epInfo.DNS.Suffix,
 		DNSServerList:  strings.Join(epInfo.DNS.Servers, ","),
-		Policies:       policy.SerializePolicies(policy.EndpointPolicy, epInfo.Policies),
+	}
+
+	// Set outbound NAT policy
+	outBoundNatPolicy := hcsshim.OutboundNatPolicy{}
+	outBoundNatPolicy.Policy.Type = hcsshim.OutboundNat
+
+	exceptionList, err := policy.GetOutBoundNatExceptionList(epInfo.Policies)
+	if err != nil {
+		log.Printf("[net] Failed to parse outbound NAT policy %v", err)
+		return nil, err
+	}
+
+	if exceptionList != nil {
+		for _, ipAddress := range exceptionList {
+			outBoundNatPolicy.Exceptions = append(outBoundNatPolicy.Exceptions, ipAddress)
+		}
+	}
+
+	if epInfo.Data[CnetAddressSpace] != nil {
+		if cnetAddressSpace := epInfo.Data[CnetAddressSpace].([]string); cnetAddressSpace != nil {
+			for _, ipAddress := range cnetAddressSpace {
+				outBoundNatPolicy.Exceptions = append(outBoundNatPolicy.Exceptions, ipAddress)
+			}
+		}
+	}
+
+	if outBoundNatPolicy.Exceptions != nil {
+		serializedOutboundNatPolicy, _ := json.Marshal(outBoundNatPolicy)
+		hnsEndpoint.Policies = append(hnsEndpoint.Policies, serializedOutboundNatPolicy)
 	}
 
 	// HNS currently supports only one IP address per endpoint.
@@ -96,13 +132,15 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) {
 
 	// Create the endpoint object.
 	ep := &endpoint{
-		Id:          infraEpName,
-		HnsId:       hnsResponse.Id,
-		SandboxKey:  epInfo.ContainerID,
-		IfName:      epInfo.IfName,
-		IPAddresses: epInfo.IPAddresses,
-		Gateways:    []net.IP{net.ParseIP(hnsResponse.GatewayAddress)},
-		DNS:         epInfo.DNS,
+		Id:               infraEpName,
+		HnsId:            hnsResponse.Id,
+		SandboxKey:       epInfo.ContainerID,
+		IfName:           epInfo.IfName,
+		IPAddresses:      epInfo.IPAddresses,
+		Gateways:         []net.IP{net.ParseIP(hnsResponse.GatewayAddress)},
+		DNS:              epInfo.DNS,
+		VlanID:           vlanid,
+		EnableSnatOnHost: epInfo.EnableSnatOnHost,
 	}
 
 	for _, route := range epInfo.Routes {

network/manager.go

@@ -15,8 +15,9 @@ import (
 
 const (
 	// Network store key.
-	storeKey  = "Network"
-	VlanIDKey = "VlanID"
+	storeKey    = "Network"
+	VlanIDKey   = "VlanID"
+	genericData = "com.docker.network.generic"
 )
 
 type NetworkClient interface {

network/network.go

@@ -5,6 +5,7 @@ package network
 
 import (
 	"net"
+	"strings"
 
 	"github.com/Azure/azure-container-networking/log"
 	"github.com/Azure/azure-container-networking/network/policy"
@@ -46,6 +47,7 @@ type network struct {
 
 // NetworkInfo contains read-only information about a container network.
 type NetworkInfo struct {
+	MasterIfName     string
 	Id               string
 	Mode             string
 	Subnets          []SubnetInfo
@@ -121,6 +123,16 @@ func (nm *networkManager) findExternalInterfaceBySubnet(subnet string) *external
 	return nil
 }
 
+// FindExternalInterfaceByName finds an external interface by name.
+func (nm *networkManager) findExternalInterfaceByName(ifName string) *externalInterface {
+	extIf, exists := nm.ExternalInterfaces[ifName]
+	if exists && extIf != nil {
+		return extIf
+	}
+
+	return nil
+}
+
 // NewNetwork creates a new container network.
 func (nm *networkManager) newNetwork(nwInfo *NetworkInfo) (*network, error) {
 	var nw *network
@@ -138,8 +150,14 @@ func (nm *networkManager) newNetwork(nwInfo *NetworkInfo) (*network, error) {
 		nwInfo.Mode = opModeDefault
 	}
 
-	// Find the external interface for this subnet.
-	extIf := nm.findExternalInterfaceBySubnet(nwInfo.Subnets[0].Prefix.String())
+	// If the master interface name is provided, find the external interface by name
+	// else use subnet to to find the interface
+	var extIf *externalInterface
+	if len(strings.TrimSpace(nwInfo.MasterIfName)) > 0 {
+		extIf = nm.findExternalInterfaceByName(nwInfo.MasterIfName)
+	} else {
+		extIf = nm.findExternalInterfaceBySubnet(nwInfo.Subnets[0].Prefix.String())
+	}
 	if extIf == nil {
 		err = errSubnetNotFound
 		return nil, err

network/network_linux.go

@@ -21,8 +21,6 @@ const (
 	// Virtual MAC address used by Azure VNET.
 	virtualMacAddress = "12:34:56:78:9a:bc"
 
-	genericData = "com.docker.network.generic"
-
 	SnatBridgeIPKey = "snatBridgeIP"
 
 	LocalIPKey = "localIP"