diff --git a/Makefile b/Makefile index fb66756ffc..2dce8cc90b 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,7 @@ CNSFILES = \ $(wildcard cns/dockerclient/*.go) \ $(wildcard cns/imdsclient/*.go) \ $(wildcard cns/ipamclient/*.go) \ + $(wildcard cns/hnsclient/*.go) \ $(wildcard cns/restserver/*.go) \ $(wildcard cns/routes/*.go) \ $(wildcard cns/service/*.go) \ diff --git a/cni/network/multitenancy.go b/cni/network/multitenancy.go index a7b6ebc707..16728e252f 100644 --- a/cni/network/multitenancy.go +++ b/cni/network/multitenancy.go @@ -64,9 +64,9 @@ func getContainerNetworkConfigurationInternal( namespace string, podName string, ifName string) (*cniTypesCurr.Result, *cns.GetNetworkContainerResponse, net.IPNet, error) { - cnsClient, err := cnsclient.NewCnsClient(address) + cnsClient, err := cnsclient.GetCnsClient() if err != nil { - log.Printf("Initializing CNS client error %v", err) + log.Printf("Failed to get CNS client. Error: %v", err) return nil, nil, net.IPNet{}, err } diff --git a/cni/network/network.go b/cni/network/network.go index 83ef22c12d..ed345af664 100644 --- a/cni/network/network.go +++ b/cni/network/network.go @@ -242,6 +242,11 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error { return err } + if nwCfg.MultiTenancy { + // Initialize CNSClient + cnsclient.InitCnsClient(nwCfg.CNSUrl) + } + k8sContainerID := args.ContainerID if len(k8sContainerID) == 0 { errMsg := "Container ID not specified in CNI Args" @@ -552,6 +557,11 @@ func (plugin *netPlugin) Get(args *cniSkel.CmdArgs) error { return err } + if nwCfg.MultiTenancy { + // Initialize CNSClient + cnsclient.InitCnsClient(nwCfg.CNSUrl) + } + // Initialize values from network config. if networkId, err = getNetworkName(k8sPodName, k8sNamespace, args.IfName, nwCfg); err != nil { log.Printf("[cni-net] Failed to extract network name from network config. error: %v", err) @@ -627,6 +637,11 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error { log.Printf("[cni-net] Failed to get POD info due to error: %v", err) } + if nwCfg.MultiTenancy { + // Initialize CNSClient + cnsclient.InitCnsClient(nwCfg.CNSUrl) + } + // Initialize values from network config. if networkId, err = getNetworkName(k8sPodName, k8sNamespace, args.IfName, nwCfg); err != nil { log.Printf("[cni-net] Failed to extract network name from network config. error: %v", err) @@ -772,7 +787,7 @@ func (plugin *netPlugin) Update(args *cniSkel.CmdArgs) error { // now query CNS to get the target routes that should be there in the networknamespace (as a result of update) log.Printf("Going to collect target routes for [name=%v, namespace=%v] from CNS.", k8sPodName, k8sNamespace) - if cnsClient, err = cnsclient.NewCnsClient(nwCfg.CNSUrl); err != nil { + if cnsClient, err = cnsclient.InitCnsClient(nwCfg.CNSUrl); err != nil { log.Printf("Initializing CNS client error in CNI Update%v", err) log.Printf(err.Error()) return plugin.Errorf(err.Error()) diff --git a/cni/network/network_linux.go b/cni/network/network_linux.go index cd771afe2a..85042ae4d8 100644 --- a/cni/network/network_linux.go +++ b/cni/network/network_linux.go @@ -58,6 +58,7 @@ func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *ne epInfo.Data[network.SnatBridgeIPKey] = cnsNwConfig.LocalIPConfiguration.GatewayIPAddress + "/" + strconv.Itoa(int(cnsNwConfig.LocalIPConfiguration.IPSubnet.PrefixLength)) epInfo.AllowInboundFromHostToNC = cnsNwConfig.AllowHostToNCCommunication epInfo.AllowInboundFromNCToHost = cnsNwConfig.AllowNCToHostCommunication + epInfo.NetworkContainerID = cnsNwConfig.NetworkContainerID } epInfo.Data[network.OptVethName] = vethName diff --git a/cni/network/network_windows.go b/cni/network/network_windows.go index 9df1fa3558..036390dd11 100644 --- a/cni/network/network_windows.go +++ b/cni/network/network_windows.go @@ -93,6 +93,9 @@ func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *ne cnetAddressMap = append(cnetAddressMap, ipSubnet.IPAddress+"/"+strconv.Itoa(int(ipSubnet.PrefixLength))) } epInfo.Data[network.CnetAddressSpace] = cnetAddressMap + epInfo.AllowInboundFromHostToNC = cnsNwConfig.AllowHostToNCCommunication + epInfo.AllowInboundFromNCToHost = cnsNwConfig.AllowNCToHostCommunication + epInfo.NetworkContainerID = cnsNwConfig.NetworkContainerID } } diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 1816e5187c..55d82e7118 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -131,6 +131,7 @@ type GetNetworkContainerRequest struct { // GetNetworkContainerResponse describes the response to retrieve a specifc network container. type GetNetworkContainerResponse struct { + NetworkContainerID string IPConfiguration IPConfiguration Routes []Route CnetAddressSpace []IPSubnet diff --git a/cns/api.go b/cns/api.go index 42ed300fa3..e933d42b56 100644 --- a/cns/api.go +++ b/cns/api.go @@ -7,20 +7,22 @@ import "encoding/json" // Container Network Service remote API Contract const ( - SetEnvironmentPath = "/network/environment" - CreateNetworkPath = "/network/create" - DeleteNetworkPath = "/network/delete" - CreateHnsNetworkPath = "/network/hns/create" - DeleteHnsNetworkPath = "/network/hns/delete" - ReserveIPAddressPath = "/network/ip/reserve" - ReleaseIPAddressPath = "/network/ip/release" - GetHostLocalIPPath = "/network/ip/hostlocal" - GetIPAddressUtilizationPath = "/network/ip/utilization" - GetUnhealthyIPAddressesPath = "/network/ipaddresses/unhealthy" - GetHealthReportPath = "/network/health" - NumberOfCPUCoresPath = "/hostcpucores" - V1Prefix = "/v0.1" - V2Prefix = "/v0.2" + SetEnvironmentPath = "/network/environment" + CreateNetworkPath = "/network/create" + DeleteNetworkPath = "/network/delete" + CreateHnsNetworkPath = "/network/hns/create" + DeleteHnsNetworkPath = "/network/hns/delete" + ReserveIPAddressPath = "/network/ip/reserve" + ReleaseIPAddressPath = "/network/ip/release" + GetHostLocalIPPath = "/network/ip/hostlocal" + GetIPAddressUtilizationPath = "/network/ip/utilization" + GetUnhealthyIPAddressesPath = "/network/ipaddresses/unhealthy" + GetHealthReportPath = "/network/health" + NumberOfCPUCoresPath = "/hostcpucores" + CreateHostNCApipaEndpointPath = "/network/createhostncapipaendpoint" + DeleteHostNCApipaEndpointPath = "/network/deletehostncapipaendpoint" + V1Prefix = "/v0.1" + V2Prefix = "/v0.2" ) // SetEnvironmentRequest describes the Request to set the environment in CNS. @@ -153,3 +155,27 @@ type OptionMap map[string]interface{} type errorResponse struct { Err string } + +// CreateHostNCApipaEndpointRequest describes request for create apipa endpoint +// for host container connectivity for the given network container +type CreateHostNCApipaEndpointRequest struct { + NetworkContainerID string +} + +// CreateHostNCApipaEndpointResponse describes response for create apipa endpoint request +// for host container connectivity. +type CreateHostNCApipaEndpointResponse struct { + Response Response + EndpointID string +} + +// DeleteHostNCApipaEndpointRequest describes request for deleting apipa endpoint created +// for host NC connectivity. +type DeleteHostNCApipaEndpointRequest struct { + NetworkContainerID string +} + +// DeleteHostNCApipaEndpointResponse describes response for delete host NC apipa endpoint request. +type DeleteHostNCApipaEndpointResponse struct { + Response Response +} diff --git a/cns/cnsclient/cnsclient.go b/cns/cnsclient/cnsclient.go index aef2749c6d..613b226fd9 100644 --- a/cns/cnsclient/cnsclient.go +++ b/cns/cnsclient/cnsclient.go @@ -19,15 +19,34 @@ const ( defaultCnsURL = "http://localhost:10090" ) -// NewCnsClient create a new cns client. -func NewCnsClient(url string) (*CNSClient, error) { - if url == "" { - url = defaultCnsURL +var ( + cnsClient *CNSClient +) + +// InitCnsClient initializes new cns client and returns the object +func InitCnsClient(url string) (*CNSClient, error) { + if cnsClient == nil { + if url == "" { + url = defaultCnsURL + } + + cnsClient = &CNSClient{ + connectionURL: url, + } } - return &CNSClient{ - connectionURL: url, - }, nil + return cnsClient, nil +} + +// GetCnsClient returns the cns client object +func GetCnsClient() (*CNSClient, error) { + var err error + + if cnsClient == nil { + err = fmt.Errorf("[Azure CNSClient] CNS Client not initialized") + } + + return cnsClient, err } // GetNetworkConfiguration Request to get network config. @@ -77,3 +96,105 @@ func (cnsClient *CNSClient) GetNetworkConfiguration(orchestratorContext []byte) return &resp, nil } + +// CreateHostNCApipaEndpoint creates an endpoint in APIPA network for host container connectivity. +func (cnsClient *CNSClient) CreateHostNCApipaEndpoint( + networkContainerID string) (string, error) { + var ( + err error + body bytes.Buffer + ) + + httpc := &http.Client{} + url := cnsClient.connectionURL + cns.CreateHostNCApipaEndpointPath + log.Printf("CreateHostNCApipaEndpoint url: %v for NC: %s", url, networkContainerID) + + payload := &cns.CreateHostNCApipaEndpointRequest{ + NetworkContainerID: networkContainerID, + } + + if err = json.NewEncoder(&body).Encode(payload); err != nil { + log.Errorf("encoding json failed with %v", err) + return "", err + } + + res, err := httpc.Post(url, "application/json", &body) + if err != nil { + log.Errorf("[Azure CNSClient] HTTP Post returned error %v", err.Error()) + return "", err + } + + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + errMsg := fmt.Sprintf("[Azure CNSClient] CreateHostNCApipaEndpoint: Invalid http status code: %v", + res.StatusCode) + log.Errorf(errMsg) + return "", fmt.Errorf(errMsg) + } + + var resp cns.CreateHostNCApipaEndpointResponse + + if err = json.NewDecoder(res.Body).Decode(&resp); err != nil { + log.Errorf("[Azure CNSClient] Error parsing CreateHostNCApipaEndpoint response resp: %v err: %v", + res.Body, err.Error()) + return "", err + } + + if resp.Response.ReturnCode != 0 { + log.Errorf("[Azure CNSClient] CreateHostNCApipaEndpoint received error response :%v", resp.Response.Message) + return "", fmt.Errorf(resp.Response.Message) + } + + return resp.EndpointID, nil +} + +// DeleteHostNCApipaEndpoint deletes the endpoint in APIPA network created for host container connectivity. +func (cnsClient *CNSClient) DeleteHostNCApipaEndpoint(networkContainerID string) error { + var body bytes.Buffer + + httpc := &http.Client{} + url := cnsClient.connectionURL + cns.DeleteHostNCApipaEndpointPath + log.Printf("DeleteHostNCApipaEndpoint url: %v for NC: %s", url, networkContainerID) + + payload := &cns.DeleteHostNCApipaEndpointRequest{ + NetworkContainerID: networkContainerID, + } + + err := json.NewEncoder(&body).Encode(payload) + if err != nil { + log.Errorf("encoding json failed with %v", err) + return err + } + + res, err := httpc.Post(url, "application/json", &body) + if err != nil { + log.Errorf("[Azure CNSClient] HTTP Post returned error %v", err.Error()) + return err + } + + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + errMsg := fmt.Sprintf("[Azure CNSClient] DeleteHostNCApipaEndpoint: Invalid http status code: %v", + res.StatusCode) + log.Errorf(errMsg) + return fmt.Errorf(errMsg) + } + + var resp cns.DeleteHostNCApipaEndpointResponse + + err = json.NewDecoder(res.Body).Decode(&resp) + if err != nil { + log.Errorf("[Azure CNSClient] Error parsing DeleteHostNCApipaEndpoint response resp: %v err: %v", + res.Body, err.Error()) + return err + } + + if resp.Response.ReturnCode != 0 { + log.Errorf("[Azure CNSClient] DeleteHostNCApipaEndpoint received error response :%v", resp.Response.Message) + return fmt.Errorf(resp.Response.Message) + } + + return nil +} diff --git a/cns/hnsclient/hnsclient_linux.go b/cns/hnsclient/hnsclient_linux.go index 186dec2eb3..a78acb9fb8 100644 --- a/cns/hnsclient/hnsclient_linux.go +++ b/cns/hnsclient/hnsclient_linux.go @@ -30,3 +30,22 @@ func CreateHnsNetwork(nwConfig cns.CreateHnsNetworkRequest) error { func DeleteHnsNetwork(networkName string) error { return fmt.Errorf("DeleteHnsNetwork shouldn't be called for linux platform") } + +// CreateHostNCApipaEndpoint creates the endpoint in the apipa network +// for host container connectivity +// This is windows platform specific. +func CreateHostNCApipaEndpoint( + networkContainerID string, + localIPConfiguration cns.IPConfiguration, + allowNCToHostCommunication bool, + allowHostToNCCommunication bool) (string, error) { + return "", nil +} + +// DeleteHostNCApipaEndpoint deletes the endpoint in the apipa network +// created for host container connectivity +// This is windows platform specific. +func DeleteHostNCApipaEndpoint( + networkContainerID string) error { + return nil +} diff --git a/cns/hnsclient/hnsclient_windows.go b/cns/hnsclient/hnsclient_windows.go index 249807efda..1fe9333fa8 100644 --- a/cns/hnsclient/hnsclient_windows.go +++ b/cns/hnsclient/hnsclient_windows.go @@ -3,11 +3,17 @@ package hnsclient import ( "encoding/json" "fmt" - "log" + "net" + "strconv" "strings" "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/networkcontainers" + "github.com/Azure/azure-container-networking/common" + "github.com/Azure/azure-container-networking/log" + "github.com/Azure/azure-container-networking/network/policy" "github.com/Microsoft/hcsshim" + "github.com/Microsoft/hcsshim/hcn" ) const ( @@ -23,6 +29,48 @@ const ( // HNS network types hnsL2Bridge = "l2bridge" hnsL2Tunnel = "l2tunnel" + + // hcnSchemaVersionMajor indicates major version number for hcn schema + hcnSchemaVersionMajor = 2 + + // hcnSchemaVersionMinor indicates minor version number for hcn schema + hcnSchemaVersionMinor = 0 + + // hcnIpamTypeStatic indicates the static type of ipam + hcnIpamTypeStatic = "Static" + + // hostNCApipaNetworkName indicates the name of the apipa network used for host container connectivity + hostNCApipaNetworkName = "HostNCApipaNetwork" + + // hostNCApipaNetworkType indicates the type of hns network set up for host NC connectivity + hostNCApipaNetworkType = hcn.L2Bridge + + // hostNCApipaEndpointName indicates the prefix for the name of the apipa endpoint used for + // the host container connectivity + hostNCApipaEndpointNamePrefix = "HostNCApipaEndpoint" + + // Name of the loopback adapter needed to create Host NC apipa network + hostNCLoopbackAdapterName = "LoopbackAdapterHostNCConnectivity" + + // protocolTCP indicates the TCP protocol identifier in HCN + protocolTCP = "6" + + // protocolUDP indicates the UDP protocol identifier in HCN + protocolUDP = "17" + + // protocolICMPv4 indicates the ICMPv4 protocol identifier in HCN + protocolICMPv4 = "1" + + // aclPriority2000 indicates the ACL priority of 2000 + aclPriority2000 = 2000 + + // aclPriority200 indicates the ACL priority of 200 + aclPriority200 = 200 +) + +var ( + // Named Lock for network and endpoint creation/deletion + namedLock = common.InitNamedLock() ) // CreateHnsNetwork creates the HNS network with the provided configuration @@ -153,3 +201,454 @@ func deleteHnsNetwork(networkName string) error { return err } + +func configureHostNCApipaNetwork(localIPConfiguration cns.IPConfiguration) (*hcn.HostComputeNetwork, error) { + network := &hcn.HostComputeNetwork{ + Name: hostNCApipaNetworkName, + Ipams: []hcn.Ipam{ + hcn.Ipam{ + Type: hcnIpamTypeStatic, + }, + }, + SchemaVersion: hcn.SchemaVersion{ + Major: hcnSchemaVersionMajor, + Minor: hcnSchemaVersionMinor, + }, + Type: hostNCApipaNetworkType, + Flags: hcn.EnableNonPersistent, // Set up the network in non-persistent mode + } + + if netAdapterNamePolicy, err := policy.GetHcnNetAdapterPolicy(hostNCLoopbackAdapterName); err == nil { + network.Policies = append(network.Policies, netAdapterNamePolicy) + } else { + return nil, fmt.Errorf("Failed to serialize network adapter policy. Error: %v", err) + } + + // Calculate subnet prefix + // Following code calculates the subnet prefix from localIPConfiguration IP + // e.g. IP: 169.254.128.7 Prefix length: 17 then resulting subnet prefix: 169.254.128.0/17 + // subnetPrefix: ffff8000 + // subnetPrefix.IP: 169.254.128.0 + var ( + subnetPrefix net.IPNet + subnetPrefixStr string + ipAddr net.IP + ) + + ipAddr = net.ParseIP(localIPConfiguration.IPSubnet.IPAddress) + if ipAddr.To4() != nil { + subnetPrefix = net.IPNet{Mask: net.CIDRMask(int(localIPConfiguration.IPSubnet.PrefixLength), 32)} + } else if ipAddr.To16() != nil { + subnetPrefix = net.IPNet{Mask: net.CIDRMask(int(localIPConfiguration.IPSubnet.PrefixLength), 128)} + } else { + return nil, fmt.Errorf("Failed get subnet prefix for localIPConfiguration: %+v", localIPConfiguration) + } + + subnetPrefix.IP = ipAddr.Mask(subnetPrefix.Mask) + subnetPrefixStr = subnetPrefix.IP.String() + "/" + strconv.Itoa(int(localIPConfiguration.IPSubnet.PrefixLength)) + + subnet := hcn.Subnet{ + IpAddressPrefix: subnetPrefixStr, + Routes: []hcn.Route{ + hcn.Route{ + NextHop: localIPConfiguration.GatewayIPAddress, + DestinationPrefix: "0.0.0.0/0", + }, + }, + } + + network.Ipams[0].Subnets = append(network.Ipams[0].Subnets, subnet) + + log.Printf("[Azure CNS] Configured HostNCApipaNetwork: %+v", network) + + return network, nil +} + +func createHostNCApipaNetwork( + localIPConfiguration cns.IPConfiguration) (*hcn.HostComputeNetwork, error) { + var ( + network *hcn.HostComputeNetwork + err error + ) + + namedLock.LockAcquire(hostNCApipaNetworkName) + defer namedLock.LockRelease(hostNCApipaNetworkName) + + // Check if the network exists for Host NC connectivity + if network, err = hcn.GetNetworkByName(hostNCApipaNetworkName); err != nil { + // If error is anything other than networkNotFound, mark this as error + if _, networkNotFound := err.(hcn.NetworkNotFoundError); !networkNotFound { + return nil, fmt.Errorf("[Azure CNS] ERROR: createApipaNetwork failed. Error with GetNetworkByName: %v", err) + } + + // Network doesn't exist. Create one. + if network, err = configureHostNCApipaNetwork(localIPConfiguration); err != nil { + return nil, fmt.Errorf("Failed to configure network. Error: %v", err) + } + + // Create loopback adapter needed for this HNS network + if interfaceExists, _ := networkcontainers.InterfaceExists(hostNCLoopbackAdapterName); !interfaceExists { + ipconfig := cns.IPConfiguration{ + IPSubnet: cns.IPSubnet{ + IPAddress: localIPConfiguration.GatewayIPAddress, + PrefixLength: localIPConfiguration.IPSubnet.PrefixLength, + }, + GatewayIPAddress: localIPConfiguration.GatewayIPAddress, + } + + if err = networkcontainers.CreateLoopbackAdapter( + hostNCLoopbackAdapterName, + ipconfig, + false, /* Flag to setWeakHostOnInterface */ + "" /* Empty primary Interface Identifier as setWeakHostOnInterface is not needed*/); err != nil { + return nil, fmt.Errorf("Failed to create loopback adapter. Error: %v", err) + } + } + + // Create the HNS network. + log.Printf("[Azure CNS] Creating HostNCApipaNetwork: %+v", network) + + if network, err = network.Create(); err != nil { + return nil, err + } + + log.Printf("[Azure CNS] Successfully created apipa network for host container connectivity: %+v", network) + } else { + log.Printf("[Azure CNS] Found existing HostNCApipaNetwork: %+v", network) + } + + return network, err +} + +func addAclToEndpointPolicy( + aclPolicySetting hcn.AclPolicySetting, + endpointPolicies *[]hcn.EndpointPolicy) error { + var ( + rawJSON []byte + err error + ) + + if rawJSON, err = json.Marshal(aclPolicySetting); err != nil { + return fmt.Errorf("Failed to marshal endpoint ACL: %+v", aclPolicySetting) + } + + endpointPolicy := hcn.EndpointPolicy{ + Type: hcn.ACL, + Settings: rawJSON, + } + + *endpointPolicies = append(*endpointPolicies, endpointPolicy) + + return nil +} + +func configureAclSettingHostNCApipaEndpoint( + protocolList []string, + networkContainerApipaIP string, + hostApipaIP string, + allowNCToHostCommunication bool, + allowHostToNCCommunication bool) ([]hcn.EndpointPolicy, error) { + var ( + err error + endpointPolicies []hcn.EndpointPolicy + ) + + if allowNCToHostCommunication { + log.Printf("[Azure CNS] Allowing NC (%s) to Host (%s) connectivity", networkContainerApipaIP, hostApipaIP) + } + + if allowHostToNCCommunication { + log.Printf("[Azure CNS] Allowing Host (%s) to NC (%s) connectivity", hostApipaIP, networkContainerApipaIP) + } + + // Iterate thru the protocol list and add ACL for each + for _, protocol := range protocolList { + // Endpoint ACL to block all outbound traffic from the Apipa IP of the container + outBlockAll := hcn.AclPolicySetting{ + Protocols: protocol, + Action: hcn.ActionTypeBlock, + Direction: hcn.DirectionTypeOut, + LocalAddresses: networkContainerApipaIP, + RuleType: hcn.RuleTypeSwitch, + Priority: aclPriority2000, + } + + if err = addAclToEndpointPolicy(outBlockAll, &endpointPolicies); err != nil { + return nil, err + } + + if allowNCToHostCommunication { + // Endpoint ACL to allow the outbound traffic from the Apipa IP of the container to + // Apipa IP of the host only + outAllowToHostOnly := hcn.AclPolicySetting{ + Protocols: protocol, + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeOut, + LocalAddresses: networkContainerApipaIP, + RemoteAddresses: hostApipaIP, + RuleType: hcn.RuleTypeSwitch, + Priority: aclPriority200, + } + + if err = addAclToEndpointPolicy(outAllowToHostOnly, &endpointPolicies); err != nil { + return nil, err + } + } + + // Endpoint ACL to block all inbound traffic to the Apipa IP of the container + inBlockAll := hcn.AclPolicySetting{ + Protocols: protocol, + Action: hcn.ActionTypeBlock, + Direction: hcn.DirectionTypeIn, + LocalAddresses: networkContainerApipaIP, + RuleType: hcn.RuleTypeSwitch, + Priority: aclPriority2000, + } + + if err = addAclToEndpointPolicy(inBlockAll, &endpointPolicies); err != nil { + return nil, err + } + + if allowHostToNCCommunication { + // Endpoint ACL to allow the inbound traffic from the apipa IP of the host to + // the apipa IP of the container only + inAllowFromHostOnly := hcn.AclPolicySetting{ + Protocols: protocol, + Action: hcn.ActionTypeAllow, + Direction: hcn.DirectionTypeIn, + LocalAddresses: networkContainerApipaIP, + RemoteAddresses: hostApipaIP, + RuleType: hcn.RuleTypeSwitch, + Priority: aclPriority200, + } + + if err = addAclToEndpointPolicy(inAllowFromHostOnly, &endpointPolicies); err != nil { + return nil, err + } + } + } + + return endpointPolicies, nil +} + +func configureHostNCApipaEndpoint( + endpointName string, + networkID string, + localIPConfiguration cns.IPConfiguration, + allowNCToHostCommunication bool, + allowHostToNCCommunication bool) (*hcn.HostComputeEndpoint, error) { + endpoint := &hcn.HostComputeEndpoint{ + Name: endpointName, + HostComputeNetwork: networkID, + SchemaVersion: hcn.SchemaVersion{ + Major: hcnSchemaVersionMajor, + Minor: hcnSchemaVersionMinor, + }, + } + + networkContainerApipaIP := localIPConfiguration.IPSubnet.IPAddress + hostApipaIP := localIPConfiguration.GatewayIPAddress + protocolList := []string{protocolICMPv4, protocolTCP, protocolUDP} + + endpointPolicies, err := configureAclSettingHostNCApipaEndpoint( + protocolList, + networkContainerApipaIP, + hostApipaIP, + allowNCToHostCommunication, + allowHostToNCCommunication) + + if err != nil { + log.Errorf("[Azure CNS] Failed to configure ACL for HostNCApipaEndpoint. Error: %v", err) + return nil, err + } + + for _, endpointPolicy := range endpointPolicies { + endpoint.Policies = append(endpoint.Policies, endpointPolicy) + } + + hcnRoute := hcn.Route{ + NextHop: hostApipaIP, + DestinationPrefix: "0.0.0.0/0", + } + + endpoint.Routes = append(endpoint.Routes, hcnRoute) + + ipConfiguration := hcn.IpConfig{ + IpAddress: networkContainerApipaIP, + PrefixLength: localIPConfiguration.IPSubnet.PrefixLength, + } + + endpoint.IpConfigurations = append(endpoint.IpConfigurations, ipConfiguration) + + log.Printf("[Azure CNS] Configured HostNCApipaEndpoint: %+v", endpoint) + + return endpoint, nil +} + +// CreateHostNCApipaEndpoint creates the endpoint in the apipa network for host container connectivity +func CreateHostNCApipaEndpoint( + networkContainerID string, + localIPConfiguration cns.IPConfiguration, + allowNCToHostCommunication bool, + allowHostToNCCommunication bool) (string, error) { + var ( + network *hcn.HostComputeNetwork + endpoint *hcn.HostComputeEndpoint + endpointName = getHostNCApipaEndpointName(networkContainerID) + err error + ) + + namedLock.LockAcquire(endpointName) + defer namedLock.LockRelease(endpointName) + + // Return if the endpoint already exists + if endpoint, err = hcn.GetEndpointByName(endpointName); err != nil { + // If error is anything other than EndpointNotFoundError, return error. + if _, endpointNotFound := err.(hcn.EndpointNotFoundError); !endpointNotFound { + return "", fmt.Errorf("ERROR: Failed to query endpoint using GetEndpointByName "+ + "due to error: %v", err) + } + } + + if endpoint != nil { + log.Debugf("[Azure CNS] Found existing endpoint: %+v", endpoint) + return endpoint.Id, nil + } + + if network, err = createHostNCApipaNetwork(localIPConfiguration); err != nil { + log.Errorf("[Azure CNS] Failed to create HostNCApipaNetwork. Error: %v", err) + return "", err + } + + log.Printf("[Azure CNS] Configuring HostNCApipaEndpoint: %s, in network: %s with localIPConfig: %+v", + endpointName, network.Id, localIPConfiguration) + + if endpoint, err = configureHostNCApipaEndpoint( + endpointName, + network.Id, + localIPConfiguration, + allowNCToHostCommunication, + allowHostToNCCommunication); err != nil { + log.Errorf("[Azure CNS] Failed to configure HostNCApipaEndpoint: %s. Error: %v", endpointName, err) + return "", err + } + + log.Printf("[Azure CNS] Creating HostNCApipaEndpoint for host container connectivity: %+v", endpoint) + if endpoint, err = endpoint.Create(); err != nil { + err = fmt.Errorf("Failed to create HostNCApipaEndpoint: %s. Error: %v", endpointName, err) + log.Errorf("[Azure CNS] %s", err.Error()) + return "", err + } + + log.Printf("[Azure CNS] Successfully created HostNCApipaEndpoint: %+v", endpoint) + + return endpoint.Id, nil +} + +func getHostNCApipaEndpointName( + networkContainerID string) string { + return hostNCApipaEndpointNamePrefix + "-" + networkContainerID +} + +func deleteNetworkByIDHnsV2( + networkID string) error { + var ( + network *hcn.HostComputeNetwork + err error + ) + + if network, err = hcn.GetNetworkByID(networkID); err != nil { + // If error is anything other than NetworkNotFoundError, return error. + // else log the error but don't return error because network is already deleted. + if _, networkNotFound := err.(hcn.NetworkNotFoundError); !networkNotFound { + return fmt.Errorf("[Azure CNS] deleteNetworkByIDHnsV2 failed due to "+ + "error with GetNetworkByID: %v", err) + } + + log.Errorf("[Azure CNS] Delete called on the Network: %s which doesn't exist. Error: %v", + networkID, err) + + return nil + } + + if err = network.Delete(); err != nil { + return fmt.Errorf("Failed to delete network: %+v. Error: %v", network, err) + } + + log.Errorf("[Azure CNS] Successfully deleted network: %+v", network) + + return nil +} + +func deleteEndpointByNameHnsV2( + endpointName string) error { + var ( + endpoint *hcn.HostComputeEndpoint + err error + ) + + // Check if the endpoint exists + if endpoint, err = hcn.GetEndpointByName(endpointName); err != nil { + // If error is anything other than EndpointNotFoundError, return error. + // else log the error but don't return error because endpoint is already deleted. + if _, endpointNotFound := err.(hcn.EndpointNotFoundError); !endpointNotFound { + return fmt.Errorf("[Azure CNS] deleteEndpointByNameHnsV2 failed due to "+ + "error with GetEndpointByName: %v", err) + } + + log.Errorf("[Azure CNS] Delete called on the Endpoint: %s which doesn't exist. Error: %v", + endpointName, err) + + return nil + } + + if err = endpoint.Delete(); err != nil { + return fmt.Errorf("Failed to delete endpoint: %+v. Error: %v", endpoint, err) + } + + log.Errorf("[Azure CNS] Successfully deleted endpoint: %+v", endpoint) + + return nil +} + +// DeleteHostNCApipaEndpoint deletes the endpoint in the apipa network created for host container connectivity +func DeleteHostNCApipaEndpoint( + networkContainerID string) error { + endpointName := getHostNCApipaEndpointName(networkContainerID) + + namedLock.LockAcquire(endpointName) + defer namedLock.LockRelease(endpointName) + + log.Debugf("[Azure CNS] Deleting HostNCApipaEndpoint: %s", endpointName) + + if err := deleteEndpointByNameHnsV2(endpointName); err != nil { + log.Errorf("[Azure CNS] Failed to delete HostNCApipaEndpoint: %s. Error: %v", endpointName, err) + return err + } + + log.Debugf("[Azure CNS] Successfully deleted HostNCApipaEndpoint: %s", endpointName) + + namedLock.LockAcquire(hostNCApipaNetworkName) + defer namedLock.LockRelease(hostNCApipaNetworkName) + + // Check if hostNCApipaNetworkName has any endpoints left + if network, err := hcn.GetNetworkByName(hostNCApipaNetworkName); err == nil { + var endpoints []hcn.HostComputeEndpoint + if endpoints, err = hcn.ListEndpointsOfNetwork(network.Id); err != nil { + log.Errorf("[Azure CNS] Failed to list endpoints in the network: %s. Error: %v", + hostNCApipaNetworkName, err) + return nil + } + + // Delete network if it doesn't have any endpoints + if len(endpoints) == 0 { + log.Debugf("[Azure CNS] Deleting network with ID: %s", network.Id) + if err = deleteNetworkByIDHnsV2(network.Id); err == nil { + // Delete the loopback adapter created for this network + networkcontainers.DeleteLoopbackAdapter(hostNCLoopbackAdapterName) + } + } + } + + return nil +} diff --git a/cns/networkcontainers/networkcontainers.go b/cns/networkcontainers/networkcontainers.go index d1218f8a71..87ea0b8efc 100644 --- a/cns/networkcontainers/networkcontainers.go +++ b/cns/networkcontainers/networkcontainers.go @@ -52,7 +52,7 @@ func NewNetPluginConfiguration(binPath, configPath string) *NetPluginConfigurati } } -func interfaceExists(iFaceName string) (bool, error) { +func InterfaceExists(iFaceName string) (bool, error) { _, err := net.InterfaceByName(iFaceName) if err != nil { errMsg := fmt.Sprintf("[Azure CNS] Unable to get interface by name %s. Error: %v", iFaceName, err) @@ -94,6 +94,25 @@ func (cn *NetworkContainers) Delete(networkContainerID string) error { return err } +// CreateLoopbackAdapter creates a loopback adapter with the specified settings +func CreateLoopbackAdapter( + adapterName string, + ipConfig cns.IPConfiguration, + setWeakHostOnInterface bool, + primaryInterfaceIdentifier string) error { + return createOrUpdateWithOperation( + adapterName, + ipConfig, + setWeakHostOnInterface, // Flag to setWeakHostOnInterface + primaryInterfaceIdentifier, + "CREATE") +} + +// DeleteLoopbackAdapter deletes loopback adapter with the specified name +func DeleteLoopbackAdapter(adapterName string) error { + return deleteInterface(adapterName) +} + // This function gets the flattened network configuration (compliant with azure cni) in byte array format func getNetworkConfig(configFilePath string) ([]byte, error) { content, err := ioutil.ReadFile(configFilePath) diff --git a/cns/networkcontainers/networkcontainers_linux.go b/cns/networkcontainers/networkcontainers_linux.go index 0da7ce76a3..e776988c4b 100644 --- a/cns/networkcontainers/networkcontainers_linux.go +++ b/cns/networkcontainers/networkcontainers_linux.go @@ -90,3 +90,12 @@ func deleteInterface(networkContainerID string) error { func configureNetworkContainerNetworking(operation, podName, podNamespace, dockerContainerid string, netPluginConfig *NetPluginConfiguration) (err error) { return fmt.Errorf("[Azure CNS] Operation is not supported in linux.") } + +func createOrUpdateWithOperation( + adapterName string, + ipConfig cns.IPConfiguration, + setWeakHost bool, + primaryInterfaceIdentifier string, + operation string) error { + return nil +} diff --git a/cns/networkcontainers/networkcontainers_windows.go b/cns/networkcontainers/networkcontainers_windows.go index b3b8b912ae..f5439b951b 100644 --- a/cns/networkcontainers/networkcontainers_windows.go +++ b/cns/networkcontainers/networkcontainers_windows.go @@ -27,11 +27,21 @@ func createOrUpdateInterface(createNetworkContainerRequest cns.CreateNetworkCont return nil } - if exists, _ := interfaceExists(createNetworkContainerRequest.NetworkContainerid); !exists { - return createOrUpdateWithOperation(createNetworkContainerRequest, "CREATE") + if exists, _ := InterfaceExists(createNetworkContainerRequest.NetworkContainerid); !exists { + return createOrUpdateWithOperation( + createNetworkContainerRequest.NetworkContainerid, + createNetworkContainerRequest.IPConfiguration, + true, // Flag to setWeakHostOnInterface + createNetworkContainerRequest.PrimaryInterfaceIdentifier, + "CREATE") } - return createOrUpdateWithOperation(createNetworkContainerRequest, "UPDATE") + return createOrUpdateWithOperation( + createNetworkContainerRequest.NetworkContainerid, + createNetworkContainerRequest.IPConfiguration, + true, // Flag to setWeakHostOnInterface + createNetworkContainerRequest.PrimaryInterfaceIdentifier, + "UPDATE") } func updateInterface(createNetworkContainerRequest cns.CreateNetworkContainerRequest, netpluginConfig *NetPluginConfiguration) error { @@ -102,28 +112,31 @@ func setWeakHostOnInterface(ipAddress, ncID string) error { return nil } -func createOrUpdateWithOperation(createNetworkContainerRequest cns.CreateNetworkContainerRequest, operation string) error { +func createOrUpdateWithOperation( + adapterName string, + ipConfig cns.IPConfiguration, + setWeakHost bool, + primaryInterfaceIdentifier string, + operation string) error { if _, err := os.Stat("./AzureNetworkContainer.exe"); err != nil { - if os.IsNotExist(err) { - return errors.New("[Azure CNS] Unable to find AzureNetworkContainer.exe. Cannot continue") - } + return fmt.Errorf("[Azure CNS] Unable to find AzureNetworkContainer.exe. Cannot continue") } - if createNetworkContainerRequest.IPConfiguration.IPSubnet.IPAddress == "" { - return errors.New("[Azure CNS] IPAddress in IPConfiguration of createNetworkContainerRequest is nil") + if ipConfig.IPSubnet.IPAddress == "" { + return fmt.Errorf("[Azure CNS] IPAddress in IPConfiguration is nil") } - ipv4AddrCidr := fmt.Sprintf("%v/%d", createNetworkContainerRequest.IPConfiguration.IPSubnet.IPAddress, createNetworkContainerRequest.IPConfiguration.IPSubnet.PrefixLength) + ipv4AddrCidr := fmt.Sprintf("%v/%d", ipConfig.IPSubnet.IPAddress, ipConfig.IPSubnet.PrefixLength) log.Printf("[Azure CNS] Created ipv4Cidr as %v", ipv4AddrCidr) ipv4Addr, _, err := net.ParseCIDR(ipv4AddrCidr) - ipv4NetInt := net.CIDRMask((int)(createNetworkContainerRequest.IPConfiguration.IPSubnet.PrefixLength), 32) + ipv4NetInt := net.CIDRMask((int)(ipConfig.IPSubnet.PrefixLength), 32) log.Printf("[Azure CNS] Created netmask as %v", ipv4NetInt) ipv4NetStr := fmt.Sprintf("%d.%d.%d.%d", ipv4NetInt[0], ipv4NetInt[1], ipv4NetInt[2], ipv4NetInt[3]) log.Printf("[Azure CNS] Created netmask in string format %v", ipv4NetStr) args := []string{"/C", "AzureNetworkContainer.exe", "/logpath", log.GetLogDirectory(), "/name", - createNetworkContainerRequest.NetworkContainerid, + adapterName, "/operation", operation, "/ip", @@ -131,7 +144,7 @@ func createOrUpdateWithOperation(createNetworkContainerRequest cns.CreateNetwork "/netmask", ipv4NetStr, "/gateway", - createNetworkContainerRequest.IPConfiguration.GatewayIPAddress, + ipConfig.GatewayIPAddress, "/weakhostsend", "true", "/weakhostreceive", @@ -142,38 +155,34 @@ func createOrUpdateWithOperation(createNetworkContainerRequest cns.CreateNetwork loopbackOperationLock.Lock() log.Printf("[Azure CNS] Going to create/update network loopback adapter: %v", args) bytes, err := c.Output() - if err == nil { - err = setWeakHostOnInterface(createNetworkContainerRequest.PrimaryInterfaceIdentifier, - createNetworkContainerRequest.NetworkContainerid) + if err == nil && setWeakHost { + err = setWeakHostOnInterface(primaryInterfaceIdentifier, adapterName) } loopbackOperationLock.Unlock() if err == nil { - log.Printf("[Azure CNS] Successfully created network loopback adapter for NC: %s. Output:%v.", - createNetworkContainerRequest.NetworkContainerid, string(bytes)) + log.Printf("[Azure CNS] Successfully created network loopback adapter with name: %s and IP config: %+v. Output:%v.", + adapterName, ipConfig, string(bytes)) } else { - log.Printf("Failed to create/update Network Container: %s. Error: %v. Output: %v", - createNetworkContainerRequest.NetworkContainerid, err.Error(), string(bytes)) + log.Printf("[Azure CNS] Failed to create network loopback adapter with name: %s and IP config: %+v."+ + " Error: %v. Output: %v", adapterName, ipConfig, err, string(bytes)) } return err } -func deleteInterface(networkContainerID string) error { - +func deleteInterface(interfaceName string) error { if _, err := os.Stat("./AzureNetworkContainer.exe"); err != nil { - if os.IsNotExist(err) { - return errors.New("[Azure CNS] Unable to find AzureNetworkContainer.exe. Cannot continue") - } + return fmt.Errorf("[Azure CNS] Unable to find AzureNetworkContainer.exe. Cannot continue") } - if networkContainerID == "" { - return errors.New("[Azure CNS] networkContainerID is nil") + if interfaceName == "" { + return fmt.Errorf("[Azure CNS] Interface name is nil") } args := []string{"/C", "AzureNetworkContainer.exe", "/logpath", log.GetLogDirectory(), "/name", - networkContainerID, + interfaceName, "/operation", "DELETE"} @@ -185,14 +194,14 @@ func deleteInterface(networkContainerID string) error { loopbackOperationLock.Unlock() if err == nil { - log.Printf("[Azure CNS] Successfully deleted network container: %s. Output: %v.", - networkContainerID, string(bytes)) + log.Printf("[Azure CNS] Successfully deleted loopack adapter with name: %s. Output: %v.", + interfaceName, string(bytes)) } else { - log.Printf("Failed to delete Network Container: %s. Error:%v. Output:%v", - networkContainerID, err.Error(), string(bytes)) - return err + log.Printf("[Azure CNS] Failed to delete loopback adapter with name: %s. Error:%v. Output:%v", + interfaceName, err.Error(), string(bytes)) } - return nil + + return err } func configureNetworkContainerNetworking(operation, podName, podNamespace, dockerContainerid string, netPluginConfig *NetPluginConfiguration) (err error) { diff --git a/cns/restserver/api.go b/cns/restserver/api.go index 25d60e3ca2..f2a4fcbf2d 100644 --- a/cns/restserver/api.go +++ b/cns/restserver/api.go @@ -23,6 +23,7 @@ const ( DockerContainerNotSpecified = 20 UnsupportedVerb = 21 UnsupportedNetworkContainerType = 22 + InvalidRequest = 23 UnexpectedError = 99 ) diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 142ca92100..f38c625d62 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -159,6 +159,8 @@ func (service *HTTPRestService) Start(config *common.ServiceConfig) error { listener.AddHandler(cns.CreateHnsNetworkPath, service.createHnsNetwork) listener.AddHandler(cns.DeleteHnsNetworkPath, service.deleteHnsNetwork) listener.AddHandler(cns.NumberOfCPUCoresPath, service.getNumberOfCPUCores) + listener.AddHandler(cns.CreateHostNCApipaEndpointPath, service.createHostNCApipaEndpoint) + listener.AddHandler(cns.DeleteHostNCApipaEndpointPath, service.deleteHostNCApipaEndpoint) // handlers for v0.2 listener.AddHandler(cns.V2Prefix+cns.SetEnvironmentPath, service.setEnvironment) @@ -180,6 +182,8 @@ func (service *HTTPRestService) Start(config *common.ServiceConfig) error { listener.AddHandler(cns.V2Prefix+cns.CreateHnsNetworkPath, service.createHnsNetwork) listener.AddHandler(cns.V2Prefix+cns.DeleteHnsNetworkPath, service.deleteHnsNetwork) listener.AddHandler(cns.V2Prefix+cns.NumberOfCPUCoresPath, service.getNumberOfCPUCores) + listener.AddHandler(cns.V2Prefix+cns.CreateHostNCApipaEndpointPath, service.createHostNCApipaEndpoint) + listener.AddHandler(cns.V2Prefix+cns.DeleteHostNCApipaEndpointPath, service.deleteHostNCApipaEndpoint) log.Printf("[Azure CNS] Listening.") return nil @@ -1099,9 +1103,7 @@ func (service *HTTPRestService) createOrUpdateNetworkContainer(w http.ResponseWr case "POST": if req.NetworkContainerType == cns.WebApps { // try to get the saved nc state if it exists - service.lock.Lock() - existing, ok := service.state.ContainerStatus[req.NetworkContainerid] - service.lock.Unlock() + existing, ok := service.getNetworkContainerDetails(req.NetworkContainerid) // create/update nc only if it doesn't exist or it exists and the requested version is different from the saved version if !ok || (ok && existing.VMVersion != req.Version) { @@ -1114,9 +1116,7 @@ func (service *HTTPRestService) createOrUpdateNetworkContainer(w http.ResponseWr } } else if req.NetworkContainerType == cns.AzureContainerInstance { // try to get the saved nc state if it exists - service.lock.Lock() - existing, ok := service.state.ContainerStatus[req.NetworkContainerid] - service.lock.Unlock() + existing, ok := service.getNetworkContainerDetails(req.NetworkContainerid) // create/update nc only if it doesn't exist or it exists and the requested version is different from the saved version if ok && existing.VMVersion != req.Version { @@ -1216,6 +1216,7 @@ func (service *HTTPRestService) getNetworkContainerResponse(req cns.GetNetworkCo savedReq := containerDetails.CreateNetworkContainerRequest getNetworkContainerResponse = cns.GetNetworkContainerResponse{ + NetworkContainerID: savedReq.NetworkContainerid, IPConfiguration: savedReq.IPConfiguration, Routes: savedReq.Routes, CnetAddressSpace: savedReq.CnetAddressSpace, @@ -1277,9 +1278,7 @@ func (service *HTTPRestService) deleteNetworkContainer(w http.ResponseWriter, r var containerStatus containerstatus var ok bool - service.lock.Lock() - containerStatus, ok = service.state.ContainerStatus[req.NetworkContainerid] - service.lock.Unlock() + containerStatus, ok = service.getNetworkContainerDetails(req.NetworkContainerid) if !ok { log.Printf("Not able to retrieve network container details for this container id %v", req.NetworkContainerid) @@ -1540,9 +1539,8 @@ func (service *HTTPRestService) attachOrDetachHelper(req cns.ConfigureContainerN Message: "[Azure CNS] Error. NetworkContainerid is empty"} } - service.lock.Lock() - existing, ok := service.state.ContainerStatus[cns.SwiftPrefix+req.NetworkContainerid] - service.lock.Unlock() + existing, ok := service.getNetworkContainerDetails(cns.SwiftPrefix + req.NetworkContainerid) + if !ok { return cns.Response{ ReturnCode: NotFound, @@ -1619,3 +1617,109 @@ func (service *HTTPRestService) getNumberOfCPUCores(w http.ResponseWriter, r *ht log.Response(service.Name, numOfCPUCoresResp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) } + +func (service *HTTPRestService) getNetworkContainerDetails(networkContainerID string) (containerstatus, bool) { + service.lock.Lock() + defer service.lock.Unlock() + + containerDetails, containerExists := service.state.ContainerStatus[networkContainerID] + + return containerDetails, containerExists +} + +func (service *HTTPRestService) createHostNCApipaEndpoint(w http.ResponseWriter, r *http.Request) { + log.Printf("[Azure-CNS] createHostNCApipaEndpoint") + + var ( + err error + req cns.CreateHostNCApipaEndpointRequest + returnCode int + returnMessage string + endpointID string + ) + + err = service.Listener.Decode(w, r, &req) + log.Request(service.Name, &req, err) + if err != nil { + return + } + + switch r.Method { + case "POST": + networkContainerDetails, found := service.getNetworkContainerDetails(req.NetworkContainerID) + if found { + if !networkContainerDetails.CreateNetworkContainerRequest.AllowNCToHostCommunication && + !networkContainerDetails.CreateNetworkContainerRequest.AllowHostToNCCommunication { + returnMessage = fmt.Sprintf("HostNCApipaEndpoint creation is not supported unless " + + "AllowNCToHostCommunication or AllowHostToNCCommunication is set to true") + returnCode = InvalidRequest + } else { + if endpointID, err = hnsclient.CreateHostNCApipaEndpoint( + req.NetworkContainerID, + networkContainerDetails.CreateNetworkContainerRequest.LocalIPConfiguration, + networkContainerDetails.CreateNetworkContainerRequest.AllowNCToHostCommunication, + networkContainerDetails.CreateNetworkContainerRequest.AllowHostToNCCommunication); err != nil { + returnMessage = fmt.Sprintf("CreateHostNCApipaEndpoint failed with error: %v", err) + returnCode = UnexpectedError + } + } + } else { + returnMessage = fmt.Sprintf("CreateHostNCApipaEndpoint failed with error: Unable to find goal state for"+ + " the given Network Container: %s", req.NetworkContainerID) + returnCode = UnknownContainerID + } + default: + returnMessage = "createHostNCApipaEndpoint API expects a POST" + returnCode = UnsupportedVerb + } + + response := cns.CreateHostNCApipaEndpointResponse{ + Response: cns.Response{ + ReturnCode: returnCode, + Message: returnMessage, + }, + EndpointID: endpointID, + } + + err = service.Listener.Encode(w, &response) + log.Response(service.Name, response, response.Response.ReturnCode, ReturnCodeToString(response.Response.ReturnCode), err) +} + +func (service *HTTPRestService) deleteHostNCApipaEndpoint(w http.ResponseWriter, r *http.Request) { + log.Printf("[Azure-CNS] deleteHostNCApipaEndpoint") + + var ( + err error + req cns.DeleteHostNCApipaEndpointRequest + returnCode int + returnMessage string + ) + + err = service.Listener.Decode(w, r, &req) + log.Request(service.Name, &req, err) + if err != nil { + return + } + + switch r.Method { + case "POST": + if err = hnsclient.DeleteHostNCApipaEndpoint(req.NetworkContainerID); err != nil { + returnMessage = fmt.Sprintf("Failed to delete endpoint for Network Container: %s "+ + "due to error: %v", req.NetworkContainerID, err) + returnCode = UnexpectedError + } + default: + returnMessage = "deleteHostNCApipaEndpoint API expects a DELETE" + returnCode = UnsupportedVerb + } + + response := cns.DeleteHostNCApipaEndpointResponse{ + Response: cns.Response{ + ReturnCode: returnCode, + Message: returnMessage, + }, + } + + err = service.Listener.Encode(w, &response) + log.Response(service.Name, response, response.Response.ReturnCode, ReturnCodeToString(response.Response.ReturnCode), err) +} diff --git a/common/namedlock.go b/common/namedlock.go new file mode 100644 index 0000000000..68f0a8b625 --- /dev/null +++ b/common/namedlock.go @@ -0,0 +1,79 @@ +package common + +import ( + "sync" + + "github.com/Azure/azure-container-networking/log" +) + +// NamedLock holds a mutex and a map of locks. Mutex is used to +// get exclusive lock on the map while initializing the lock in the +// map. +type NamedLock struct { + mutex sync.Mutex + lockMap map[string]*refCountedLock +} + +// refCountedLock holds the lock and ref count for it +type refCountedLock struct { + mutex sync.RWMutex + refCount int +} + +// InitNamedLock initializes the named lock struct +func InitNamedLock() *NamedLock { + return &NamedLock{ + mutex: sync.Mutex{}, + lockMap: make(map[string]*refCountedLock), + } +} + +// LockAcquire acquires the lock with specified name +func (namedLock *NamedLock) LockAcquire(lockName string) { + namedLock.mutex.Lock() + _, ok := namedLock.lockMap[lockName] + if !ok { + namedLock.lockMap[lockName] = &refCountedLock{refCount: 0} + } + + namedLock.lockMap[lockName].AddRef() + namedLock.mutex.Unlock() + namedLock.lockMap[lockName].Lock() +} + +// LockRelease releases the lock with specified name +func (namedLock *NamedLock) LockRelease(lockName string) { + namedLock.mutex.Lock() + defer namedLock.mutex.Unlock() + + lock, ok := namedLock.lockMap[lockName] + if ok { + lock.Unlock() + lock.RemoveRef() + if lock.refCount == 0 { + delete(namedLock.lockMap, lockName) + } + } else { + log.Printf("[Azure CNS] Attempt to unlock: %s without acquiring the lock", lockName) + } +} + +// AddRef increments the ref count on the lock +func (refCountedLock *refCountedLock) AddRef() { + refCountedLock.refCount++ +} + +// RemoveRef decrements the ref count on the lock +func (refCountedLock *refCountedLock) RemoveRef() { + refCountedLock.refCount-- +} + +// Lock locks the named lock +func (refCountedLock *refCountedLock) Lock() { + refCountedLock.mutex.Lock() +} + +// Unlock unlocks the named lock +func (refCountedLock *refCountedLock) Unlock() { + refCountedLock.mutex.Unlock() +} diff --git a/network/endpoint.go b/network/endpoint.go index fa1755d061..8464105c55 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -35,6 +35,7 @@ type endpoint struct { EnableMultitenancy bool AllowInboundFromHostToNC bool AllowInboundFromNCToHost bool + NetworkContainerID string NetworkNameSpace string `json:",omitempty"` ContainerID string PODName string `json:",omitempty"` @@ -63,6 +64,7 @@ type EndpointInfo struct { EnableMultiTenancy bool AllowInboundFromHostToNC bool AllowInboundFromNCToHost bool + NetworkContainerID string PODName string PODNameSpace string Data map[string]interface{} @@ -202,11 +204,12 @@ func (ep *endpoint) getInfo() *EndpointInfo { EnableMultiTenancy: ep.EnableMultitenancy, AllowInboundFromHostToNC: ep.AllowInboundFromHostToNC, AllowInboundFromNCToHost: ep.AllowInboundFromNCToHost, - IfName: ep.IfName, - ContainerID: ep.ContainerID, - NetNsPath: ep.NetworkNameSpace, - PODName: ep.PODName, - PODNameSpace: ep.PODNameSpace, + IfName: ep.IfName, + ContainerID: ep.ContainerID, + NetNsPath: ep.NetworkNameSpace, + PODName: ep.PODName, + PODNameSpace: ep.PODNameSpace, + NetworkContainerID: ep.NetworkContainerID, } for _, route := range ep.Routes { diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index 86cf768264..22a5ccefb6 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -9,6 +9,7 @@ import ( "net" "strings" + "github.com/Azure/azure-container-networking/cns/cnsclient" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/network/policy" "github.com/Microsoft/hcsshim" @@ -205,6 +206,63 @@ func (nw *network) configureHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeE return hcnEndpoint, nil } +func (nw *network) deleteHostNCApipaEndpoint(networkContainerID string) error { + cnsClient, err := cnsclient.GetCnsClient() + if err != nil { + log.Errorf("Failed to get CNS client. Error %v", err) + return err + } + + log.Printf("[net] Deleting HostNCApipaEndpoint for network container: %s", networkContainerID) + err = cnsClient.DeleteHostNCApipaEndpoint(networkContainerID) + log.Printf("[net] Completed HostNCApipaEndpoint deletion for network container: %s"+ + " with error: %v", networkContainerID, err) + + return nil +} + +// createHostNCApipaEndpoint creates a new endpoint in the HostNCApipaNetwork +// for host container connectivity +func (nw *network) createHostNCApipaEndpoint(epInfo *EndpointInfo) error { + var ( + err error + cnsClient *cnsclient.CNSClient + hostNCApipaEndpointID string + namespace *hcn.HostComputeNamespace + ) + + if namespace, err = hcn.GetNamespaceByID(epInfo.NetNsPath); err != nil { + return fmt.Errorf("Failed to retrieve namespace with GetNamespaceByID for NetNsPath: %s"+ + " due to error: %v", epInfo.NetNsPath, err) + } + + if cnsClient, err = cnsclient.GetCnsClient(); err != nil { + log.Errorf("Failed to get CNS client. Error %v", err) + return err + } + + log.Printf("[net] Creating HostNCApipaEndpoint for host container connectivity for NC: %s", + epInfo.NetworkContainerID) + + if hostNCApipaEndpointID, err = + cnsClient.CreateHostNCApipaEndpoint(epInfo.NetworkContainerID); err != nil { + return err + } + + defer func() { + if err != nil { + nw.deleteHostNCApipaEndpoint(epInfo.NetworkContainerID) + } + }() + + if err = hcn.AddNamespaceEndpoint(namespace.Id, hostNCApipaEndpointID); err != nil { + return fmt.Errorf("[net] Failed to add HostNCApipaEndpoint: %s to namespace: %s due to error: %v", + hostNCApipaEndpointID, namespace.Id, err) + } + + return nil +} + // newEndpointImplHnsV2 creates a new endpoint in the network using HnsV2 func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) { hcnEndpoint, err := nw.configureHcnEndpoint(epInfo) @@ -240,6 +298,22 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) hnsResponse.Id, namespace.Id, err) } + defer func() { + if err != nil { + if errRemoveNsEp := hcn.RemoveNamespaceEndpoint(namespace.Id, hnsResponse.Id); errRemoveNsEp != nil { + log.Printf("[net] Failed to remove endpoint: %s from namespace: %s due to error: %v", + hnsResponse.Id, hnsResponse.Id, errRemoveNsEp) + } + } + }() + + // If the Host - container connectivity is requested, create endpoint in HostNCApipaNetwork + if epInfo.AllowInboundFromHostToNC || epInfo.AllowInboundFromNCToHost { + if err = nw.createHostNCApipaEndpoint(epInfo); err != nil { + return nil, fmt.Errorf("Failed to create HostNCApipaEndpoint due to error: %v", err) + } + } + var vlanid int if epInfo.Data != nil { if vlanData, ok := epInfo.Data[VlanIDKey]; ok { @@ -264,6 +338,9 @@ func (nw *network) newEndpointImplHnsV2(epInfo *EndpointInfo) (*endpoint, error) VlanID: vlanid, EnableSnatOnHost: epInfo.EnableSnatOnHost, NetNs: epInfo.NetNsPath, + AllowInboundFromNCToHost: epInfo.AllowInboundFromNCToHost, + AllowInboundFromHostToNC: epInfo.AllowInboundFromHostToNC, + NetworkContainerID: epInfo.NetworkContainerID, } for _, route := range epInfo.Routes { @@ -299,8 +376,17 @@ func (nw *network) deleteEndpointImplHnsV1(ep *endpoint) error { // deleteEndpointImplHnsV2 deletes an existing endpoint from the network using HNS v2. func (nw *network) deleteEndpointImplHnsV2(ep *endpoint) error { - var hcnEndpoint *hcn.HostComputeEndpoint - var err error + var ( + hcnEndpoint *hcn.HostComputeEndpoint + err error + ) + + if ep.AllowInboundFromHostToNC || ep.AllowInboundFromNCToHost { + if err = nw.deleteHostNCApipaEndpoint(ep.NetworkContainerID); err != nil { + log.Errorf("[net] Failed to delete HostNCApipaEndpoint due to error: %v", err) + return err + } + } log.Printf("[net] Deleting hcn endpoint with id: %s", ep.HnsId)