Add Host NC communication support in Windows with HnsV2 (#424)

This PR adds support for host NC bidirectional communication with windows HnsV2. This is supported in multitenant scenario only. AllowHostToNCCommunication and AllowNCToHostCommunication flags are used to enable Host to NC and NC to host communication respectively.

Makefile

@@ -41,6 +41,7 @@ CNSFILES = \
 	$(wildcard cns/dockerclient/*.go) \
 	$(wildcard cns/imdsclient/*.go) \
 	$(wildcard cns/ipamclient/*.go) \
+	$(wildcard cns/hnsclient/*.go) \
 	$(wildcard cns/restserver/*.go) \
 	$(wildcard cns/routes/*.go) \
 	$(wildcard cns/service/*.go) \

cni/network/multitenancy.go

@@ -64,9 +64,9 @@ func getContainerNetworkConfigurationInternal(
 	namespace string,
 	podName string,
 	ifName string) (*cniTypesCurr.Result, *cns.GetNetworkContainerResponse, net.IPNet, error) {
-	cnsClient, err := cnsclient.NewCnsClient(address)
+	cnsClient, err := cnsclient.GetCnsClient()
 	if err != nil {
-		log.Printf("Initializing CNS client error %v", err)
+		log.Printf("Failed to get CNS client. Error: %v", err)
 		return nil, nil, net.IPNet{}, err
 	}
 

cni/network/network.go

@@ -242,6 +242,11 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 		return err
 	}
 
+	if nwCfg.MultiTenancy {
+		// Initialize CNSClient
+		cnsclient.InitCnsClient(nwCfg.CNSUrl)
+	}
+
 	k8sContainerID := args.ContainerID
 	if len(k8sContainerID) == 0 {
 		errMsg := "Container ID not specified in CNI Args"
@@ -552,6 +557,11 @@ func (plugin *netPlugin) Get(args *cniSkel.CmdArgs) error {
 		return err
 	}
 
+	if nwCfg.MultiTenancy {
+		// Initialize CNSClient
+		cnsclient.InitCnsClient(nwCfg.CNSUrl)
+	}
+
 	// Initialize values from network config.
 	if networkId, err = getNetworkName(k8sPodName, k8sNamespace, args.IfName, nwCfg); err != nil {
 		log.Printf("[cni-net] Failed to extract network name from network config. error: %v", err)
@@ -627,6 +637,11 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error {
 		log.Printf("[cni-net] Failed to get POD info due to error: %v", err)
 	}
 
+	if nwCfg.MultiTenancy {
+		// Initialize CNSClient
+		cnsclient.InitCnsClient(nwCfg.CNSUrl)
+	}
+
 	// Initialize values from network config.
 	if networkId, err = getNetworkName(k8sPodName, k8sNamespace, args.IfName, nwCfg); err != nil {
 		log.Printf("[cni-net] Failed to extract network name from network config. error: %v", err)
@@ -772,7 +787,7 @@ func (plugin *netPlugin) Update(args *cniSkel.CmdArgs) error {
 
 	// now query CNS to get the target routes that should be there in the networknamespace (as a result of update)
 	log.Printf("Going to collect target routes for [name=%v, namespace=%v] from CNS.", k8sPodName, k8sNamespace)
-	if cnsClient, err = cnsclient.NewCnsClient(nwCfg.CNSUrl); err != nil {
+	if cnsClient, err = cnsclient.InitCnsClient(nwCfg.CNSUrl); err != nil {
 		log.Printf("Initializing CNS client error in CNI Update%v", err)
 		log.Printf(err.Error())
 		return plugin.Errorf(err.Error())

cni/network/network_linux.go

@@ -58,6 +58,7 @@ func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *ne
 		epInfo.Data[network.SnatBridgeIPKey] = cnsNwConfig.LocalIPConfiguration.GatewayIPAddress + "/" + strconv.Itoa(int(cnsNwConfig.LocalIPConfiguration.IPSubnet.PrefixLength))
 		epInfo.AllowInboundFromHostToNC = cnsNwConfig.AllowHostToNCCommunication
 		epInfo.AllowInboundFromNCToHost = cnsNwConfig.AllowNCToHostCommunication
+		epInfo.NetworkContainerID = cnsNwConfig.NetworkContainerID
 	}
 
 	epInfo.Data[network.OptVethName] = vethName

cni/network/network_windows.go

@@ -93,6 +93,9 @@ func setEndpointOptions(cnsNwConfig *cns.GetNetworkContainerResponse, epInfo *ne
 			cnetAddressMap = append(cnetAddressMap, ipSubnet.IPAddress+"/"+strconv.Itoa(int(ipSubnet.PrefixLength)))
 		}
 		epInfo.Data[network.CnetAddressSpace] = cnetAddressMap
+		epInfo.AllowInboundFromHostToNC = cnsNwConfig.AllowHostToNCCommunication
+		epInfo.AllowInboundFromNCToHost = cnsNwConfig.AllowNCToHostCommunication
+		epInfo.NetworkContainerID = cnsNwConfig.NetworkContainerID
 	}
 }
 

cns/NetworkContainerContract.go

@@ -131,6 +131,7 @@ type GetNetworkContainerRequest struct {
 
 // GetNetworkContainerResponse describes the response to retrieve a specifc network container.
 type GetNetworkContainerResponse struct {
+	NetworkContainerID         string
 	IPConfiguration            IPConfiguration
 	Routes                     []Route
 	CnetAddressSpace           []IPSubnet

cns/api.go

@@ -7,20 +7,22 @@ import "encoding/json"
 
 // Container Network Service remote API Contract
 const (
-	SetEnvironmentPath          = "/network/environment"
-	CreateNetworkPath           = "/network/create"
-	DeleteNetworkPath           = "/network/delete"
-	CreateHnsNetworkPath        = "/network/hns/create"
-	DeleteHnsNetworkPath        = "/network/hns/delete"
-	ReserveIPAddressPath        = "/network/ip/reserve"
-	ReleaseIPAddressPath        = "/network/ip/release"
-	GetHostLocalIPPath          = "/network/ip/hostlocal"
-	GetIPAddressUtilizationPath = "/network/ip/utilization"
-	GetUnhealthyIPAddressesPath = "/network/ipaddresses/unhealthy"
-	GetHealthReportPath         = "/network/health"
-	NumberOfCPUCoresPath        = "/hostcpucores"
-	V1Prefix                    = "/v0.1"
-	V2Prefix                    = "/v0.2"
+	SetEnvironmentPath            = "/network/environment"
+	CreateNetworkPath             = "/network/create"
+	DeleteNetworkPath             = "/network/delete"
+	CreateHnsNetworkPath          = "/network/hns/create"
+	DeleteHnsNetworkPath          = "/network/hns/delete"
+	ReserveIPAddressPath          = "/network/ip/reserve"
+	ReleaseIPAddressPath          = "/network/ip/release"
+	GetHostLocalIPPath            = "/network/ip/hostlocal"
+	GetIPAddressUtilizationPath   = "/network/ip/utilization"
+	GetUnhealthyIPAddressesPath   = "/network/ipaddresses/unhealthy"
+	GetHealthReportPath           = "/network/health"
+	NumberOfCPUCoresPath          = "/hostcpucores"
+	CreateHostNCApipaEndpointPath = "/network/createhostncapipaendpoint"
+	DeleteHostNCApipaEndpointPath = "/network/deletehostncapipaendpoint"
+	V1Prefix                      = "/v0.1"
+	V2Prefix                      = "/v0.2"
 )
 
 // SetEnvironmentRequest describes the Request to set the environment in CNS.
@@ -153,3 +155,27 @@ type OptionMap map[string]interface{}
 type errorResponse struct {
 	Err string
 }
+
+// CreateHostNCApipaEndpointRequest describes request for create apipa endpoint
+// for host container connectivity for the given network container
+type CreateHostNCApipaEndpointRequest struct {
+	NetworkContainerID string
+}
+
+// CreateHostNCApipaEndpointResponse describes response for create apipa endpoint request
+// for host container connectivity.
+type CreateHostNCApipaEndpointResponse struct {
+	Response   Response
+	EndpointID string
+}
+
+// DeleteHostNCApipaEndpointRequest describes request for deleting apipa endpoint created
+// for host NC connectivity.
+type DeleteHostNCApipaEndpointRequest struct {
+	NetworkContainerID string
+}
+
+// DeleteHostNCApipaEndpointResponse describes response for delete host NC apipa endpoint request.
+type DeleteHostNCApipaEndpointResponse struct {
+	Response Response
+}

cns/cnsclient/cnsclient.go

@@ -19,15 +19,34 @@ const (
 	defaultCnsURL = "http://localhost:10090"
 )
 
-// NewCnsClient create a new cns client.
-func NewCnsClient(url string) (*CNSClient, error) {
-	if url == "" {
-		url = defaultCnsURL
+var (
+	cnsClient *CNSClient
+)
+
+// InitCnsClient initializes new cns client and returns the object
+func InitCnsClient(url string) (*CNSClient, error) {
+	if cnsClient == nil {
+		if url == "" {
+			url = defaultCnsURL
+		}
+
+		cnsClient = &CNSClient{
+			connectionURL: url,
+		}
 	}
 
-	return &CNSClient{
-		connectionURL: url,
-	}, nil
+	return cnsClient, nil
+}
+
+// GetCnsClient returns the cns client object
+func GetCnsClient() (*CNSClient, error) {
+	var err error
+
+	if cnsClient == nil {
+		err = fmt.Errorf("[Azure CNSClient] CNS Client not initialized")
+	}
+
+	return cnsClient, err
 }
 
 // GetNetworkConfiguration Request to get network config.
@@ -77,3 +96,105 @@ func (cnsClient *CNSClient) GetNetworkConfiguration(orchestratorContext []byte)
 
 	return &resp, nil
 }
+
+// CreateHostNCApipaEndpoint creates an endpoint in APIPA network for host container connectivity.
+func (cnsClient *CNSClient) CreateHostNCApipaEndpoint(
+	networkContainerID string) (string, error) {
+	var (
+		err  error
+		body bytes.Buffer
+	)
+
+	httpc := &http.Client{}
+	url := cnsClient.connectionURL + cns.CreateHostNCApipaEndpointPath
+	log.Printf("CreateHostNCApipaEndpoint url: %v for NC: %s", url, networkContainerID)
+
+	payload := &cns.CreateHostNCApipaEndpointRequest{
+		NetworkContainerID: networkContainerID,
+	}
+
+	if err = json.NewEncoder(&body).Encode(payload); err != nil {
+		log.Errorf("encoding json failed with %v", err)
+		return "", err
+	}
+
+	res, err := httpc.Post(url, "application/json", &body)
+	if err != nil {
+		log.Errorf("[Azure CNSClient] HTTP Post returned error %v", err.Error())
+		return "", err
+	}
+
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		errMsg := fmt.Sprintf("[Azure CNSClient] CreateHostNCApipaEndpoint: Invalid http status code: %v",
+			res.StatusCode)
+		log.Errorf(errMsg)
+		return "", fmt.Errorf(errMsg)
+	}
+
+	var resp cns.CreateHostNCApipaEndpointResponse
+
+	if err = json.NewDecoder(res.Body).Decode(&resp); err != nil {
+		log.Errorf("[Azure CNSClient] Error parsing CreateHostNCApipaEndpoint response resp: %v err: %v",
+			res.Body, err.Error())
+		return "", err
+	}
+
+	if resp.Response.ReturnCode != 0 {
+		log.Errorf("[Azure CNSClient] CreateHostNCApipaEndpoint received error response :%v", resp.Response.Message)
+		return "", fmt.Errorf(resp.Response.Message)
+	}
+
+	return resp.EndpointID, nil
+}
+
+// DeleteHostNCApipaEndpoint deletes the endpoint in APIPA network created for host container connectivity.
+func (cnsClient *CNSClient) DeleteHostNCApipaEndpoint(networkContainerID string) error {
+	var body bytes.Buffer
+
+	httpc := &http.Client{}
+	url := cnsClient.connectionURL + cns.DeleteHostNCApipaEndpointPath
+	log.Printf("DeleteHostNCApipaEndpoint url: %v for NC: %s", url, networkContainerID)
+
+	payload := &cns.DeleteHostNCApipaEndpointRequest{
+		NetworkContainerID: networkContainerID,
+	}
+
+	err := json.NewEncoder(&body).Encode(payload)
+	if err != nil {
+		log.Errorf("encoding json failed with %v", err)
+		return err
+	}
+
+	res, err := httpc.Post(url, "application/json", &body)
+	if err != nil {
+		log.Errorf("[Azure CNSClient] HTTP Post returned error %v", err.Error())
+		return err
+	}
+
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		errMsg := fmt.Sprintf("[Azure CNSClient] DeleteHostNCApipaEndpoint: Invalid http status code: %v",
+			res.StatusCode)
+		log.Errorf(errMsg)
+		return fmt.Errorf(errMsg)
+	}
+
+	var resp cns.DeleteHostNCApipaEndpointResponse
+
+	err = json.NewDecoder(res.Body).Decode(&resp)
+	if err != nil {
+		log.Errorf("[Azure CNSClient] Error parsing DeleteHostNCApipaEndpoint response resp: %v err: %v",
+			res.Body, err.Error())
+		return err
+	}
+
+	if resp.Response.ReturnCode != 0 {
+		log.Errorf("[Azure CNSClient] DeleteHostNCApipaEndpoint received error response :%v", resp.Response.Message)
+		return fmt.Errorf(resp.Response.Message)
+	}
+
+	return nil
+}

cns/hnsclient/hnsclient_linux.go

@@ -30,3 +30,22 @@ func CreateHnsNetwork(nwConfig cns.CreateHnsNetworkRequest) error {
 func DeleteHnsNetwork(networkName string) error {
 	return fmt.Errorf("DeleteHnsNetwork shouldn't be called for linux platform")
 }
+
+// CreateHostNCApipaEndpoint creates the endpoint in the apipa network
+// for host container connectivity
+// This is windows platform specific.
+func CreateHostNCApipaEndpoint(
+	networkContainerID string,
+	localIPConfiguration cns.IPConfiguration,
+	allowNCToHostCommunication bool,
+	allowHostToNCCommunication bool) (string, error) {
+	return "", nil
+}
+
+// DeleteHostNCApipaEndpoint deletes the endpoint in the apipa network
+// created for host container connectivity
+// This is windows platform specific.
+func DeleteHostNCApipaEndpoint(
+	networkContainerID string) error {
+	return nil
+}