config pnet on smc-tool & smc module installed

Signed-off-by: bingshen.wbs <[email protected]>

pkg/smc/pnet.go

@@ -0,0 +1,83 @@
+package smc
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+
+	"github.com/containernetworking/plugins/pkg/ns"
+	"github.com/vishvananda/netlink"
+)
+
+const (
+	smcPnet = "smc_pnet"
+)
+
+func supportSMCR() bool {
+	// smc module is load
+	_, err := os.Stat("/proc/sys/net/smc/tcp2smc")
+	if err != nil {
+		return false
+	}
+	// rdma device attached
+	rdmaLink, err := netlink.RdmaLinkList()
+	if err != nil || len(rdmaLink) == 0 {
+		return false
+	}
+	// smc-tools installed
+	_, err = exec.LookPath(smcPnet)
+	return err == nil
+}
+
+func pnetID(name string) string {
+	nameSlice := strings.Split(name, "_")
+	return nameSlice[len(nameSlice)-1]
+}
+
+// fixme: reflect to netlink
+func ensureForERDMADev(name string) error {
+	output, err := exec.Command(smcPnet, "-s").CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("failed to get smc-pnet stat: %v, output: %v", err, string(output))
+	}
+	if bytes.Contains(output, []byte(name)) {
+		return nil
+	}
+	output, err = exec.Command(smcPnet, "-a", pnetID(name), "-D", name).CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("failed to config smc-pnet rdma device: %v, output: %v", err, string(output))
+	}
+	return nil
+}
+
+func ensureForNetDevice(erdmaDev string, netDevice string) error {
+	output, err := exec.Command(smcPnet, "-s").CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("failed to get smc-pnet stat for net device: %v, output: %v", err, string(output))
+	}
+	if bytes.Contains(output, []byte(netDevice)) {
+		return nil
+	}
+	output, err = exec.Command(smcPnet, "-a", pnetID(erdmaDev), "-I", netDevice).CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("failed to config smc-pnet net device: %v, output: %v", err, string(output))
+	}
+	return nil
+}
+
+func ConfigSMCForDevice(erdmaDev string, netDevice string, netns ns.NetNS) error {
+	if !supportSMCR() {
+		return nil
+	}
+	if err := ensureForERDMADev(erdmaDev); err != nil {
+		return err
+	}
+	if netns != nil {
+		return netns.Do(func(ns ns.NetNS) error {
+			return ensureForNetDevice(erdmaDev, netDevice)
+		})
+	}
+	return ensureForNetDevice(erdmaDev, netDevice)
+}

plugin/datapath/policy_router_linux.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"net"
 
+	"github.com/AliyunContainerService/terway/pkg/smc"
+
 	"github.com/AliyunContainerService/terway/plugin/driver/nic"
 	"github.com/AliyunContainerService/terway/plugin/driver/types"
 	"github.com/AliyunContainerService/terway/plugin/driver/utils"
@@ -348,6 +350,13 @@ func (d *PolicyRoute) Setup(cfg *types.SetupConfig, netNS ns.NetNS) error {
 		return fmt.Errorf("setup container, %w", err)
 	}
 
+	if cfg.ERDMA {
+		err = smc.ConfigSMCForDevice("erdma_0", cfg.ContainerIfName, netNS)
+		if err != nil {
+			return fmt.Errorf("error setup pnet config for pod: %w", err)
+		}
+	}
+
 	if cfg.EnableNetworkPriority {
 		err = utils.SetEgressPriority(eni, cfg.NetworkPriority, cfg.ContainerIPNet)
 		if err != nil {