diff --git a/policy/cilium/0032-fix-disable-per-package-lb.patch b/policy/cilium/0032-fix-disable-per-package-lb.patch new file mode 100644 index 00000000..a990048c --- /dev/null +++ b/policy/cilium/0032-fix-disable-per-package-lb.patch @@ -0,0 +1,101 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: l1b0k +Date: Sat, 12 Oct 2024 09:33:24 +0800 +Subject: fix disable per-package lb + +Signed-off-by: l1b0k +--- + bpf/bpf_lxc.c | 5 ++--- + daemon/cmd/daemon_main.go | 3 +++ + pkg/datapath/linux/config/config.go | 5 +++++ + pkg/option/config.go | 6 ++++++ + 4 files changed, 16 insertions(+), 3 deletions(-) + +diff --git a/bpf/bpf_lxc.c b/bpf/bpf_lxc.c +index 2773e2c276..7e424204e2 100644 +--- a/bpf/bpf_lxc.c ++++ b/bpf/bpf_lxc.c +@@ -58,9 +58,9 @@ + * Most services with L7 LB flag can not be redirected to their proxy port + * in bpf_sock, so we must check for those via per packet LB as well. + */ +-#if !defined(ENABLE_SOCKET_LB_FULL) || \ ++#if (!defined(ENABLE_SOCKET_LB_FULL) || \ + defined(ENABLE_SOCKET_LB_HOST_ONLY) || \ +- defined(ENABLE_L7_LB) ++ defined(ENABLE_L7_LB) ) && !defined(DISABLE_PER_PACKET_LB) + # define ENABLE_PER_PACKET_LB 1 + #endif + +@@ -1247,7 +1247,6 @@ static __always_inline int __tail_handle_ipv4(struct __ctx_buff *ctx) + + if (unlikely(!is_valid_lxc_src_ipv4(ip4))) + return DROP_INVALID_SIP; +- + #ifdef ENABLE_PER_PACKET_LB + { + struct ipv4_ct_tuple tuple = {}; +diff --git a/daemon/cmd/daemon_main.go b/daemon/cmd/daemon_main.go +index de2ad4900b..8b495d972f 100644 +--- a/daemon/cmd/daemon_main.go ++++ b/daemon/cmd/daemon_main.go +@@ -1147,6 +1147,9 @@ func initializeFlags() { + flags.MarkHidden(option.EnableStaleCiliumEndpointCleanup) + option.BindEnv(option.EnableStaleCiliumEndpointCleanup) + ++ flags.Bool(option.DisablePerPacketLB, false, "Disable svc func.") ++ option.BindEnv(option.DisablePerPacketLB) ++ + viper.BindPFlags(flags) + } + +diff --git a/pkg/datapath/linux/config/config.go b/pkg/datapath/linux/config/config.go +index 060328adf9..4201e878fb 100644 +--- a/pkg/datapath/linux/config/config.go ++++ b/pkg/datapath/linux/config/config.go +@@ -98,6 +98,11 @@ func (h *HeaderfileWriter) WriteNodeConfig(w io.Writer, cfg *datapath.LocalNodeC + + cDefinesMap["KERNEL_HZ"] = fmt.Sprintf("%d", option.Config.KernelHz) + ++ if option.Config.DisablePerPacketLB { ++ log.Infof("Disabling per-packet LB policy") ++ cDefinesMap["DISABLE_PER_PACKET_LB"] = "1" ++ } ++ + if option.Config.EnableIPv6 { + extraMacrosMap["ROUTER_IP"] = routerIP.String() + fw.WriteString(defineIPv6("ROUTER_IP", routerIP)) +diff --git a/pkg/option/config.go b/pkg/option/config.go +index d4d9643f4e..80b5eb2c7c 100644 +--- a/pkg/option/config.go ++++ b/pkg/option/config.go +@@ -1104,6 +1104,8 @@ const ( + // EnableStaleCiliumEndpointCleanup sets whether Cilium should perform cleanup of + // stale CiliumEndpoints during init. + EnableStaleCiliumEndpointCleanup = "enable-stale-cilium-endpoint-cleanup" ++ ++ DisablePerPacketLB = "disable-per-package-lb" + ) + + // Default string arguments +@@ -2262,6 +2264,8 @@ type DaemonConfig struct { + // This will attempt to remove local CiliumEndpoints that are not managed by Cilium + // following Endpoint restoration. + EnableStaleCiliumEndpointCleanup bool ++ ++ DisablePerPacketLB bool + } + + var ( +@@ -3257,6 +3261,8 @@ func (c *DaemonConfig) Populate() { + c.EnableK8sTerminatingEndpoint = viper.GetBool(EnableK8sTerminatingEndpoint) + c.EnableStaleCiliumEndpointCleanup = viper.GetBool(EnableStaleCiliumEndpointCleanup) + ++ c.DisablePerPacketLB = viper.GetBool(DisablePerPacketLB) ++ + // Disable Envoy version check if L7 proxy is disabled. + c.DisableEnvoyVersionCheck = viper.GetBool(DisableEnvoyVersionCheck) + if !c.EnableL7Proxy { +-- +2.46.0 + diff --git a/policy/policyinit.sh b/policy/policyinit.sh index 2d151580..2f55b7a5 100755 --- a/policy/policyinit.sh +++ b/policy/policyinit.sh @@ -5,19 +5,19 @@ mount -o remount rw /proc/sys export DATASTORE_TYPE=kubernetes masq_eni_only() { - if ! "$1" -t nat -L terway-masq; then - # Create a new chain in nat table. - "$1" -t nat -N terway-masq - fi + if ! "$1" -t nat -L terway-masq; then + # Create a new chain in nat table. + "$1" -t nat -N terway-masq + fi - if ! "$1" -t nat -L POSTROUTING | grep -q terway-masq; then - # Append that chain to POSTROUTING table. - "$1" -t nat -A POSTROUTING -m comment --comment "terway:masq-outgoing" ! -o lo -j terway-masq - fi + if ! "$1" -t nat -L POSTROUTING | grep -q terway-masq; then + # Append that chain to POSTROUTING table. + "$1" -t nat -A POSTROUTING -m comment --comment "terway:masq-outgoing" ! -o lo -j terway-masq + fi - if ! "$1" -t nat -L terway-masq | grep -q MASQUERADE; then - "$1" -t nat -A terway-masq -j MASQUERADE - fi + if ! "$1" -t nat -L terway-masq | grep -q MASQUERADE; then + "$1" -t nat -A terway-masq -j MASQUERADE + fi } terway_config_val() { @@ -51,16 +51,16 @@ if grep -q "cni_exclusive_eni *= *eniOnly" "$node_capabilities"; then # for health check if [ "$FELIX_HEALTHPORT" != "" ]; then - # shellcheck disable=SC2016 - exec socat TCP-LISTEN:"$FELIX_HEALTHPORT",bind=127.0.0.1,fork,reuseaddr system:'sleep 2;kill -9 $SOCAT_PID 2>/dev/null' + # shellcheck disable=SC2016 + exec socat TCP-LISTEN:"$FELIX_HEALTHPORT",bind=127.0.0.1,fork,reuseaddr system:'sleep 2;kill -9 $SOCAT_PID 2>/dev/null' else - # shellcheck disable=SC2016 - exec socat TCP-LISTEN:9099,bind=127.0.0.1,fork,reuseaddr system:'sleep 2;kill -9 $SOCAT_PID 2>/dev/null' + # shellcheck disable=SC2016 + exec socat TCP-LISTEN:9099,bind=127.0.0.1,fork,reuseaddr system:'sleep 2;kill -9 $SOCAT_PID 2>/dev/null' fi fi if grep -q "datapath *= *datapathv2" "$node_capabilities"; then - datapath_mode=veth + datapath_mode=veth fi # kernel version has already checked in initContainer, so just determine whether plugin chaining exists @@ -69,7 +69,7 @@ if [ "$virtyal_type" = "ipvlan" ] || [ "$virtyal_type" = "datapathv2" ]; then # kernel version equal and above 4.19 if { [ "$KERNEL_MAJOR_VERSION" -eq 4 ] && [ "$KERNEL_MINOR_VERSION" -ge 19 ]; } || - [ "$KERNEL_MAJOR_VERSION" -gt 4 ]; then + [ "$KERNEL_MAJOR_VERSION" -gt 4 ]; then extra_args=$(terway_config_val 'cilium_args') if [ -z "$DISABLE_POLICY" ] || [ "$DISABLE_POLICY" = "false" ] || [ "$DISABLE_POLICY" = "0" ]; then @@ -95,106 +95,115 @@ if [ "$virtyal_type" = "ipvlan" ] || [ "$virtyal_type" = "datapathv2" ]; then echo "turning up hubble, passing args \"${extra_args}\"" fi - if [ "$IN_CLUSTER_LOADBALANCE" = "true" ]; then - extra_args="${extra_args} --enable-in-cluster-loadbalance=true " - echo "turning up in cluster loadbalance, passing args \"${extra_args}\"" - fi + if [ "$IN_CLUSTER_LOADBALANCE" = "true" ]; then + extra_args="${extra_args} --enable-in-cluster-loadbalance=true " + echo "turning up in cluster loadbalance, passing args \"${extra_args}\"" + fi - if bpftool -j feature probe | grep bpf_skb_ecn_set_ce ; then - extra_args="${extra_args} --enable-bandwidth-manager=true " - fi + if bpftool -j feature probe | grep bpf_skb_ecn_set_ce; then + extra_args="${extra_args} --enable-bandwidth-manager=true " + fi echo "using cilium as network routing & policy" # shellcheck disable=SC2086 exec cilium-agent --tunnel=disabled --enable-ipv4-masquerade=false --enable-ipv6-masquerade=false \ - --enable-policy=$ENABLE_POLICY \ - --agent-health-port=9099 --disable-envoy-version-check=true \ - --enable-local-node-route=false --ipv4-range=169.254.10.0/30 --ipv6-range=fe80:2400:3200:baba::/30 --enable-endpoint-health-checking=false \ - --enable-health-checking=false --enable-service-topology=true --disable-cnp-status-updates=true --k8s-heartbeat-timeout=0 --enable-session-affinity=true \ - --install-iptables-rules=false --enable-l7-proxy=false \ - --ipam=cluster-pool --datapath-mode=${datapath_mode} --enable-runtime-device-detection=true ${extra_args} + --enable-policy=$ENABLE_POLICY \ + --agent-health-port=9099 --disable-envoy-version-check=true \ + --enable-local-node-route=false --ipv4-range=169.254.10.0/30 --ipv6-range=fe80:2400:3200:baba::/30 --enable-endpoint-health-checking=false \ + --enable-health-checking=false --enable-service-topology=true --disable-cnp-status-updates=true --k8s-heartbeat-timeout=0 --enable-session-affinity=true \ + --install-iptables-rules=false --enable-l7-proxy=false \ + --ipam=cluster-pool --datapath-mode=${datapath_mode} --enable-runtime-device-detection=true ${extra_args} fi fi - # shellcheck disable=SC1091 - source uninstall_policy.sh - - # check kernel version +# shellcheck disable=SC1091 +source uninstall_policy.sh - export FELIX_IPTABLESBACKEND=Auto - if ( uname -r | grep -E "el7|an7" && [ "${KERNEL_MAJOR_VERSION}" -eq 3 ] ) || ( uname -r | grep -E "al7" && [ "${KERNEL_MAJOR_VERSION}" -eq 4 ] ); then - export FELIX_IPTABLESBACKEND=Legacy - elif ( uname -r | grep -E "el8|an8" && [ "${KERNEL_MAJOR_VERSION}" -ge 4 ] ) || ( uname -r | grep -E "al8|lifsea8" && [ "${KERNEL_MAJOR_VERSION}" -ge 5 ] ); then - export FELIX_IPTABLESBACKEND=NFT +# check kernel version - # clean legacy rules if exist - cleanup_legacy - fi +export FELIX_IPTABLESBACKEND=Auto +if (uname -r | grep -E "el7|an7" && [ "${KERNEL_MAJOR_VERSION}" -eq 3 ]) || (uname -r | grep -E "al7" && [ "${KERNEL_MAJOR_VERSION}" -eq 4 ]); then + export FELIX_IPTABLESBACKEND=Legacy +elif (uname -r | grep -E "el8|an8" && [ "${KERNEL_MAJOR_VERSION}" -ge 4 ]) || (uname -r | grep -E "al8|lifsea8" && [ "${KERNEL_MAJOR_VERSION}" -ge 5 ]); then + export FELIX_IPTABLESBACKEND=NFT - # default for veth - export FELIX_LOGSEVERITYSYS=none - export FELIX_LOGSEVERITYSCREEN=info - export CALICO_NETWORKING_BACKEND=none - export CLUSTER_TYPE=k8s,aliyun - export CALICO_DISABLE_FILE_LOGGING=true - # shellcheck disable=SC2154 - export CALICO_IPV4POOL_CIDR="${Network}" - export FELIX_IPTABLESREFRESHINTERVAL="${IPTABLESREFRESHINTERVAL:-60}" - export FELIX_IPV6SUPPORT=true - export WAIT_FOR_DATASTORE=true - export IP="" - export NO_DEFAULT_POOLS=true - export FELIX_DEFAULTENDPOINTTOHOSTACTION=ACCEPT - export FELIX_HEALTHENABLED=true - export FELIX_LOGFILEPATH=/dev/null - export FELIX_BPFENABLED=false - export FELIX_XDPENABLED=false - export FELIX_BPFCONNECTTIMELOADBALANCINGENABLED=false - export FELIX_BPFKUBEPROXYIPTABLESCLEANUPENABLED=false - exec 2>&1 - if [ -n "$NODENAME" ]; then - export FELIX_FELIXHOSTNAME="$NODENAME" - fi - if [ -n "$DATASTORE_TYPE" ]; then - export FELIX_DATASTORETYPE="$DATASTORE_TYPE" - fi + # clean legacy rules if exist + cleanup_legacy +fi - if [ "$network_policy_provider" = "ebpf" ]; then - cleanup_felix - # kernel version equal and above 4.19 - if { [ "$KERNEL_MAJOR_VERSION" -eq 4 ] && [ "$KERNEL_MINOR_VERSION" -ge 19 ]; } || - [ "$KERNEL_MAJOR_VERSION" -gt 4 ]; then - - if [ -z "$DISABLE_POLICY" ] || [ "$DISABLE_POLICY" = "false" ] || [ "$DISABLE_POLICY" = "0" ]; then - ENABLE_POLICY="default" - else - ENABLE_POLICY="never" - fi - # shellcheck disable=SC2086 - exec cilium-agent --kube-proxy-replacement=disabled --tunnel=disabled --enable-ipv4-masquerade=false --enable-ipv6-masquerade=false \ - --enable-policy=$ENABLE_POLICY \ - --agent-health-port=9099 --disable-envoy-version-check=true \ - --enable-local-node-route=false --ipv4-range=169.254.10.0/30 --ipv6-range=fe80:2400:3200:baba::/30 --enable-endpoint-health-checking=false \ - --enable-health-checking=false --enable-service-topology=true --disable-cnp-status-updates=true --k8s-heartbeat-timeout=0 --enable-session-affinity=true \ - --install-iptables-rules=false --enable-l7-proxy=false \ - --ipam=cluster-pool - else - echo "unsupported kernel version" - exit 1 - fi - else - if [ -z "$DISABLE_POLICY" ] || [ "$DISABLE_POLICY" = "false" ] || [ "$DISABLE_POLICY" = "0" ]; then - exec calico-felix - fi - fi +# default for veth +export FELIX_LOGSEVERITYSYS=none +export FELIX_LOGSEVERITYSCREEN=info +export CALICO_NETWORKING_BACKEND=none +export CLUSTER_TYPE=k8s,aliyun +export CALICO_DISABLE_FILE_LOGGING=true +# shellcheck disable=SC2154 +export CALICO_IPV4POOL_CIDR="${Network}" +export FELIX_IPTABLESREFRESHINTERVAL="${IPTABLESREFRESHINTERVAL:-60}" +export FELIX_IPV6SUPPORT=true +export WAIT_FOR_DATASTORE=true +export IP="" +export NO_DEFAULT_POOLS=true +export FELIX_DEFAULTENDPOINTTOHOSTACTION=ACCEPT +export FELIX_HEALTHENABLED=true +export FELIX_LOGFILEPATH=/dev/null +export FELIX_BPFENABLED=false +export FELIX_XDPENABLED=false +export FELIX_BPFCONNECTTIMELOADBALANCINGENABLED=false +export FELIX_BPFKUBEPROXYIPTABLESCLEANUPENABLED=false +exec 2>&1 +if [ -n "$NODENAME" ]; then + export FELIX_FELIXHOSTNAME="$NODENAME" +fi +if [ -n "$DATASTORE_TYPE" ]; then + export FELIX_DATASTORETYPE="$DATASTORE_TYPE" +fi - config_masquerade +if [ "$network_policy_provider" = "ebpf" ]; then cleanup_felix - # for health check - if [ "$FELIX_HEALTHPORT" != "" ]; then - # shellcheck disable=SC2016 - exec socat TCP-LISTEN:"$FELIX_HEALTHPORT",bind=127.0.0.1,fork,reuseaddr system:'sleep 2;kill -9 $SOCAT_PID 2>/dev/null' + # kernel version equal and above 4.19 + if { [ "$KERNEL_MAJOR_VERSION" -eq 4 ] && [ "$KERNEL_MINOR_VERSION" -ge 19 ]; } || + [ "$KERNEL_MAJOR_VERSION" -gt 4 ]; then + + extra_args=$(terway_config_val 'cilium_args') + + if [ -z "$DISABLE_POLICY" ] || [ "$DISABLE_POLICY" = "false" ] || [ "$DISABLE_POLICY" = "0" ]; then + ENABLE_POLICY="default" + else + ENABLE_POLICY="never" + extra_args="${extra_args} --labels=k8s:io\\.kubernetes\\.pod\\.namespace " + fi + + if [ "$IN_CLUSTER_LOADBALANCE" = "true" ]; then + extra_args="${extra_args} --enable-in-cluster-loadbalance=true " + echo "turning up in cluster loadbalance, passing args \"${extra_args}\"" + fi + + # shellcheck disable=SC2086 + exec cilium-agent --kube-proxy-replacement=disabled --tunnel=disabled --enable-ipv4-masquerade=false --enable-ipv6-masquerade=false \ + --enable-policy=$ENABLE_POLICY \ + --agent-health-port=9099 --disable-envoy-version-check=true \ + --enable-local-node-route=false --ipv4-range=169.254.10.0/30 --ipv6-range=fe80:2400:3200:baba::/30 --enable-endpoint-health-checking=false \ + --enable-health-checking=false --enable-service-topology=true --disable-cnp-status-updates=true --k8s-heartbeat-timeout=0 --enable-session-affinity=true \ + --install-iptables-rules=false --enable-l7-proxy=false \ + --ipam=cluster-pool ${extra_args} else - # shellcheck disable=SC2016 - exec socat TCP-LISTEN:9099,bind=127.0.0.1,fork,reuseaddr system:'sleep 2;kill -9 $SOCAT_PID 2>/dev/null' + echo "unsupported kernel version" + exit 1 + fi +else + if [ -z "$DISABLE_POLICY" ] || [ "$DISABLE_POLICY" = "false" ] || [ "$DISABLE_POLICY" = "0" ]; then + exec calico-felix fi +fi + +config_masquerade +cleanup_felix +# for health check +if [ "$FELIX_HEALTHPORT" != "" ]; then + # shellcheck disable=SC2016 + exec socat TCP-LISTEN:"$FELIX_HEALTHPORT",bind=127.0.0.1,fork,reuseaddr system:'sleep 2;kill -9 $SOCAT_PID 2>/dev/null' +else + # shellcheck disable=SC2016 + exec socat TCP-LISTEN:9099,bind=127.0.0.1,fork,reuseaddr system:'sleep 2;kill -9 $SOCAT_PID 2>/dev/null' +fi