shortcut-fe: rework netfilter conntrack notification (#5760)

The original patch over rode the nf_conntrack_un/register_notifier API, which will
break other modules relying on the API. Reworked the notification APIs to play nice
with others. Also avoid to touch the code of fullcone nat.

Co-authored-by: quarkysg <hhtan72@yahoo.com>
This commit is contained in:
AmadeusGhost 2020-11-10 14:53:05 +08:00 committed by GitHub
parent 047d6ea928
commit 5323b53533
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 418 additions and 384 deletions

View File

@ -49,7 +49,7 @@ Example user space program that communicates with fast
classifier kernel module
endef
SFE_MAKE_OPTS:=SFE_SUPPORT_IPV6=n
SFE_MAKE_OPTS:=SFE_SUPPORT_IPV6=y
define Build/Compile/kmod
+$(MAKE) $(PKG_JOBS) -C "$(LINUX_DIR)" $(strip $(SFE_MAKE_OPTS)) \

View File

@ -346,13 +346,11 @@ static bool fast_classifier_find_dev_and_mac_addr(struct sk_buff *skb, sfe_ip_ad
dst = (struct dst_entry *)rt;
} else {
#ifdef SFE_SUPPORT_IPV6
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0))
rt6 = rt6_lookup(&init_net, (struct in6_addr *)addr->ip6, 0, 0, NULL, 0);
#else
rt6 = rt6_lookup(&init_net, (struct in6_addr *)addr->ip6, 0, 0, 0);
#endif /*KERNEL_VERSION(4, 17, 0)*/
#endif
if (!rt6) {
goto ret_fail;
}
@ -1809,10 +1807,12 @@ static int __init fast_classifier_init(void)
goto exit3;
}
#ifdef CONFIG_NF_CONNTRACK_EVENTS
/*
* Register a notifier hook to get fast notifications of expired connections.
*/
#ifdef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
result = nf_conntrack_register_chain_notifier(&init_net, &fast_classifier_conntrack_notifier);
#else
result = nf_conntrack_register_notifier(&init_net, &fast_classifier_conntrack_notifier);
if (result < 0) {
DEBUG_ERROR("can't register nf notifier hook: %d\n", result);
@ -1879,7 +1879,11 @@ exit6:
exit5:
#ifdef CONFIG_NF_CONNTRACK_EVENTS
#ifdef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
nf_conntrack_unregister_chain_notifier(&init_net, &fast_classifier_conntrack_notifier);
#else
nf_conntrack_unregister_notifier(&init_net, &fast_classifier_conntrack_notifier);
#endif
exit4:
#endif
@ -1947,8 +1951,11 @@ static void __exit fast_classifier_exit(void)
}
#ifdef CONFIG_NF_CONNTRACK_EVENTS
#ifdef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
nf_conntrack_unregister_chain_notifier(&init_net, &fast_classifier_conntrack_notifier);
#else
nf_conntrack_unregister_notifier(&init_net, &fast_classifier_conntrack_notifier);
#endif
#endif
nf_unregister_net_hooks(&init_net, fast_classifier_ops_post_routing, ARRAY_SIZE(fast_classifier_ops_post_routing));

View File

@ -19,18 +19,13 @@ PKG_RELEASE:=2
include $(INCLUDE_DIR)/package.mk
SFE_SUPPORT_IPV6=n
define KernelPackage/shortcut-fe
SECTION:=kernel
CATEGORY:=Kernel modules
SUBMENU:=Network Support
DEPENDS:=
TITLE:=Kernel driver for SFE
ifeq (${SFE_SUPPORT_IPV6},y)
FILES:=$(PKG_BUILD_DIR)/shortcut-fe.ko $(PKG_BUILD_DIR)/shortcut-fe-ipv6.ko
else
FILES:=$(PKG_BUILD_DIR)/shortcut-fe.ko
endif
KCONFIG:=CONFIG_NF_CONNTRACK_EVENTS=y \
CONFIG_NF_CONNTRACK_TIMEOUT=y \
CONFIG_SHORTCUT_FE=y \
@ -43,8 +38,6 @@ Shortcut is an in-Linux-kernel IP packet forwarding engine.
endef
define KernelPackage/shortcut-fe/install
$(INSTALL_DIR) $(1)/etc/init.d
$(INSTALL_BIN) ./files/etc/init.d/shortcut-fe $(1)/etc/init.d
$(INSTALL_DIR) $(1)/usr/bin
$(INSTALL_BIN) ./files/usr/bin/sfe_dump $(1)/usr/bin
endef
@ -63,15 +56,7 @@ define KernelPackage/shortcut-fe-cm/Description
Simple connection manager for the Shortcut forwarding engine.
endef
ifeq (${SFE_SUPPORT_IPV6},y)
EXTRA_CFLAGS+=-DSFE_SUPPORT_IPV6
ISV6=SFE_SUPPORT_IPV6=1
else
ISV6=SFE_SUPPORT_IPV6=""
endif
EXTRA_CFLAGS+=-DSFE_SUPPORT_IPV6
define Build/Compile
+$(MAKE) $(PKG_JOBS) -C "$(LINUX_DIR)" \
@ -79,7 +64,7 @@ define Build/Compile
$(PKG_MAKE_FLAGS) \
M="$(PKG_BUILD_DIR)" \
EXTRA_CFLAGS="$(EXTRA_CFLAGS)" \
${ISV6} \
SFE_SUPPORT_IPV6=1 \
modules
endef

View File

@ -1,49 +0,0 @@
#!/bin/sh /etc/rc.common
#
# Copyright (c) 2014-2015 The Linux Foundation. All rights reserved.
# Permission to use, copy, modify, and/or distribute this software for
# any purpose with or without fee is hereby granted, provided that the
# above copyright notice and this permission notice appear in all copies.
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
#SFE connection manager has a lower priority, it should be started after other connection manager
#to detect the existence of connection manager with higher priority
START=72
have_cm() {
[ -d "/sys/kernel/debug/ecm" ] && echo 1 && return
echo 0
}
#load shortcut-fe connection manager
load_sfe_cm() {
local kernel_version=$(uname -r)
#shortcut-fe-drv.ko is not needed because other connection manager is not enabled
[ -d "/sys/module/shortcut_fe_drv" ] && rmmod shortcut_fe_drv
[ -e "/lib/modules/$kernel_version/fast-classifier.ko" ] && {
[ -d /sys/module/fast_classifier ] || insmod /lib/modules/$kernel_version/fast-classifier.ko && return
}
[ -e "/lib/modules/$kernel_version/shortcut-fe-cm.ko" ] && {
[ -d /sys/module/shortcut_fe_cm ] || insmod /lib/modules/$kernel_version/shortcut-fe-cm.ko && return
}
}
start() {
[ "$(have_cm)" = "1" ] || load_sfe_cm
}
stop() {
[ -d /sys/module/shortcut_fe_cm ] && rmmod shortcut_fe_cm
[ -d /sys/module/fast_classifier ] && rmmod fast_classifier
}

View File

@ -1049,7 +1049,7 @@ static int __init sfe_cm_init(void)
*/
#ifdef CONFIG_NF_CONNTRACK_EVENTS
#ifdef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
(void)nf_conntrack_register_notifier(&init_net, &sfe_cm_conntrack_notifier);
(void)nf_conntrack_register_chain_notifier(&init_net, &sfe_cm_conntrack_notifier);
#else
result = nf_conntrack_register_notifier(&init_net, &sfe_cm_conntrack_notifier);
if (result < 0) {
@ -1123,8 +1123,11 @@ static void __exit sfe_cm_exit(void)
sfe_ipv6_destroy_all_rules_for_dev(NULL);
#ifdef CONFIG_NF_CONNTRACK_EVENTS
#ifdef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
nf_conntrack_unregister_chain_notifier(&init_net, &sfe_cm_conntrack_notifier);
#else
nf_conntrack_unregister_notifier(&init_net, &sfe_cm_conntrack_notifier);
#endif
#endif
nf_unregister_net_hooks(&init_net, sfe_cm_ops_post_routing, ARRAY_SIZE(sfe_cm_ops_post_routing));

View File

@ -1,5 +1,5 @@
--- a/include/linux/skbuff.h 2019-01-16 20:16:08.325745306 +0800
+++ b/include/linux/skbuff.h 2019-01-16 20:31:47.288028493 +0800
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -783,6 +783,9 @@ struct sk_buff {
__u8 tc_from_ingress:1;
#endif
@ -10,8 +10,8 @@
#ifdef CONFIG_NET_SCHED
__u16 tc_index; /* traffic control index */
--- a/include/linux/if_bridge.h 2019-01-16 20:51:47.871445535 +0800
+++ b/include/linux/if_bridge.h 2019-01-16 20:52:26.220269649 +0800
--- a/include/linux/if_bridge.h
+++ b/include/linux/if_bridge.h
@@ -54,6 +54,8 @@ struct br_ip_list {
#define BR_DEFAULT_AGEING_TIME (300 * HZ)
@ -33,8 +33,19 @@
#ifdef CONFIG_LOCKDEP
struct lockdep_map lockdep_map;
--- a/net/Kconfig 2019-01-16 20:36:30.266465286 +0800
+++ b/net/Kconfig 2019-01-16 20:36:41.980609067 +0800
--- a/include/net/netfilter/nf_conntrack_ecache.h
+++ b/include/net/netfilter/nf_conntrack_ecache.h
@@ -74,6 +74,8 @@ struct nf_ct_event {
#ifdef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
extern int nf_conntrack_register_notifier(struct net *net, struct notifier_block *nb);
extern int nf_conntrack_unregister_notifier(struct net *net, struct notifier_block *nb);
+extern int nf_conntrack_register_chain_notifier(struct net *net, struct notifier_block *nb);
+extern int nf_conntrack_unregister_chain_notifier(struct net *net, struct notifier_block *nb);
#else
struct nf_ct_event_notifier {
int (*fcn)(unsigned int events, struct nf_ct_event *item);
--- a/net/Kconfig
+++ b/net/Kconfig
@@ -463,3 +463,6 @@ config HAVE_CBPF_JIT
# Extended BPF JIT (eBPF)
config HAVE_EBPF_JIT
@ -42,9 +53,9 @@
+
+config SHORTCUT_FE
+ bool "Enables kernel network stack path for Shortcut Forwarding Engine
--- a/net/core/dev.c 2019-01-16 20:38:37.274933833 +0800
+++ b/net/core/dev.c 2019-01-16 20:44:07.773594898 +0800
@@ -3001,8 +3001,17 @@ static int xmit_one(struct sk_buff *skb,
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -2998,8 +2998,17 @@ static int xmit_one(struct sk_buff *skb,
unsigned int len;
int rc;
@ -62,7 +73,7 @@
#ifdef CONFIG_ETHERNET_PACKET_MANGLE
if (!dev->eth_mangle_tx ||
@@ -4315,6 +4324,11 @@ void netdev_rx_handler_unregister(struct
@@ -4312,6 +4321,11 @@ void netdev_rx_handler_unregister(struct
}
EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
@ -74,7 +85,7 @@
/*
* Limit the use of PFMEMALLOC reserves to those protocols that implement
* the special handling of PFMEMALLOC skbs.
@@ -4362,6 +4376,9 @@ static int __netif_receive_skb_core(stru
@@ -4359,6 +4373,9 @@ static int __netif_receive_skb_core(stru
bool deliver_exact = false;
int ret = NET_RX_DROP;
__be16 type;
@ -84,7 +95,7 @@
net_timestamp_check(!netdev_tstamp_prequeue, skb);
@@ -4388,6 +4405,16 @@ another_round:
@@ -4385,6 +4402,16 @@ another_round:
goto out;
}
@ -101,8 +112,8 @@
if (skb_skip_tc_classify(skb))
goto skip_classify;
--- a/net/netfilter/nf_conntrack_proto_tcp.c 2019-01-16 20:47:40.886993297 +0800
+++ b/net/netfilter/nf_conntrack_proto_tcp.c 2019-01-16 20:48:57.700570104 +0800
--- a/net/netfilter/nf_conntrack_proto_tcp.c
+++ b/net/netfilter/nf_conntrack_proto_tcp.c
@@ -35,11 +35,17 @@
/* Do not check the TCP window for incoming packets */
@ -121,8 +132,8 @@
/* If it is set to zero, we disable picking up already established
connections. */
--- a/net/bridge/br_if.c 2019-01-16 20:54:51.919367044 +0800
+++ b/net/bridge/br_if.c 2019-01-16 20:55:53.812401263 +0800
--- a/net/bridge/br_if.c
+++ b/net/bridge/br_if.c
@@ -653,3 +653,26 @@ void br_port_flags_change(struct net_bri
if (mask & BR_AUTO_MASK)
nbp_update_port_count(br);
@ -150,77 +161,48 @@
+ u64_stats_update_end(&stats->syncp);
+}
+EXPORT_SYMBOL_GPL(br_dev_update_stats);
--- a/net/netfilter/Kconfig 2019-01-16 21:07:34.543460920 +0800
+++ b/net/netfilter/Kconfig 2019-01-16 21:08:14.739465937 +0800
@@ -146,6 +146,14 @@ config NF_CONNTRACK_TIMEOUT
If unsure, say `N'.
+config NF_CONNTRACK_CHAIN_EVENTS
+ bool "Register multiple callbacks to ct events"
+ depends on NF_CONNTRACK_EVENTS
+ help
+ Support multiple registrations.
+
+ If unsure, say `N'.
+
config NF_CONNTRACK_TIMESTAMP
bool 'Connection tracking timestamping'
depends on NETFILTER_ADVANCED
--- a/net/netfilter/nf_conntrack_ecache.c 2019-01-16 21:12:22.183462975 +0800
+++ b/net/netfilter/nf_conntrack_ecache.c 2019-01-16 21:26:10.379462031 +0800
@@ -122,13 +125,17 @@ int nf_conntrack_eventmask_report(unsign
{
int ret = 0;
struct net *net = nf_ct_net(ct);
+#ifndef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
struct nf_ct_event_notifier *notify;
+#endif
struct nf_conntrack_ecache *e;
--- a/net/netfilter/nf_conntrack_ecache.c
+++ b/net/netfilter/nf_conntrack_ecache.c
@@ -162,7 +162,11 @@ int nf_conntrack_eventmask_report(unsigned int eventmask, struct nf_conn *ct,
rcu_read_lock();
+#ifndef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
notify = rcu_dereference(net->ct.nf_conntrack_event_cb);
+#ifdef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
+ if (!notify && !rcu_dereference_raw(net->ct.nf_conntrack_chain.head))
+#else
if (!notify)
goto out_unlock;
+#endif
goto out_unlock;
e = nf_ct_ecache_find(ct);
if (!e)
@@ -146,7 +153,12 @@ int nf_conntrack_eventmask_report(unsign
@@ -181,7 +185,14 @@ int nf_conntrack_eventmask_report(unsigned int eventmask, struct nf_conn *ct,
if (!((eventmask | missed) & e->ctmask))
goto out_unlock;
+#ifdef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
+ ret = atomic_notifier_call_chain(&net->ct.nf_conntrack_chain,
+ eventmask | missed, &item);
+ if (notify)
+ ret = notify->fcn(eventmask | missed, &item);
+#else
ret = notify->fcn(eventmask | missed, &item);
+#endif
if (unlikely(ret < 0 || missed)) {
spin_lock_bh(&ct->lock);
if (ret < 0) {
@@ -179,15 +191,19 @@ void nf_ct_deliver_cached_events(struct
{
struct net *net = nf_ct_net(ct);
unsigned long events, missed;
+#ifndef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
struct nf_ct_event_notifier *notify;
+#endif
struct nf_conntrack_ecache *e;
struct nf_ct_event item;
int ret;
@@ -263,7 +274,11 @@ void nf_ct_deliver_cached_events(struct nf_conn *ct)
rcu_read_lock();
+#ifndef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
notify = rcu_dereference(net->ct.nf_conntrack_event_cb);
+#ifdef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
+ if ((notify == NULL) && !rcu_dereference_raw(net->ct.nf_conntrack_chain.head))
+#else
if (notify == NULL)
goto out_unlock;
+#endif
goto out_unlock;
e = nf_ct_ecache_find(ct);
if (e == NULL)
@@ -210,7 +226,13 @@ void nf_ct_deliver_cached_events(struct
@@ -287,7 +302,15 @@ void nf_ct_deliver_cached_events(struct nf_conn *ct)
item.portid = 0;
item.report = 0;
@ -228,9 +210,35 @@
+ ret = atomic_notifier_call_chain(&net->ct.nf_conntrack_chain,
+ events | missed,
+ &item);
+ if (notify != NULL)
+ ret = notify->fcn(events | missed, &item);
+#else
ret = notify->fcn(events | missed, &item);
+#endif
if (likely(ret == 0 && !missed))
goto out_unlock;
goto out_unlock;
@@ -340,6 +363,11 @@ int nf_conntrack_register_notifier(struct net *net, struct notifier_block *nb)
{
return atomic_notifier_chain_register(&net->ct.nf_conntrack_chain, nb);
}
+int nf_conntrack_register_chain_notifier(struct net *net, struct notifier_block *nb)
+{
+ return atomic_notifier_chain_register(&net->ct.nf_conntrack_chain, nb);
+}
+EXPORT_SYMBOL_GPL(nf_conntrack_register_chain_notifier);
#else
int nf_conntrack_register_notifier(struct net *net,
struct nf_ct_event_notifier *new)
@@ -369,6 +397,11 @@ int nf_conntrack_unregister_notifier(struct net *net, struct notifier_block *nb)
{
return atomic_notifier_chain_unregister(&net->ct.nf_conntrack_chain, nb);
}
+int nf_conntrack_unregister_chain_notifier(struct net *net, struct notifier_block *nb)
+{
+ return atomic_notifier_chain_unregister(&net->ct.nf_conntrack_chain, nb);
+}
+EXPORT_SYMBOL_GPL(nf_conntrack_unregister_chain_notifier);
#else
void nf_conntrack_unregister_notifier(struct net *net,
struct nf_ct_event_notifier *new)

View File

@ -1,6 +1,6 @@
--- a/include/linux/skbuff.h 2019-01-16 20:16:08.325745306 +0800
+++ b/include/linux/skbuff.h 2019-01-16 20:31:47.288028493 +0800
@@ -783,6 +783,9 @@ struct sk_buff {
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -800,6 +800,9 @@ struct sk_buff {
__u8 tc_from_ingress:1;
#endif
__u8 gro_skip:1;
@ -10,9 +10,9 @@
#ifdef CONFIG_NET_SCHED
__u16 tc_index; /* traffic control index */
--- a/include/linux/if_bridge.h 2019-01-16 20:51:47.871445535 +0800
+++ b/include/linux/if_bridge.h 2019-01-16 20:52:26.220269649 +0800
@@ -54,6 +54,8 @@ struct br_ip_list {
--- a/include/linux/if_bridge.h
+++ b/include/linux/if_bridge.h
@@ -55,6 +55,8 @@ struct br_ip_list {
#define BR_DEFAULT_AGEING_TIME (300 * HZ)
extern void brioctl_set(int (*ioctl_hook)(struct net *, unsigned int, void __user *));
@ -23,7 +23,7 @@
extern br_should_route_hook_t __rcu *br_should_route_hook;
--- a/include/linux/timer.h
+++ b/include/linux/timer.h
@@ -20,6 +20,9 @@ struct timer_list {
@@ -17,6 +17,9 @@ struct timer_list {
void (*function)(unsigned long);
unsigned long data;
u32 flags;
@ -33,18 +33,29 @@
#ifdef CONFIG_LOCKDEP
struct lockdep_map lockdep_map;
--- a/net/Kconfig 2019-01-16 20:36:30.266465286 +0800
+++ b/net/Kconfig 2019-01-16 20:36:41.980609067 +0800
@@ -463,3 +463,6 @@ config HAVE_CBPF_JIT
--- a/include/net/netfilter/nf_conntrack_ecache.h
+++ b/include/net/netfilter/nf_conntrack_ecache.h
@@ -74,6 +74,8 @@ struct nf_ct_event {
#ifdef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
extern int nf_conntrack_register_notifier(struct net *net, struct notifier_block *nb);
extern int nf_conntrack_unregister_notifier(struct net *net, struct notifier_block *nb);
+extern int nf_conntrack_register_chain_notifier(struct net *net, struct notifier_block *nb);
+extern int nf_conntrack_unregister_chain_notifier(struct net *net, struct notifier_block *nb);
#else
struct nf_ct_event_notifier {
int (*fcn)(unsigned int events, struct nf_ct_event *item);
--- a/net/Kconfig
+++ b/net/Kconfig
@@ -467,3 +467,6 @@ config HAVE_CBPF_JIT
# Extended BPF JIT (eBPF)
config HAVE_EBPF_JIT
bool
+
+config SHORTCUT_FE
+ bool "Enables kernel network stack path for Shortcut Forwarding Engine
--- a/net/core/dev.c 2019-01-16 20:38:37.274933833 +0800
+++ b/net/core/dev.c 2019-01-16 20:44:07.773594898 +0800
@@ -3001,8 +3001,17 @@ static int xmit_one(struct sk_buff *skb,
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -3249,8 +3249,17 @@ static int xmit_one(struct sk_buff *skb,
unsigned int len;
int rc;
@ -62,7 +73,7 @@
#ifdef CONFIG_ETHERNET_PACKET_MANGLE
if (!dev->eth_mangle_tx ||
@@ -4315,6 +4324,11 @@ void netdev_rx_handler_unregister(struct
@@ -4744,6 +4753,11 @@ void netdev_rx_handler_unregister(struct
}
EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
@ -74,7 +85,7 @@
/*
* Limit the use of PFMEMALLOC reserves to those protocols that implement
* the special handling of PFMEMALLOC skbs.
@@ -4362,6 +4376,9 @@ static int __netif_receive_skb_core(stru
@@ -4793,6 +4807,9 @@ static int __netif_receive_skb_core(stru
bool deliver_exact = false;
int ret = NET_RX_DROP;
__be16 type;
@ -84,7 +95,7 @@
net_timestamp_check(!netdev_tstamp_prequeue, skb);
@@ -4388,6 +4405,16 @@ another_round:
@@ -4833,6 +4850,16 @@ another_round:
goto out;
}
@ -101,9 +112,9 @@
if (skb_skip_tc_classify(skb))
goto skip_classify;
--- a/net/netfilter/nf_conntrack_proto_tcp.c 2019-01-16 20:47:40.886993297 +0800
+++ b/net/netfilter/nf_conntrack_proto_tcp.c 2019-01-16 20:48:57.700570104 +0800
@@ -35,11 +35,17 @@
--- a/net/netfilter/nf_conntrack_proto_tcp.c
+++ b/net/netfilter/nf_conntrack_proto_tcp.c
@@ -36,11 +36,17 @@
/* Do not check the TCP window for incoming packets */
static int nf_ct_tcp_no_window_check __read_mostly = 1;
@ -121,9 +132,9 @@
/* If it is set to zero, we disable picking up already established
connections. */
--- a/net/bridge/br_if.c 2019-01-16 20:54:51.919367044 +0800
+++ b/net/bridge/br_if.c 2019-01-16 20:55:53.812401263 +0800
@@ -653,3 +653,26 @@ void br_port_flags_change(struct net_bri
--- a/net/bridge/br_if.c
+++ b/net/bridge/br_if.c
@@ -741,3 +741,26 @@ void br_port_flags_change(struct net_bri
if (mask & BR_AUTO_MASK)
nbp_update_port_count(br);
}
@ -150,77 +161,48 @@
+ u64_stats_update_end(&stats->syncp);
+}
+EXPORT_SYMBOL_GPL(br_dev_update_stats);
--- a/net/netfilter/Kconfig 2019-01-16 21:07:34.543460920 +0800
+++ b/net/netfilter/Kconfig 2019-01-16 21:08:14.739465937 +0800
@@ -146,6 +146,14 @@ config NF_CONNTRACK_TIMEOUT
If unsure, say `N'.
+config NF_CONNTRACK_CHAIN_EVENTS
+ bool "Register multiple callbacks to ct events"
+ depends on NF_CONNTRACK_EVENTS
+ help
+ Support multiple registrations.
+
+ If unsure, say `N'.
+
config NF_CONNTRACK_TIMESTAMP
bool 'Connection tracking timestamping'
depends on NETFILTER_ADVANCED
--- a/net/netfilter/nf_conntrack_ecache.c 2019-01-16 21:12:22.183462975 +0800
+++ b/net/netfilter/nf_conntrack_ecache.c 2019-01-16 21:26:10.379462031 +0800
@@ -122,13 +125,17 @@ int nf_conntrack_eventmask_report(unsign
{
int ret = 0;
struct net *net = nf_ct_net(ct);
+#ifndef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
struct nf_ct_event_notifier *notify;
+#endif
struct nf_conntrack_ecache *e;
--- a/net/netfilter/nf_conntrack_ecache.c
+++ b/net/netfilter/nf_conntrack_ecache.c
@@ -164,7 +164,11 @@ int nf_conntrack_eventmask_report(unsigned int eventmask, struct nf_conn *ct,
rcu_read_lock();
+#ifndef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
notify = rcu_dereference(net->ct.nf_conntrack_event_cb);
+#ifdef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
+ if (!notify && !rcu_dereference_raw(net->ct.nf_conntrack_chain.head))
+#else
if (!notify)
goto out_unlock;
+#endif
goto out_unlock;
e = nf_ct_ecache_find(ct);
if (!e)
@@ -146,7 +153,12 @@ int nf_conntrack_eventmask_report(unsign
@@ -183,7 +187,14 @@ int nf_conntrack_eventmask_report(unsigned int eventmask, struct nf_conn *ct,
if (!((eventmask | missed) & e->ctmask))
goto out_unlock;
+#ifdef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
+ ret = atomic_notifier_call_chain(&net->ct.nf_conntrack_chain,
+ eventmask | missed, &item);
+ if (notify)
+ ret = notify->fcn(eventmask | missed, &item);
+#else
ret = notify->fcn(eventmask | missed, &item);
+#endif
if (unlikely(ret < 0 || missed)) {
spin_lock_bh(&ct->lock);
if (ret < 0) {
@@ -179,15 +191,19 @@ void nf_ct_deliver_cached_events(struct
{
struct net *net = nf_ct_net(ct);
unsigned long events, missed;
+#ifndef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
struct nf_ct_event_notifier *notify;
+#endif
struct nf_conntrack_ecache *e;
struct nf_ct_event item;
int ret;
@@ -265,7 +276,11 @@ void nf_ct_deliver_cached_events(struct nf_conn *ct)
rcu_read_lock();
+#ifndef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
notify = rcu_dereference(net->ct.nf_conntrack_event_cb);
+#ifdef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
+ if ((notify == NULL) && !rcu_dereference_raw(net->ct.nf_conntrack_chain.head))
+#else
if (notify == NULL)
goto out_unlock;
+#endif
goto out_unlock;
e = nf_ct_ecache_find(ct);
if (e == NULL)
@@ -210,7 +226,13 @@ void nf_ct_deliver_cached_events(struct
@@ -289,7 +304,15 @@ void nf_ct_deliver_cached_events(struct nf_conn *ct)
item.portid = 0;
item.report = 0;
@ -228,9 +210,35 @@
+ ret = atomic_notifier_call_chain(&net->ct.nf_conntrack_chain,
+ events | missed,
+ &item);
+ if (notify != NULL)
+ ret = notify->fcn(events | missed, &item);
+#else
ret = notify->fcn(events | missed, &item);
+#endif
if (likely(ret == 0 && !missed))
goto out_unlock;
goto out_unlock;
@@ -342,6 +365,11 @@ int nf_conntrack_register_notifier(struct net *net, struct notifier_block *nb)
{
return atomic_notifier_chain_register(&net->ct.nf_conntrack_chain, nb);
}
+int nf_conntrack_register_chain_notifier(struct net *net, struct notifier_block *nb)
+{
+ return atomic_notifier_chain_register(&net->ct.nf_conntrack_chain, nb);
+}
+EXPORT_SYMBOL_GPL(nf_conntrack_register_chain_notifier);
#else
int nf_conntrack_register_notifier(struct net *net,
struct nf_ct_event_notifier *new)
@@ -371,6 +399,11 @@ int nf_conntrack_unregister_notifier(struct net *net, struct notifier_block *nb)
{
return atomic_notifier_chain_unregister(&net->ct.nf_conntrack_chain, nb);
}
+int nf_conntrack_unregister_chain_notifier(struct net *net, struct notifier_block *nb)
+{
+ return atomic_notifier_chain_unregister(&net->ct.nf_conntrack_chain, nb);
+}
+EXPORT_SYMBOL_GPL(nf_conntrack_unregister_chain_notifier);
#else
void nf_conntrack_unregister_notifier(struct net *net,
struct nf_ct_event_notifier *new)

View File

@ -0,0 +1,253 @@
--- a/include/linux/if_bridge.h
+++ b/include/linux/if_bridge.h
@@ -52,6 +52,9 @@ struct br_ip_list {
extern void brioctl_set(int (*ioctl_hook)(struct net *, unsigned int, void __user *));
+extern void br_dev_update_stats(struct net_device *dev,
+ struct rtnl_link_stats64 *nlstats);
+
#if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_BRIDGE_IGMP_SNOOPING)
int br_multicast_list_adjacent(struct net_device *dev,
struct list_head *br_ip_list);
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -826,6 +826,10 @@ struct sk_buff {
#endif
__u8 gro_skip:1;
+#ifdef CONFIG_SHORTCUT_FE
+ __u8 fast_forwarded:1;
+#endif
+
#ifdef CONFIG_NET_SCHED
__u16 tc_index; /* traffic control index */
#endif
--- a/include/linux/timer.h
+++ b/include/linux/timer.h
@@ -18,6 +18,10 @@ struct timer_list {
void (*function)(struct timer_list *);
u32 flags;
+#ifdef CONFIG_SHORTCUT_FE
+ unsigned long cust_data;
+#endif
+
#ifdef CONFIG_LOCKDEP
struct lockdep_map lockdep_map;
#endif
--- a/include/net/netfilter/nf_conntrack_ecache.h
+++ b/include/net/netfilter/nf_conntrack_ecache.h
@@ -75,6 +75,8 @@ struct nf_ct_event {
#ifdef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
extern int nf_conntrack_register_notifier(struct net *net, struct notifier_block *nb);
extern int nf_conntrack_unregister_notifier(struct net *net, struct notifier_block *nb);
+extern int nf_conntrack_register_chain_notifier(struct net *net, struct notifier_block *nb);
+extern int nf_conntrack_unregister_chain_notifier(struct net *net, struct notifier_block *nb);
#else
struct nf_ct_event_notifier {
int (*fcn)(unsigned int events, struct nf_ct_event *item);
--- a/net/bridge/br_if.c
+++ b/net/bridge/br_if.c
@@ -746,6 +746,28 @@ void br_port_flags_change(struct net_bri
br_recalculate_neigh_suppress_enabled(br);
}
+void br_dev_update_stats(struct net_device *dev,
+ struct rtnl_link_stats64 *nlstats)
+{
+ struct net_bridge *br;
+ struct pcpu_sw_netstats *stats;
+
+ /* Is this a bridge? */
+ if (!(dev->priv_flags & IFF_EBRIDGE))
+ return;
+
+ br = netdev_priv(dev);
+ stats = this_cpu_ptr(br->stats);
+
+ u64_stats_update_begin(&stats->syncp);
+ stats->rx_packets += nlstats->rx_packets;
+ stats->rx_bytes += nlstats->rx_bytes;
+ stats->tx_packets += nlstats->tx_packets;
+ stats->tx_bytes += nlstats->tx_bytes;
+ u64_stats_update_end(&stats->syncp);
+}
+EXPORT_SYMBOL_GPL(br_dev_update_stats);
+
bool br_port_flag_is_set(const struct net_device *dev, unsigned long flag)
{
struct net_bridge_port *p;
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -3189,8 +3189,17 @@ static int xmit_one(struct sk_buff *skb, struct net_device *dev,
unsigned int len;
int rc;
+#ifdef CONFIG_SHORTCUT_FE
+ /* If this skb has been fast forwarded then we don't want it to
+ * go to any taps (by definition we're trying to bypass them).
+ */
+ if (!skb->fast_forwarded) {
+#endif
if (dev_nit_active(dev))
dev_queue_xmit_nit(skb, dev);
+#ifdef CONFIG_SHORTCUT_FE
+ }
+#endif
#ifdef CONFIG_ETHERNET_PACKET_MANGLE
if (!dev->eth_mangle_tx ||
@@ -4683,6 +4691,11 @@ void netdev_rx_handler_unregister(struct
}
EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
+#ifdef CONFIG_SHORTCUT_FE
+int (*athrs_fast_nat_recv)(struct sk_buff *skb) __rcu __read_mostly;
+EXPORT_SYMBOL_GPL(athrs_fast_nat_recv);
+#endif
+
/*
* Limit the use of PFMEMALLOC reserves to those protocols that implement
* the special handling of PFMEMALLOC skbs.
@@ -4733,6 +4746,10 @@ static int __netif_receive_skb_core(stru
int ret = NET_RX_DROP;
__be16 type;
+#ifdef CONFIG_SHORTCUT_FE
+ int (*fast_recv)(struct sk_buff *skb);
+#endif
+
net_timestamp_check(!netdev_tstamp_prequeue, skb);
trace_netif_receive_skb(skb);
@@ -4772,6 +4789,16 @@ another_round:
goto out;
}
+#ifdef CONFIG_SHORTCUT_FE
+ fast_recv = rcu_dereference(athrs_fast_nat_recv);
+ if (fast_recv) {
+ if (fast_recv(skb)) {
+ ret = NET_RX_SUCCESS;
+ goto out;
+ }
+ }
+#endif
+
if (skb_skip_tc_classify(skb))
goto skip_classify;
--- a/net/Kconfig
+++ b/net/Kconfig
@@ -473,3 +473,6 @@ config HAVE_CBPF_JIT
# Extended BPF JIT (eBPF)
config HAVE_EBPF_JIT
bool
+
+config SHORTCUT_FE
+ bool "Enables kernel network stack path for Shortcut Forwarding Engine
--- a/net/netfilter/nf_conntrack_proto_tcp.c
+++ b/net/netfilter/nf_conntrack_proto_tcp.c
@@ -34,11 +34,19 @@
/* Do not check the TCP window for incoming packets */
static int nf_ct_tcp_no_window_check __read_mostly = 1;
+#ifdef CONFIG_SHORTCUT_FE
+EXPORT_SYMBOL_GPL(nf_ct_tcp_no_window_check);
+#endif
+
/* "Be conservative in what you do,
be liberal in what you accept from others."
If it's non-zero, we mark only out of window RST segments as INVALID. */
static int nf_ct_tcp_be_liberal __read_mostly = 0;
+#ifdef CONFIG_SHORTCUT_FE
+EXPORT_SYMBOL_GPL(nf_ct_tcp_be_liberal);
+#endif
+
/* If it is set to zero, we disable picking up already established
connections. */
static int nf_ct_tcp_loose __read_mostly = 1;
--- a/net/netfilter/nf_conntrack_ecache.c
+++ b/net/netfilter/nf_conntrack_ecache.c
@@ -162,7 +162,11 @@ int nf_conntrack_eventmask_report(unsigned int eventmask, struct nf_conn *ct,
rcu_read_lock();
notify = rcu_dereference(net->ct.nf_conntrack_event_cb);
+#ifdef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
+ if (!notify && !rcu_dereference_raw(net->ct.nf_conntrack_chain.head))
+#else
if (!notify)
+#endif
goto out_unlock;
e = nf_ct_ecache_find(ct);
@@ -181,7 +185,14 @@ int nf_conntrack_eventmask_report(unsigned int eventmask, struct nf_conn *ct,
if (!((eventmask | missed) & e->ctmask))
goto out_unlock;
+#ifdef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
+ ret = atomic_notifier_call_chain(&net->ct.nf_conntrack_chain,
+ eventmask | missed, &item);
+ if (notify)
+ ret = notify->fcn(eventmask | missed, &item);
+#else
ret = notify->fcn(eventmask | missed, &item);
+#endif
if (unlikely(ret < 0 || missed)) {
spin_lock_bh(&ct->lock);
if (ret < 0) {
@@ -263,7 +274,11 @@ void nf_ct_deliver_cached_events(struct nf_conn *ct)
rcu_read_lock();
notify = rcu_dereference(net->ct.nf_conntrack_event_cb);
+#ifdef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
+ if ((notify == NULL) && !rcu_dereference_raw(net->ct.nf_conntrack_chain.head))
+#else
if (notify == NULL)
+#endif
goto out_unlock;
e = nf_ct_ecache_find(ct);
@@ -287,7 +302,15 @@ void nf_ct_deliver_cached_events(struct nf_conn *ct)
item.portid = 0;
item.report = 0;
+#ifdef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
+ ret = atomic_notifier_call_chain(&net->ct.nf_conntrack_chain,
+ events | missed,
+ &item);
+ if (notify != NULL)
+ ret = notify->fcn(events | missed, &item);
+#else
ret = notify->fcn(events | missed, &item);
+#endif
if (likely(ret == 0 && !missed))
goto out_unlock;
@@ -340,6 +363,11 @@ int nf_conntrack_register_notifier(struct net *net, struct notifier_block *nb)
{
return atomic_notifier_chain_register(&net->ct.nf_conntrack_chain, nb);
}
+int nf_conntrack_register_chain_notifier(struct net *net, struct notifier_block *nb)
+{
+ return atomic_notifier_chain_register(&net->ct.nf_conntrack_chain, nb);
+}
+EXPORT_SYMBOL_GPL(nf_conntrack_register_chain_notifier);
#else
int nf_conntrack_register_notifier(struct net *net,
struct nf_ct_event_notifier *new)
@@ -369,6 +397,11 @@ int nf_conntrack_unregister_notifier(struct net *net, struct notifier_block *nb)
{
return atomic_notifier_chain_unregister(&net->ct.nf_conntrack_chain, nb);
}
+int nf_conntrack_unregister_chain_notifier(struct net *net, struct notifier_block *nb)
+{
+ return atomic_notifier_chain_unregister(&net->ct.nf_conntrack_chain, nb);
+}
+EXPORT_SYMBOL_GPL(nf_conntrack_unregister_chain_notifier);
#else
void nf_conntrack_unregister_notifier(struct net *net,
struct nf_ct_event_notifier *new)

View File

@ -1,181 +0,0 @@
--- a/include/linux/if_bridge.h 2020-06-22 00:31:27.000000000 -0700
+++ b/include/linux/if_bridge.h 2020-06-27 18:17:02.739634872 -0700
@@ -51,7 +51,8 @@ struct br_ip_list {
#define BR_DEFAULT_AGEING_TIME (300 * HZ)
extern void brioctl_set(int (*ioctl_hook)(struct net *, unsigned int, void __user *));
-
+extern void br_dev_update_stats(struct net_device *dev,
+ struct rtnl_link_stats64 *nlstats);
#if IS_ENABLED(CONFIG_BRIDGE) && IS_ENABLED(CONFIG_BRIDGE_IGMP_SNOOPING)
int br_multicast_list_adjacent(struct net_device *dev,
struct list_head *br_ip_list);
--- a/include/linux/skbuff.h 2020-06-23 11:43:45.724010000 -0700
+++ b/include/linux/skbuff.h 2020-06-27 18:16:21.428010018 -0700
@@ -825,6 +825,9 @@ struct sk_buff {
__u8 decrypted:1;
#endif
__u8 gro_skip:1;
+#ifdef CONFIG_SHORTCUT_FE
+ __u8 fast_forwarded:1;
+#endif
#ifdef CONFIG_NET_SCHED
__u16 tc_index; /* traffic control index */
--- a/include/linux/timer.h 2020-06-22 00:31:27.000000000 -0700
+++ b/include/linux/timer.h 2020-06-27 18:23:29.015910189 -0700
@@ -17,7 +17,9 @@ struct timer_list {
unsigned long expires;
void (*function)(struct timer_list *);
u32 flags;
-
+#ifdef CONFIG_SHORTCUT_FE
+ unsigned long cust_data;
+#endif
#ifdef CONFIG_LOCKDEP
struct lockdep_map lockdep_map;
#endif
--- a/include/net/netns/conntrack.h 2020-06-23 11:43:45.752009000 -0700
+++ b/include/net/netns/conntrack.h 2020-06-27 18:18:26.138861119 -0700
@@ -115,8 +115,12 @@ struct netns_ct {
#ifdef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
struct atomic_notifier_head nf_conntrack_chain;
#else
+#ifdef CONFIG_NF_CONNTRACK_CHAIN_EVENTS
+ struct atomic_notifier_head nf_conntrack_chain;
+#else
struct nf_ct_event_notifier __rcu *nf_conntrack_event_cb;
#endif
+#endif
struct nf_exp_event_notifier __rcu *nf_expect_event_cb;
struct nf_ip_net nf_ct_proto;
#if defined(CONFIG_NF_CONNTRACK_LABELS)
--- a/net/bridge/br_if.c 2020-06-22 00:31:27.000000000 -0700
+++ b/net/bridge/br_if.c 2020-06-27 18:31:59.765638309 -0700
@@ -746,6 +746,28 @@ void br_port_flags_change(struct net_bri
br_recalculate_neigh_suppress_enabled(br);
}
+void br_dev_update_stats(struct net_device *dev,
+ struct rtnl_link_stats64 *nlstats)
+{
+ struct net_bridge *br;
+ struct pcpu_sw_netstats *stats;
+
+ /* Is this a bridge? */
+ if (!(dev->priv_flags & IFF_EBRIDGE))
+ return;
+
+ br = netdev_priv(dev);
+ stats = this_cpu_ptr(br->stats);
+
+ u64_stats_update_begin(&stats->syncp);
+ stats->rx_packets += nlstats->rx_packets;
+ stats->rx_bytes += nlstats->rx_bytes;
+ stats->tx_packets += nlstats->tx_packets;
+ stats->tx_bytes += nlstats->tx_bytes;
+ u64_stats_update_end(&stats->syncp);
+}
+EXPORT_SYMBOL_GPL(br_dev_update_stats);
+
bool br_port_flag_is_set(const struct net_device *dev, unsigned long flag)
{
struct net_bridge_port *p;
--- a/net/core/dev.c 2020-06-23 11:43:45.728009000 -0700
+++ b/net/core/dev.c 2020-06-27 18:27:40.161351260 -0700
@@ -3191,10 +3191,17 @@ static int xmit_one(struct sk_buff *skb,
{
unsigned int len;
int rc;
-
+#ifdef CONFIG_SHORTCUT_FE
+ /* If this skb has been fast forwarded then we don't want it to
+ * go to any taps (by definition we're trying to bypass them).
+ */
+ if (!skb->fast_forwarded) {
+#endif
if (dev_nit_active(dev))
dev_queue_xmit_nit(skb, dev);
-
+#ifdef CONFIG_SHORTCUT_FE
+ }
+#endif
#ifdef CONFIG_ETHERNET_PACKET_MANGLE
if (!dev->eth_mangle_tx ||
(skb = dev->eth_mangle_tx(dev, skb)) != NULL)
@@ -4683,6 +4690,10 @@ void netdev_rx_handler_unregister(struct
RCU_INIT_POINTER(dev->rx_handler_data, NULL);
}
EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
+#ifdef CONFIG_SHORTCUT_FE
+int (*athrs_fast_nat_recv)(struct sk_buff *skb) __rcu __read_mostly;
+EXPORT_SYMBOL_GPL(athrs_fast_nat_recv);
+#endif
/*
* Limit the use of PFMEMALLOC reserves to those protocols that implement
@@ -4733,7 +4744,9 @@ static int __netif_receive_skb_core(stru
bool deliver_exact = false;
int ret = NET_RX_DROP;
__be16 type;
-
+#ifdef CONFIG_SHORTCUT_FE
+ int (*fast_recv)(struct sk_buff *skb);
+#endif
net_timestamp_check(!netdev_tstamp_prequeue, skb);
trace_netif_receive_skb(skb);
@@ -4772,6 +4785,15 @@ another_round:
if (unlikely(!skb))
goto out;
}
+#ifdef CONFIG_SHORTCUT_FE
+ fast_recv = rcu_dereference(athrs_fast_nat_recv);
+ if (fast_recv) {
+ if (fast_recv(skb)) {
+ ret = NET_RX_SUCCESS;
+ goto out;
+ }
+ }
+#endif
if (skb_skip_tc_classify(skb))
goto skip_classify;
--- a/net/Kconfig 2020-06-23 11:43:45.732009000 -0700
+++ b/net/Kconfig 2020-06-27 18:24:06.035538006 -0700
@@ -473,3 +473,6 @@ config HAVE_CBPF_JIT
# Extended BPF JIT (eBPF)
config HAVE_EBPF_JIT
bool
+
+config SHORTCUT_FE
+ bool "Enables kernel network stack path for Shortcut Forwarding Engine
+
--- a/net/netfilter/nf_conntrack_proto_tcp.c 2020-06-23 11:43:45.404018000 -0700
+++ b/net/netfilter/nf_conntrack_proto_tcp.c 2020-06-27 18:28:24.610214943 -0700
@@ -33,12 +33,16 @@
/* Do not check the TCP window for incoming packets */
static int nf_ct_tcp_no_window_check __read_mostly = 1;
-
+#ifdef CONFIG_SHORTCUT_FE
+EXPORT_SYMBOL_GPL(nf_ct_tcp_no_window_check);
+#endif
/* "Be conservative in what you do,
be liberal in what you accept from others."
If it's non-zero, we mark only out of window RST segments as INVALID. */
static int nf_ct_tcp_be_liberal __read_mostly = 0;
-
+#ifdef CONFIG_SHORTCUT_FE
+EXPORT_SYMBOL_GPL(nf_ct_tcp_be_liberal);
+#endif
/* If it is set to zero, we disable picking up already established
connections. */
static int nf_ct_tcp_loose __read_mostly = 1;