[rdkb][common][bsp][Refactor and sync kernel from openwrt]
[Description]
76cd4287 [mt7986][trng][change trng version]
37641870 [openwrt][mt7987][bsp][Add support for the mt7987 platform]
73fe5221 [kernel][mt7987][hnat][Refactor fqos flag assignment when HQoS is enabled for the NETSYSv3.1]
b4d8a1ed [kernel][common][eth][Add nf hw offload]
9872cf38 [kernel][mt7987][eth][Switch to real clock driver for the ETH]
e49e90e7 [set trng clock is ao]
[Release-log]
Change-Id: I47177c3ac8287a95f5bff802123e5bdb068fb009
diff --git a/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-1718-v5.15-net-netfilter-add-nf-hw-offload.patch b/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-1718-v5.15-net-netfilter-add-nf-hw-offload.patch
new file mode 100644
index 0000000..f162e51
--- /dev/null
+++ b/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-1718-v5.15-net-netfilter-add-nf-hw-offload.patch
@@ -0,0 +1,4987 @@
+From 3a4256932000744357ba358140fd2895756060e6 Mon Sep 17 00:00:00 2001
+From: Bo-Cun Chen <bc-bocun.chen@mediatek.com>
+Date: Tue, 4 Jun 2024 09:14:34 +0800
+Subject: net netfilter add nf hw offload
+
+---
+ drivers/net/ppp/ppp_generic.c | 22 +
+ drivers/net/ppp/pppoe.c | 24 +
+ include/linux/netdevice.h | 60 +
+ include/linux/ppp_channel.h | 3 +
+ include/net/flow_offload.h | 4 +
+ include/net/ip6_route.h | 5 +-
+ .../net/netfilter/ipv6/nf_conntrack_ipv6.h | 3 -
+ include/net/netfilter/nf_conntrack.h | 14 +
+ include/net/netfilter/nf_conntrack_acct.h | 11 +
+ include/net/netfilter/nf_conntrack_l4proto.h | 7 +
+ include/net/netfilter/nf_flow_table.h | 268 +++-
+ include/net/netns/conntrack.h | 6 +
+ .../linux/netfilter/nf_conntrack_common.h | 9 +-
+ include/uapi/linux/netfilter/xt_FLOWOFFLOAD.h | 17 +
+ net/8021q/vlan_dev.c | 21 +
+ net/bridge/br_device.c | 49 +
+ net/bridge/br_private.h | 20 +
+ net/bridge/br_vlan.c | 55 +
+ net/core/dev.c | 46 +
+ net/dsa/slave.c | 36 +-
+ net/ipv4/netfilter/Kconfig | 4 +-
+ net/ipv6/ip6_output.c | 2 +-
+ net/ipv6/netfilter/Kconfig | 3 +-
+ net/ipv6/route.c | 22 +-
+ net/netfilter/Kconfig | 14 +-
+ net/netfilter/Makefile | 4 +-
+ net/netfilter/nf_conntrack_core.c | 20 +-
+ net/netfilter/nf_conntrack_proto_tcp.c | 10 +-
+ net/netfilter/nf_conntrack_proto_udp.c | 4 +
+ net/netfilter/nf_conntrack_standalone.c | 34 +-
+ net/netfilter/nf_flow_table_core.c | 470 ++++---
+ net/netfilter/nf_flow_table_ip.c | 447 +++---
+ net/netfilter/nf_flow_table_offload.c | 1199 +++++++++++++++++
+ net/netfilter/xt_FLOWOFFLOAD.c | 780 +++++++++++
+ 34 files changed, 3244 insertions(+), 499 deletions(-)
+ create mode 100644 include/uapi/linux/netfilter/xt_FLOWOFFLOAD.h
+ create mode 100644 net/netfilter/nf_flow_table_offload.c
+ create mode 100644 net/netfilter/xt_FLOWOFFLOAD.c
+
+diff --git a/drivers/net/ppp/ppp_generic.c b/drivers/net/ppp/ppp_generic.c
+index 76ed2d4..ec2fbd1 100644
+--- a/drivers/net/ppp/ppp_generic.c
++++ b/drivers/net/ppp/ppp_generic.c
+@@ -1395,12 +1395,34 @@ static void ppp_dev_priv_destructor(struct net_device *dev)
+ ppp_destroy_interface(ppp);
+ }
+
++static int ppp_fill_forward_path(struct net_device_path_ctx *ctx,
++ struct net_device_path *path)
++{
++ struct ppp *ppp = netdev_priv(ctx->dev);
++ struct ppp_channel *chan;
++ struct channel *pch;
++
++ if (ppp->flags & SC_MULTILINK)
++ return -EOPNOTSUPP;
++
++ if (list_empty(&ppp->channels))
++ return -ENODEV;
++
++ pch = list_first_entry(&ppp->channels, struct channel, clist);
++ chan = pch->chan;
++ if (!chan->ops->fill_forward_path)
++ return -EOPNOTSUPP;
++
++ return chan->ops->fill_forward_path(ctx, path, chan);
++}
++
+ static const struct net_device_ops ppp_netdev_ops = {
+ .ndo_init = ppp_dev_init,
+ .ndo_uninit = ppp_dev_uninit,
+ .ndo_start_xmit = ppp_start_xmit,
+ .ndo_do_ioctl = ppp_net_ioctl,
+ .ndo_get_stats64 = ppp_get_stats64,
++ .ndo_fill_forward_path = ppp_fill_forward_path,
+ };
+
+ static struct device_type ppp_type = {
+diff --git a/drivers/net/ppp/pppoe.c b/drivers/net/ppp/pppoe.c
+index 087b016..7a8c246 100644
+--- a/drivers/net/ppp/pppoe.c
++++ b/drivers/net/ppp/pppoe.c
+@@ -974,8 +974,32 @@ static int pppoe_xmit(struct ppp_channel *chan, struct sk_buff *skb)
+ return __pppoe_xmit(sk, skb);
+ }
+
++static int pppoe_fill_forward_path(struct net_device_path_ctx *ctx,
++ struct net_device_path *path,
++ const struct ppp_channel *chan)
++{
++ struct sock *sk = (struct sock *)chan->private;
++ struct pppox_sock *po = pppox_sk(sk);
++ struct net_device *dev = po->pppoe_dev;
++
++ if (sock_flag(sk, SOCK_DEAD) ||
++ !(sk->sk_state & PPPOX_CONNECTED) || !dev)
++ return -1;
++
++ path->type = DEV_PATH_PPPOE;
++ path->encap.proto = htons(ETH_P_PPP_SES);
++ path->encap.id = be16_to_cpu(po->num);
++ memcpy(path->encap.h_dest, po->pppoe_pa.remote, ETH_ALEN);
++ memcpy(ctx->daddr, po->pppoe_pa.remote, ETH_ALEN);
++ path->dev = ctx->dev;
++ ctx->dev = dev;
++
++ return 0;
++}
++
+ static const struct ppp_channel_ops pppoe_chan_ops = {
+ .start_xmit = pppoe_xmit,
++ .fill_forward_path = pppoe_fill_forward_path,
+ };
+
+ static int pppoe_recvmsg(struct socket *sock, struct msghdr *m,
+diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
+index ac36fe9..2d1aa35 100644
+--- a/include/linux/netdevice.h
++++ b/include/linux/netdevice.h
+@@ -843,6 +843,59 @@ typedef u16 (*select_queue_fallback_t)(struct net_device *dev,
+ struct sk_buff *skb,
+ struct net_device *sb_dev);
+
++enum net_device_path_type {
++ DEV_PATH_ETHERNET = 0,
++ DEV_PATH_VLAN,
++ DEV_PATH_BRIDGE,
++ DEV_PATH_PPPOE,
++ DEV_PATH_DSA,
++};
++
++struct net_device_path {
++ enum net_device_path_type type;
++ const struct net_device *dev;
++ union {
++ struct {
++ u16 id;
++ __be16 proto;
++ u8 h_dest[ETH_ALEN];
++ } encap;
++ struct {
++ enum {
++ DEV_PATH_BR_VLAN_KEEP,
++ DEV_PATH_BR_VLAN_TAG,
++ DEV_PATH_BR_VLAN_UNTAG,
++ DEV_PATH_BR_VLAN_UNTAG_HW,
++ } vlan_mode;
++ u16 vlan_id;
++ __be16 vlan_proto;
++ } bridge;
++ struct {
++ int port;
++ u16 proto;
++ } dsa;
++ };
++};
++
++#define NET_DEVICE_PATH_STACK_MAX 5
++#define NET_DEVICE_PATH_VLAN_MAX 2
++
++struct net_device_path_stack {
++ int num_paths;
++ struct net_device_path path[NET_DEVICE_PATH_STACK_MAX];
++};
++
++struct net_device_path_ctx {
++ const struct net_device *dev;
++ u8 daddr[ETH_ALEN];
++
++ int num_vlans;
++ struct {
++ u16 id;
++ __be16 proto;
++ } vlan[NET_DEVICE_PATH_VLAN_MAX];
++};
++
+ enum tc_setup_type {
+ TC_SETUP_QDISC_MQPRIO,
+ TC_SETUP_CLSU32,
+@@ -858,6 +911,7 @@ enum tc_setup_type {
+ TC_SETUP_ROOT_QDISC,
+ TC_SETUP_QDISC_GRED,
+ TC_SETUP_QDISC_TAPRIO,
++ TC_SETUP_FT,
+ };
+
+ /* These structures hold the attributes of bpf state that are being passed
+@@ -1253,6 +1307,8 @@ struct tlsdev_ops;
+ * Get devlink port instance associated with a given netdev.
+ * Called with a reference on the netdevice and devlink locks only,
+ * rtnl_lock is not held.
++ * int (*ndo_fill_forward_path)(struct net_device_path_ctx *ctx, struct net_device_path *path);
++ * Get the forwarding path to reach the real device from the HW destination address
+ */
+ struct net_device_ops {
+ int (*ndo_init)(struct net_device *dev);
+@@ -1450,6 +1506,8 @@ struct net_device_ops {
+ int (*ndo_xsk_wakeup)(struct net_device *dev,
+ u32 queue_id, u32 flags);
+ struct devlink_port * (*ndo_get_devlink_port)(struct net_device *dev);
++ int (*ndo_fill_forward_path)(struct net_device_path_ctx *ctx,
++ struct net_device_path *path);
+ };
+
+ /**
+@@ -2689,6 +2747,8 @@ void dev_remove_offload(struct packet_offload *po);
+
+ int dev_get_iflink(const struct net_device *dev);
+ int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb);
++int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr,
++ struct net_device_path_stack *stack);
+ struct net_device *__dev_get_by_flags(struct net *net, unsigned short flags,
+ unsigned short mask);
+ struct net_device *dev_get_by_name(struct net *net, const char *name);
+diff --git a/include/linux/ppp_channel.h b/include/linux/ppp_channel.h
+index 9896606..91f9a92 100644
+--- a/include/linux/ppp_channel.h
++++ b/include/linux/ppp_channel.h
+@@ -28,6 +28,9 @@ struct ppp_channel_ops {
+ int (*start_xmit)(struct ppp_channel *, struct sk_buff *);
+ /* Handle an ioctl call that has come in via /dev/ppp. */
+ int (*ioctl)(struct ppp_channel *, unsigned int, unsigned long);
++ int (*fill_forward_path)(struct net_device_path_ctx *,
++ struct net_device_path *,
++ const struct ppp_channel *);
+ };
+
+ struct ppp_channel {
+diff --git a/include/net/flow_offload.h b/include/net/flow_offload.h
+index c6f7bd2..59b8736 100644
+--- a/include/net/flow_offload.h
++++ b/include/net/flow_offload.h
+@@ -138,6 +138,7 @@ enum flow_action_id {
+ FLOW_ACTION_MPLS_PUSH,
+ FLOW_ACTION_MPLS_POP,
+ FLOW_ACTION_MPLS_MANGLE,
++ FLOW_ACTION_PPPOE_PUSH,
+ NUM_FLOW_ACTIONS,
+ };
+
+@@ -213,6 +214,9 @@ struct flow_action_entry {
+ u8 bos;
+ u8 ttl;
+ } mpls_mangle;
++ struct { /* FLOW_ACTION_PPPOE_PUSH */
++ u16 sid;
++ } pppoe;
+ };
+ };
+
+diff --git a/include/net/ip6_route.h b/include/net/ip6_route.h
+index 2c739fc..89ab8f1 100644
+--- a/include/net/ip6_route.h
++++ b/include/net/ip6_route.h
+@@ -314,12 +314,13 @@ static inline bool rt6_duplicate_nexthop(struct fib6_info *a, struct fib6_info *
+ !lwtunnel_cmp_encap(nha->fib_nh_lws, nhb->fib_nh_lws);
+ }
+
+-static inline unsigned int ip6_dst_mtu_forward(const struct dst_entry *dst)
++static inline unsigned int ip6_dst_mtu_maybe_forward(const struct dst_entry *dst,
++ bool forwarding)
+ {
+ struct inet6_dev *idev;
+ unsigned int mtu;
+
+- if (dst_metric_locked(dst, RTAX_MTU)) {
++ if (!forwarding || dst_metric_locked(dst, RTAX_MTU)) {
+ mtu = dst_metric_raw(dst, RTAX_MTU);
+ if (mtu)
+ goto out;
+diff --git a/include/net/netfilter/ipv6/nf_conntrack_ipv6.h b/include/net/netfilter/ipv6/nf_conntrack_ipv6.h
+index 7b3c873..e954831 100644
+--- a/include/net/netfilter/ipv6/nf_conntrack_ipv6.h
++++ b/include/net/netfilter/ipv6/nf_conntrack_ipv6.h
+@@ -4,7 +4,4 @@
+
+ extern const struct nf_conntrack_l4proto nf_conntrack_l4proto_icmpv6;
+
+-#include <linux/sysctl.h>
+-extern struct ctl_table nf_ct_ipv6_sysctl_table[];
+-
+ #endif /* _NF_CONNTRACK_IPV6_H*/
+diff --git a/include/net/netfilter/nf_conntrack.h b/include/net/netfilter/nf_conntrack.h
+index 90690e3..38a8d3f 100644
+--- a/include/net/netfilter/nf_conntrack.h
++++ b/include/net/netfilter/nf_conntrack.h
+@@ -105,6 +105,8 @@ struct nf_conn {
+
+ /* Storage reserved for other modules, must be the last member */
+ union nf_conntrack_proto proto;
++
++ u16 inet6_mode;
+ };
+
+ static inline struct nf_conn *
+@@ -279,6 +281,18 @@ static inline bool nf_ct_should_gc(const struct nf_conn *ct)
+ !nf_ct_is_dying(ct);
+ }
+
++#define NF_CT_DAY (86400 * HZ)
++
++/* Set an arbitrary timeout large enough not to ever expire, this save
++ * us a check for the IPS_OFFLOAD_BIT from the packet path via
++ * nf_ct_is_expired().
++ */
++static inline void nf_ct_offload_timeout(struct nf_conn *ct)
++{
++ if (nf_ct_expires(ct) < NF_CT_DAY / 2)
++ WRITE_ONCE(ct->timeout, nfct_time_stamp + NF_CT_DAY);
++}
++
+ struct kernel_param;
+
+ int nf_conntrack_set_hashsize(const char *val, const struct kernel_param *kp);
+diff --git a/include/net/netfilter/nf_conntrack_acct.h b/include/net/netfilter/nf_conntrack_acct.h
+index f7a060c..7f44a77 100644
+--- a/include/net/netfilter/nf_conntrack_acct.h
++++ b/include/net/netfilter/nf_conntrack_acct.h
+@@ -65,6 +65,17 @@ static inline void nf_ct_set_acct(struct net *net, bool enable)
+ #endif
+ }
+
++void nf_ct_acct_add(struct nf_conn *ct, u32 dir, unsigned int packets,
++ unsigned int bytes);
++
++static inline void nf_ct_acct_update(struct nf_conn *ct, u32 dir,
++ unsigned int bytes)
++{
++#if IS_ENABLED(CONFIG_NF_CONNTRACK)
++ nf_ct_acct_add(ct, dir, 1, bytes);
++#endif
++}
++
+ void nf_conntrack_acct_pernet_init(struct net *net);
+
+ int nf_conntrack_acct_init(void);
+diff --git a/include/net/netfilter/nf_conntrack_l4proto.h b/include/net/netfilter/nf_conntrack_l4proto.h
+index 4cad1f0..886ac6b 100644
+--- a/include/net/netfilter/nf_conntrack_l4proto.h
++++ b/include/net/netfilter/nf_conntrack_l4proto.h
+@@ -203,6 +203,13 @@ static inline struct nf_icmp_net *nf_icmpv6_pernet(struct net *net)
+ }
+ #endif
+
++/* Caller must check nf_ct_protonum(ct) is IPPROTO_TCP before calling. */
++static bool nf_conntrack_tcp_established(const struct nf_conn *ct)
++{
++ return ct->proto.tcp.state == TCP_CONNTRACK_ESTABLISHED &&
++ test_bit(IPS_ASSURED_BIT, &ct->status);
++}
++
+ #ifdef CONFIG_NF_CT_PROTO_DCCP
+ static inline struct nf_dccp_net *nf_dccp_pernet(struct net *net)
+ {
+diff --git a/include/net/netfilter/nf_flow_table.h b/include/net/netfilter/nf_flow_table.h
+index 68d7fc9..7374cb2 100644
+--- a/include/net/netfilter/nf_flow_table.h
++++ b/include/net/netfilter/nf_flow_table.h
+@@ -8,31 +8,101 @@
+ #include <linux/rcupdate.h>
+ #include <linux/netfilter.h>
+ #include <linux/netfilter/nf_conntrack_tuple_common.h>
++#include <net/flow_offload.h>
+ #include <net/dst.h>
++#include <linux/if_pppox.h>
++#include <linux/ppp_defs.h>
+
+ struct nf_flowtable;
++struct nf_flow_rule;
++struct flow_offload;
++enum flow_offload_tuple_dir;
++
++#define CT_INET_MODE_IPV6 2
++
++struct nf_flow_key {
++ struct flow_dissector_key_meta meta;
++ struct flow_dissector_key_control control;
++ struct flow_dissector_key_control enc_control;
++ struct flow_dissector_key_basic basic;
++ struct flow_dissector_key_vlan vlan;
++ struct flow_dissector_key_vlan cvlan;
++ union {
++ struct flow_dissector_key_ipv4_addrs ipv4;
++ struct flow_dissector_key_ipv6_addrs ipv6;
++ };
++ struct flow_dissector_key_keyid enc_key_id;
++ union {
++ struct flow_dissector_key_ipv4_addrs enc_ipv4;
++ struct flow_dissector_key_ipv6_addrs enc_ipv6;
++ };
++ struct flow_dissector_key_tcp tcp;
++ struct flow_dissector_key_ports tp;
++} __aligned(BITS_PER_LONG / 8); /* Ensure that we can do comparisons as longs. */
++
++struct nf_flow_match {
++ struct flow_dissector dissector;
++ struct nf_flow_key key;
++ struct nf_flow_key mask;
++};
++
++struct nf_flow_rule {
++ struct nf_flow_match match;
++ struct flow_rule *rule;
++};
+
+ struct nf_flowtable_type {
+ struct list_head list;
+ int family;
+ int (*init)(struct nf_flowtable *ft);
++ int (*setup)(struct nf_flowtable *ft,
++ struct net_device *dev,
++ enum flow_block_command cmd);
++ int (*action)(struct net *net,
++ const struct flow_offload *flow,
++ enum flow_offload_tuple_dir dir,
++ struct nf_flow_rule *flow_rule);
+ void (*free)(struct nf_flowtable *ft);
+ nf_hookfn *hook;
+ struct module *owner;
+ };
+
++enum nf_flowtable_flags {
++ NF_FLOWTABLE_HW_OFFLOAD = 0x1, /* NFT_FLOWTABLE_HW_OFFLOAD */
++ NF_FLOWTABLE_COUNTER = 0x2, /* NFT_FLOWTABLE_COUNTER */
++};
++
+ struct nf_flowtable {
+ struct list_head list;
+ struct rhashtable rhashtable;
++ int priority;
+ const struct nf_flowtable_type *type;
+ struct delayed_work gc_work;
++ unsigned int flags;
++ struct flow_block flow_block;
++ struct rw_semaphore flow_block_lock; /* Guards flow_block */
++ possible_net_t net;
+ };
+
++static inline bool nf_flowtable_hw_offload(struct nf_flowtable *flowtable)
++{
++ return flowtable->flags & NF_FLOWTABLE_HW_OFFLOAD;
++}
++
+ enum flow_offload_tuple_dir {
+ FLOW_OFFLOAD_DIR_ORIGINAL = IP_CT_DIR_ORIGINAL,
+ FLOW_OFFLOAD_DIR_REPLY = IP_CT_DIR_REPLY,
+- FLOW_OFFLOAD_DIR_MAX = IP_CT_DIR_MAX
+ };
++#define FLOW_OFFLOAD_DIR_MAX IP_CT_DIR_MAX
++
++enum flow_offload_xmit_type {
++ FLOW_OFFLOAD_XMIT_UNSPEC = 0,
++ FLOW_OFFLOAD_XMIT_NEIGH,
++ FLOW_OFFLOAD_XMIT_XFRM,
++ FLOW_OFFLOAD_XMIT_DIRECT,
++};
++
++#define NF_FLOW_TABLE_ENCAP_MAX 2
+
+ struct flow_offload_tuple {
+ union {
+@@ -52,13 +122,30 @@ struct flow_offload_tuple {
+
+ u8 l3proto;
+ u8 l4proto;
+- u8 dir;
++ struct {
++ u16 id;
++ __be16 proto;
++ } encap[NF_FLOW_TABLE_ENCAP_MAX];
+
+- u16 mtu;
++ /* All members above are keys for lookups, see flow_offload_hash(). */
++ struct { } __hash;
+
+- struct {
+- struct dst_entry *dst_cache;
+- u32 dst_cookie;
++ u8 dir:2,
++ xmit_type:2,
++ encap_num:2,
++ in_vlan_ingress:2;
++ u16 mtu;
++ union {
++ struct {
++ struct dst_entry *dst_cache;
++ u32 dst_cookie;
++ };
++ struct {
++ u32 ifidx;
++ u32 hw_ifidx;
++ u8 h_source[ETH_ALEN];
++ u8 h_dest[ETH_ALEN];
++ } out;
+ };
+ };
+
+@@ -67,52 +154,140 @@ struct flow_offload_tuple_rhash {
+ struct flow_offload_tuple tuple;
+ };
+
+-#define FLOW_OFFLOAD_SNAT 0x1
+-#define FLOW_OFFLOAD_DNAT 0x2
+-#define FLOW_OFFLOAD_DYING 0x4
+-#define FLOW_OFFLOAD_TEARDOWN 0x8
++enum nf_flow_flags {
++ NF_FLOW_SNAT,
++ NF_FLOW_DNAT,
++ NF_FLOW_TEARDOWN,
++ NF_FLOW_HW,
++ NF_FLOW_HW_ACCT_DYING,
++ NF_FLOW_HW_DYING,
++ NF_FLOW_HW_DEAD,
++ NF_FLOW_HW_PENDING,
++};
++
++enum flow_offload_type {
++ NF_FLOW_OFFLOAD_UNSPEC = 0,
++ NF_FLOW_OFFLOAD_ROUTE,
++};
+
+ struct flow_offload {
+ struct flow_offload_tuple_rhash tuplehash[FLOW_OFFLOAD_DIR_MAX];
+- u32 flags;
+- union {
+- /* Your private driver data here. */
+- u32 timeout;
+- };
++ struct nf_conn *ct;
++ unsigned long flags;
++ u16 type;
++ u32 timeout;
++ struct rcu_head rcu_head;
+ };
+
+ #define NF_FLOW_TIMEOUT (30 * HZ)
++#define nf_flowtable_time_stamp (u32)jiffies
++
++unsigned long flow_offload_get_timeout(struct flow_offload *flow);
++
++static inline __s32 nf_flow_timeout_delta(unsigned int timeout)
++{
++ return (__s32)(timeout - nf_flowtable_time_stamp);
++}
+
+ struct nf_flow_route {
+ struct {
+- struct dst_entry *dst;
++ struct dst_entry *dst;
++ struct {
++ u32 ifindex;
++ struct {
++ u16 id;
++ __be16 proto;
++ } encap[NF_FLOW_TABLE_ENCAP_MAX];
++ u8 num_encaps:2,
++ ingress_vlans:2;
++ } in;
++ struct {
++ u32 ifindex;
++ u32 hw_ifindex;
++ u8 h_source[ETH_ALEN];
++ u8 h_dest[ETH_ALEN];
++ } out;
++ enum flow_offload_xmit_type xmit_type;
+ } tuple[FLOW_OFFLOAD_DIR_MAX];
+ };
+
+-struct flow_offload *flow_offload_alloc(struct nf_conn *ct,
+- struct nf_flow_route *route);
++struct flow_offload *flow_offload_alloc(struct nf_conn *ct);
+ void flow_offload_free(struct flow_offload *flow);
+
++static inline int
++nf_flow_table_offload_add_cb(struct nf_flowtable *flow_table,
++ flow_setup_cb_t *cb, void *cb_priv)
++{
++ struct flow_block *block = &flow_table->flow_block;
++ struct flow_block_cb *block_cb;
++ int err = 0;
++
++ down_write(&flow_table->flow_block_lock);
++ block_cb = flow_block_cb_lookup(block, cb, cb_priv);
++ if (block_cb) {
++ err = -EEXIST;
++ goto unlock;
++ }
++
++ block_cb = flow_block_cb_alloc(cb, cb_priv, cb_priv, NULL);
++ if (IS_ERR(block_cb)) {
++ err = PTR_ERR(block_cb);
++ goto unlock;
++ }
++
++ list_add_tail(&block_cb->list, &block->cb_list);
++
++unlock:
++ up_write(&flow_table->flow_block_lock);
++ return err;
++}
++
++static inline void
++nf_flow_table_offload_del_cb(struct nf_flowtable *flow_table,
++ flow_setup_cb_t *cb, void *cb_priv)
++{
++ struct flow_block *block = &flow_table->flow_block;
++ struct flow_block_cb *block_cb;
++
++ down_write(&flow_table->flow_block_lock);
++ block_cb = flow_block_cb_lookup(block, cb, cb_priv);
++ if (block_cb) {
++ list_del(&block_cb->list);
++ flow_block_cb_free(block_cb);
++ } else {
++ WARN_ON(true);
++ }
++ up_write(&flow_table->flow_block_lock);
++}
++
++int flow_offload_route_init(struct flow_offload *flow,
++ const struct nf_flow_route *route);
++
+ int flow_offload_add(struct nf_flowtable *flow_table, struct flow_offload *flow);
++void flow_offload_refresh(struct nf_flowtable *flow_table,
++ struct flow_offload *flow);
++
+ struct flow_offload_tuple_rhash *flow_offload_lookup(struct nf_flowtable *flow_table,
+ struct flow_offload_tuple *tuple);
++void nf_flow_table_gc_cleanup(struct nf_flowtable *flowtable,
++ struct net_device *dev);
+ void nf_flow_table_cleanup(struct net_device *dev);
+
+ int nf_flow_table_init(struct nf_flowtable *flow_table);
+ void nf_flow_table_free(struct nf_flowtable *flow_table);
+
+ void flow_offload_teardown(struct flow_offload *flow);
+-static inline void flow_offload_dead(struct flow_offload *flow)
+-{
+- flow->flags |= FLOW_OFFLOAD_DYING;
+-}
+
+-int nf_flow_snat_port(const struct flow_offload *flow,
+- struct sk_buff *skb, unsigned int thoff,
+- u8 protocol, enum flow_offload_tuple_dir dir);
+-int nf_flow_dnat_port(const struct flow_offload *flow,
+- struct sk_buff *skb, unsigned int thoff,
+- u8 protocol, enum flow_offload_tuple_dir dir);
++int nf_flow_table_iterate(struct nf_flowtable *flow_table,
++ void (*iter)(struct flow_offload *flow, void *data),
++ void *data);
++
++void nf_flow_snat_port(const struct flow_offload *flow,
++ struct sk_buff *skb, unsigned int thoff,
++ u8 protocol, enum flow_offload_tuple_dir dir);
++void nf_flow_dnat_port(const struct flow_offload *flow,
++ struct sk_buff *skb, unsigned int thoff,
++ u8 protocol, enum flow_offload_tuple_dir dir);
+
+ struct flow_ports {
+ __be16 source, dest;
+@@ -126,4 +301,41 @@ unsigned int nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb,
+ #define MODULE_ALIAS_NF_FLOWTABLE(family) \
+ MODULE_ALIAS("nf-flowtable-" __stringify(family))
+
++void nf_flow_offload_add(struct nf_flowtable *flowtable,
++ struct flow_offload *flow);
++void nf_flow_offload_del(struct nf_flowtable *flowtable,
++ struct flow_offload *flow);
++void nf_flow_offload_stats(struct nf_flowtable *flowtable,
++ struct flow_offload *flow, bool force);
++
++void nf_flow_table_offload_flush(struct nf_flowtable *flowtable);
++int nf_flow_table_offload_setup(struct nf_flowtable *flowtable,
++ struct net_device *dev,
++ enum flow_block_command cmd);
++int nf_flow_rule_route_ipv4(struct net *net, const struct flow_offload *flow,
++ enum flow_offload_tuple_dir dir,
++ struct nf_flow_rule *flow_rule);
++int nf_flow_rule_route_ipv6(struct net *net, const struct flow_offload *flow,
++ enum flow_offload_tuple_dir dir,
++ struct nf_flow_rule *flow_rule);
++
++int nf_flow_table_offload_init(void);
++void nf_flow_table_offload_exit(void);
++
++static inline __be16 nf_flow_pppoe_proto(const struct sk_buff *skb)
++{
++ __be16 proto;
++
++ proto = *((__be16 *)(skb_mac_header(skb) + ETH_HLEN +
++ sizeof(struct pppoe_hdr)));
++ switch (proto) {
++ case htons(PPP_IP):
++ return htons(ETH_P_IP);
++ case htons(PPP_IPV6):
++ return htons(ETH_P_IPV6);
++ }
++
++ return 0;
++}
++
+ #endif /* _NF_FLOW_TABLE_H */
+diff --git a/include/net/netns/conntrack.h b/include/net/netns/conntrack.h
+index 806454e..9e3963c 100644
+--- a/include/net/netns/conntrack.h
++++ b/include/net/netns/conntrack.h
+@@ -27,6 +27,9 @@ struct nf_tcp_net {
+ int tcp_loose;
+ int tcp_be_liberal;
+ int tcp_max_retrans;
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++ unsigned int offload_timeout;
++#endif
+ };
+
+ enum udp_conntrack {
+@@ -37,6 +40,9 @@ enum udp_conntrack {
+
+ struct nf_udp_net {
+ unsigned int timeouts[UDP_CT_MAX];
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++ unsigned int offload_timeout;
++#endif
+ };
+
+ struct nf_icmp_net {
+diff --git a/include/uapi/linux/netfilter/nf_conntrack_common.h b/include/uapi/linux/netfilter/nf_conntrack_common.h
+index 336014b..ae698d1 100644
+--- a/include/uapi/linux/netfilter/nf_conntrack_common.h
++++ b/include/uapi/linux/netfilter/nf_conntrack_common.h
+@@ -105,14 +105,19 @@ enum ip_conntrack_status {
+ IPS_OFFLOAD_BIT = 14,
+ IPS_OFFLOAD = (1 << IPS_OFFLOAD_BIT),
+
++ /* Conntrack has been offloaded to hardware. */
++ IPS_HW_OFFLOAD_BIT = 15,
++ IPS_HW_OFFLOAD = (1 << IPS_HW_OFFLOAD_BIT),
++
+ /* Be careful here, modifying these bits can make things messy,
+ * so don't let users modify them directly.
+ */
+ IPS_UNCHANGEABLE_MASK = (IPS_NAT_DONE_MASK | IPS_NAT_MASK |
+ IPS_EXPECTED | IPS_CONFIRMED | IPS_DYING |
+- IPS_SEQ_ADJUST | IPS_TEMPLATE | IPS_OFFLOAD),
++ IPS_SEQ_ADJUST | IPS_TEMPLATE |
++ IPS_OFFLOAD | IPS_HW_OFFLOAD),
+
+- __IPS_MAX_BIT = 15,
++ __IPS_MAX_BIT = 16,
+ };
+
+ /* Connection tracking event types */
+diff --git a/include/uapi/linux/netfilter/xt_FLOWOFFLOAD.h b/include/uapi/linux/netfilter/xt_FLOWOFFLOAD.h
+new file mode 100644
+index 0000000..5841bbe
+--- /dev/null
++++ b/include/uapi/linux/netfilter/xt_FLOWOFFLOAD.h
+@@ -0,0 +1,17 @@
++/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
++#ifndef _XT_FLOWOFFLOAD_H
++#define _XT_FLOWOFFLOAD_H
++
++#include <linux/types.h>
++
++enum {
++ XT_FLOWOFFLOAD_HW = 1 << 0,
++
++ XT_FLOWOFFLOAD_MASK = XT_FLOWOFFLOAD_HW
++};
++
++struct xt_flowoffload_target_info {
++ __u32 flags;
++};
++
++#endif /* _XT_FLOWOFFLOAD_H */
+diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c
+index dd7e09e..c373f1d 100644
+--- a/net/8021q/vlan_dev.c
++++ b/net/8021q/vlan_dev.c
+@@ -771,6 +771,26 @@ static int vlan_dev_get_iflink(const struct net_device *dev)
+ return real_dev->ifindex;
+ }
+
++static int vlan_dev_fill_forward_path(struct net_device_path_ctx *ctx,
++ struct net_device_path *path)
++{
++ struct vlan_dev_priv *vlan = vlan_dev_priv(ctx->dev);
++
++ path->type = DEV_PATH_VLAN;
++ path->encap.id = vlan->vlan_id;
++ path->encap.proto = vlan->vlan_proto;
++ path->dev = ctx->dev;
++ ctx->dev = vlan->real_dev;
++ if (ctx->num_vlans >= ARRAY_SIZE(ctx->vlan))
++ return -ENOSPC;
++
++ ctx->vlan[ctx->num_vlans].id = vlan->vlan_id;
++ ctx->vlan[ctx->num_vlans].proto = vlan->vlan_proto;
++ ctx->num_vlans++;
++
++ return 0;
++}
++
+ static const struct ethtool_ops vlan_ethtool_ops = {
+ .get_link_ksettings = vlan_ethtool_get_link_ksettings,
+ .get_drvinfo = vlan_ethtool_get_drvinfo,
+@@ -809,6 +829,7 @@ static const struct net_device_ops vlan_netdev_ops = {
+ #endif
+ .ndo_fix_features = vlan_dev_fix_features,
+ .ndo_get_iflink = vlan_dev_get_iflink,
++ .ndo_fill_forward_path = vlan_dev_fill_forward_path,
+ };
+
+ static void vlan_dev_free(struct net_device *dev)
+diff --git a/net/bridge/br_device.c b/net/bridge/br_device.c
+index 44bdb01..03934dd 100644
+--- a/net/bridge/br_device.c
++++ b/net/bridge/br_device.c
+@@ -387,6 +387,54 @@ static int br_del_slave(struct net_device *dev, struct net_device *slave_dev)
+ return br_del_if(br, slave_dev);
+ }
+
++static int br_fill_forward_path(struct net_device_path_ctx *ctx,
++ struct net_device_path *path)
++{
++ struct net_bridge_fdb_entry *f;
++ struct net_bridge_port *dst;
++ struct net_bridge *br;
++
++ if (netif_is_bridge_port(ctx->dev))
++ return -1;
++
++ br = netdev_priv(ctx->dev);
++
++ br_vlan_fill_forward_path_pvid(br, ctx, path);
++
++ f = br_fdb_find_rcu(br, ctx->daddr, path->bridge.vlan_id);
++ if (!f || !f->dst)
++ return -1;
++
++ dst = READ_ONCE(f->dst);
++ if (!dst)
++ return -1;
++
++ if (br_vlan_fill_forward_path_mode(br, dst, path))
++ return -1;
++
++ path->type = DEV_PATH_BRIDGE;
++ path->dev = dst->br->dev;
++ ctx->dev = dst->dev;
++
++ switch (path->bridge.vlan_mode) {
++ case DEV_PATH_BR_VLAN_TAG:
++ if (ctx->num_vlans >= ARRAY_SIZE(ctx->vlan))
++ return -ENOSPC;
++ ctx->vlan[ctx->num_vlans].id = path->bridge.vlan_id;
++ ctx->vlan[ctx->num_vlans].proto = path->bridge.vlan_proto;
++ ctx->num_vlans++;
++ break;
++ case DEV_PATH_BR_VLAN_UNTAG_HW:
++ case DEV_PATH_BR_VLAN_UNTAG:
++ ctx->num_vlans--;
++ break;
++ case DEV_PATH_BR_VLAN_KEEP:
++ break;
++ }
++
++ return 0;
++}
++
+ static const struct ethtool_ops br_ethtool_ops = {
+ .get_drvinfo = br_getinfo,
+ .get_link = ethtool_op_get_link,
+@@ -420,6 +468,7 @@ static const struct net_device_ops br_netdev_ops = {
+ .ndo_bridge_setlink = br_setlink,
+ .ndo_bridge_dellink = br_dellink,
+ .ndo_features_check = passthru_features_check,
++ .ndo_fill_forward_path = br_fill_forward_path,
+ };
+
+ static struct device_type br_type = {
+diff --git a/net/bridge/br_private.h b/net/bridge/br_private.h
+index a736be8..4bd9e9b 100644
+--- a/net/bridge/br_private.h
++++ b/net/bridge/br_private.h
+@@ -912,6 +912,13 @@ void br_vlan_port_event(struct net_bridge_port *p, unsigned long event);
+ int br_vlan_bridge_event(struct net_device *dev, unsigned long event,
+ void *ptr);
+
++void br_vlan_fill_forward_path_pvid(struct net_bridge *br,
++ struct net_device_path_ctx *ctx,
++ struct net_device_path *path);
++int br_vlan_fill_forward_path_mode(struct net_bridge *br,
++ struct net_bridge_port *dst,
++ struct net_device_path *path);
++
+ static inline struct net_bridge_vlan_group *br_vlan_group(
+ const struct net_bridge *br)
+ {
+@@ -1066,6 +1073,19 @@ static inline int nbp_get_num_vlan_infos(struct net_bridge_port *p,
+ return 0;
+ }
+
++static inline void br_vlan_fill_forward_path_pvid(struct net_bridge *br,
++ struct net_device_path_ctx *ctx,
++ struct net_device_path *path)
++{
++}
++
++static inline int br_vlan_fill_forward_path_mode(struct net_bridge *br,
++ struct net_bridge_port *dst,
++ struct net_device_path *path)
++{
++ return 0;
++}
++
+ static inline struct net_bridge_vlan_group *br_vlan_group(
+ const struct net_bridge *br)
+ {
+diff --git a/net/bridge/br_vlan.c b/net/bridge/br_vlan.c
+index 9257292..bcfd169 100644
+--- a/net/bridge/br_vlan.c
++++ b/net/bridge/br_vlan.c
+@@ -1268,6 +1268,61 @@ int br_vlan_get_pvid_rcu(const struct net_device *dev, u16 *p_pvid)
+ }
+ EXPORT_SYMBOL_GPL(br_vlan_get_pvid_rcu);
+
++void br_vlan_fill_forward_path_pvid(struct net_bridge *br,
++ struct net_device_path_ctx *ctx,
++ struct net_device_path *path)
++{
++ struct net_bridge_vlan_group *vg;
++ int idx = ctx->num_vlans - 1;
++ u16 vid;
++
++ path->bridge.vlan_mode = DEV_PATH_BR_VLAN_KEEP;
++
++ if (!br_opt_get(br, BROPT_VLAN_ENABLED))
++ return;
++
++ vg = br_vlan_group(br);
++
++ if (idx >= 0 &&
++ ctx->vlan[idx].proto == br->vlan_proto) {
++ vid = ctx->vlan[idx].id;
++ } else {
++ path->bridge.vlan_mode = DEV_PATH_BR_VLAN_TAG;
++ vid = br_get_pvid(vg);
++ }
++
++ path->bridge.vlan_id = vid;
++ path->bridge.vlan_proto = br->vlan_proto;
++}
++
++int br_vlan_fill_forward_path_mode(struct net_bridge *br,
++ struct net_bridge_port *dst,
++ struct net_device_path *path)
++{
++ struct net_bridge_vlan_group *vg;
++ struct net_bridge_vlan *v;
++
++ if (!br_opt_get(br, BROPT_VLAN_ENABLED))
++ return 0;
++
++ vg = nbp_vlan_group_rcu(dst);
++ v = br_vlan_find(vg, path->bridge.vlan_id);
++ if (!v || !br_vlan_should_use(v))
++ return -EINVAL;
++
++ if (!(v->flags & BRIDGE_VLAN_INFO_UNTAGGED))
++ return 0;
++
++ if (path->bridge.vlan_mode == DEV_PATH_BR_VLAN_TAG)
++ path->bridge.vlan_mode = DEV_PATH_BR_VLAN_KEEP;
++ else if (v->priv_flags & BR_VLFLAG_ADDED_BY_SWITCHDEV)
++ path->bridge.vlan_mode = DEV_PATH_BR_VLAN_UNTAG_HW;
++ else
++ path->bridge.vlan_mode = DEV_PATH_BR_VLAN_UNTAG;
++
++ return 0;
++}
++
+ int br_vlan_get_info(const struct net_device *dev, u16 vid,
+ struct bridge_vlan_info *p_vinfo)
+ {
+diff --git a/net/core/dev.c b/net/core/dev.c
+index 6c0aefe..7d29e7b 100644
+--- a/net/core/dev.c
++++ b/net/core/dev.c
+@@ -722,6 +722,52 @@ int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
+ }
+ EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
+
++static struct net_device_path *dev_fwd_path(struct net_device_path_stack *stack)
++{
++ int k = stack->num_paths++;
++
++ if (WARN_ON_ONCE(k >= NET_DEVICE_PATH_STACK_MAX))
++ return NULL;
++
++ return &stack->path[k];
++}
++
++int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr,
++ struct net_device_path_stack *stack)
++{
++ const struct net_device *last_dev;
++ struct net_device_path_ctx ctx = {
++ .dev = dev,
++ };
++ struct net_device_path *path;
++ int ret = 0;
++
++ memcpy(ctx.daddr, daddr, sizeof(ctx.daddr));
++ stack->num_paths = 0;
++ while (ctx.dev && ctx.dev->netdev_ops->ndo_fill_forward_path) {
++ last_dev = ctx.dev;
++ path = dev_fwd_path(stack);
++ if (!path)
++ return -1;
++
++ memset(path, 0, sizeof(struct net_device_path));
++ ret = ctx.dev->netdev_ops->ndo_fill_forward_path(&ctx, path);
++ if (ret < 0)
++ return -1;
++
++ if (WARN_ON_ONCE(last_dev == ctx.dev))
++ return -1;
++ }
++ path = dev_fwd_path(stack);
++ if (!path)
++ return -1;
++ path->type = DEV_PATH_ETHERNET;
++ path->dev = ctx.dev;
++
++ return ret;
++}
++EXPORT_SYMBOL_GPL(dev_fill_forward_path);
++
+ /**
+ * __dev_get_by_name - find a device by its name
+ * @net: the applicable net namespace
+diff --git a/net/dsa/slave.c b/net/dsa/slave.c
+index cd5b354..2ea9ec1 100644
+--- a/net/dsa/slave.c
++++ b/net/dsa/slave.c
+@@ -1031,14 +1031,32 @@ static int dsa_slave_setup_tc_block(struct net_device *dev,
+ }
+ }
+
++static int dsa_slave_setup_ft_block(struct dsa_switch *ds, int port,
++ void *type_data)
++{
++ struct dsa_port *cpu_dp = dsa_to_port(ds, port)->cpu_dp;
++ struct net_device *master = cpu_dp->master;
++
++ if (!master->netdev_ops->ndo_setup_tc)
++ return -EOPNOTSUPP;
++
++ return master->netdev_ops->ndo_setup_tc(master, TC_SETUP_FT, type_data);
++}
++
+ static int dsa_slave_setup_tc(struct net_device *dev, enum tc_setup_type type,
+ void *type_data)
+ {
+ struct dsa_port *dp = dsa_slave_to_port(dev);
+ struct dsa_switch *ds = dp->ds;
+
+- if (type == TC_SETUP_BLOCK)
++ switch (type) {
++ case TC_SETUP_BLOCK:
+ return dsa_slave_setup_tc_block(dev, type_data);
++ case TC_SETUP_FT:
++ return dsa_slave_setup_ft_block(ds, dp->index, type_data);
++ default:
++ break;
++ }
+
+ if (!ds->ops->port_setup_tc)
+ return -EOPNOTSUPP;
+@@ -1224,6 +1242,21 @@ static struct devlink_port *dsa_slave_get_devlink_port(struct net_device *dev)
+ return dp->ds->devlink ? &dp->devlink_port : NULL;
+ }
+
++static int dsa_slave_fill_forward_path(struct net_device_path_ctx *ctx,
++ struct net_device_path *path)
++{
++ struct dsa_port *dp = dsa_slave_to_port(ctx->dev);
++ struct dsa_port *cpu_dp = dp->cpu_dp;
++
++ path->dev = ctx->dev;
++ path->type = DEV_PATH_DSA;
++ path->dsa.proto = cpu_dp->tag_ops->proto;
++ path->dsa.port = dp->index;
++ ctx->dev = cpu_dp->master;
++
++ return 0;
++}
++
+ static const struct net_device_ops dsa_slave_netdev_ops = {
+ .ndo_open = dsa_slave_open,
+ .ndo_stop = dsa_slave_close,
+@@ -1248,6 +1281,7 @@ static const struct net_device_ops dsa_slave_netdev_ops = {
+ .ndo_vlan_rx_add_vid = dsa_slave_vlan_rx_add_vid,
+ .ndo_vlan_rx_kill_vid = dsa_slave_vlan_rx_kill_vid,
+ .ndo_get_devlink_port = dsa_slave_get_devlink_port,
++ .ndo_fill_forward_path = dsa_slave_fill_forward_path,
+ };
+
+ static struct device_type dsa_type = {
+diff --git a/net/ipv4/netfilter/Kconfig b/net/ipv4/netfilter/Kconfig
+index f17b402..803b92e 100644
+--- a/net/ipv4/netfilter/Kconfig
++++ b/net/ipv4/netfilter/Kconfig
+@@ -56,8 +56,6 @@ config NF_TABLES_ARP
+ help
+ This option enables the ARP support for nf_tables.
+
+-endif # NF_TABLES
+-
+ config NF_FLOW_TABLE_IPV4
+ tristate "Netfilter flow table IPv4 module"
+ depends on NF_FLOW_TABLE
+@@ -66,6 +64,8 @@ config NF_FLOW_TABLE_IPV4
+
+ To compile it as a module, choose M here.
+
++endif # NF_TABLES
++
+ config NF_DUP_IPV4
+ tristate "Netfilter IPv4 packet duplication to alternate destination"
+ depends on !NF_CONNTRACK || NF_CONNTRACK
+diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c
+index c67d634..ef836d6 100644
+--- a/net/ipv6/ip6_output.c
++++ b/net/ipv6/ip6_output.c
+@@ -613,7 +613,7 @@ int ip6_forward(struct sk_buff *skb)
+ }
+ }
+
+- mtu = ip6_dst_mtu_forward(dst);
++ mtu = ip6_dst_mtu_maybe_forward(dst, true);
+ if (mtu < IPV6_MIN_MTU)
+ mtu = IPV6_MIN_MTU;
+
+diff --git a/net/ipv6/netfilter/Kconfig b/net/ipv6/netfilter/Kconfig
+index 69443e9..0b481d2 100644
+--- a/net/ipv6/netfilter/Kconfig
++++ b/net/ipv6/netfilter/Kconfig
+@@ -45,7 +45,6 @@ config NFT_FIB_IPV6
+ multicast or blackhole.
+
+ endif # NF_TABLES_IPV6
+-endif # NF_TABLES
+
+ config NF_FLOW_TABLE_IPV6
+ tristate "Netfilter flow table IPv6 module"
+@@ -55,6 +54,8 @@ config NF_FLOW_TABLE_IPV6
+
+ To compile it as a module, choose M here.
+
++endif # NF_TABLES
++
+ config NF_DUP_IPV6
+ tristate "Netfilter IPv6 packet duplication to alternate destination"
+ depends on !NF_CONNTRACK || NF_CONNTRACK
+diff --git a/net/ipv6/route.c b/net/ipv6/route.c
+index f0677e1..d928855 100644
+--- a/net/ipv6/route.c
++++ b/net/ipv6/route.c
+@@ -83,7 +83,7 @@ enum rt6_nud_state {
+
+ static struct dst_entry *ip6_dst_check(struct dst_entry *dst, u32 cookie);
+ static unsigned int ip6_default_advmss(const struct dst_entry *dst);
+-static unsigned int ip6_mtu(const struct dst_entry *dst);
++static unsigned int ip6_mtu(const struct dst_entry *dst);
+ static void ip6_negative_advice(struct sock *sk,
+ struct dst_entry *dst);
+ static void ip6_dst_destroy(struct dst_entry *);
+@@ -3129,25 +3129,7 @@ static unsigned int ip6_default_advmss(const struct dst_entry *dst)
+
+ static unsigned int ip6_mtu(const struct dst_entry *dst)
+ {
+- struct inet6_dev *idev;
+- unsigned int mtu;
+-
+- mtu = dst_metric_raw(dst, RTAX_MTU);
+- if (mtu)
+- goto out;
+-
+- mtu = IPV6_MIN_MTU;
+-
+- rcu_read_lock();
+- idev = __in6_dev_get(dst->dev);
+- if (idev)
+- mtu = idev->cnf.mtu6;
+- rcu_read_unlock();
+-
+-out:
+- mtu = min_t(unsigned int, mtu, IP6_MAX_MTU);
+-
+- return mtu - lwtunnel_headroom(dst->lwtstate, mtu);
++ return ip6_dst_mtu_maybe_forward(dst, false);
+ }
+
+ /* MTU selection:
+diff --git a/net/netfilter/Kconfig b/net/netfilter/Kconfig
+index b6e0a62..5d690ab 100644
+--- a/net/netfilter/Kconfig
++++ b/net/netfilter/Kconfig
+@@ -689,8 +689,6 @@ config NFT_FIB_NETDEV
+
+ endif # NF_TABLES_NETDEV
+
+-endif # NF_TABLES
+-
+ config NF_FLOW_TABLE_INET
+ tristate "Netfilter flow table mixed IPv4/IPv6 module"
+ depends on NF_FLOW_TABLE
+@@ -699,11 +697,12 @@ config NF_FLOW_TABLE_INET
+
+ To compile it as a module, choose M here.
+
++endif # NF_TABLES
++
+ config NF_FLOW_TABLE
+ tristate "Netfilter flow table module"
+ depends on NETFILTER_INGRESS
+ depends on NF_CONNTRACK
+- depends on NF_TABLES
+ help
+ This option adds the flow table core infrastructure.
+
+@@ -983,6 +982,15 @@ config NETFILTER_XT_TARGET_NOTRACK
+ depends on NETFILTER_ADVANCED
+ select NETFILTER_XT_TARGET_CT
+
++config NETFILTER_XT_TARGET_FLOWOFFLOAD
++ tristate '"FLOWOFFLOAD" target support'
++ depends on NF_FLOW_TABLE
++ depends on NETFILTER_INGRESS
++ help
++ This option adds a `FLOWOFFLOAD' target, which uses the nf_flow_offload
++ module to speed up processing of packets by bypassing the usual
++ netfilter chains
++
+ config NETFILTER_XT_TARGET_RATEEST
+ tristate '"RATEEST" target support'
+ depends on NETFILTER_ADVANCED
+diff --git a/net/netfilter/Makefile b/net/netfilter/Makefile
+index 4fc075b..d93a121 100644
+--- a/net/netfilter/Makefile
++++ b/net/netfilter/Makefile
+@@ -120,7 +120,8 @@ obj-$(CONFIG_NFT_FWD_NETDEV) += nft_fwd_netdev.o
+
+ # flow table infrastructure
+ obj-$(CONFIG_NF_FLOW_TABLE) += nf_flow_table.o
+-nf_flow_table-objs := nf_flow_table_core.o nf_flow_table_ip.o
++nf_flow_table-objs := nf_flow_table_core.o nf_flow_table_ip.o \
++ nf_flow_table_offload.o
+
+ obj-$(CONFIG_NF_FLOW_TABLE_INET) += nf_flow_table_inet.o
+
+@@ -140,6 +141,7 @@ obj-$(CONFIG_NETFILTER_XT_TARGET_CLASSIFY) += xt_CLASSIFY.o
+ obj-$(CONFIG_NETFILTER_XT_TARGET_CONNSECMARK) += xt_CONNSECMARK.o
+ obj-$(CONFIG_NETFILTER_XT_TARGET_CT) += xt_CT.o
+ obj-$(CONFIG_NETFILTER_XT_TARGET_DSCP) += xt_DSCP.o
++obj-$(CONFIG_NETFILTER_XT_TARGET_FLOWOFFLOAD) += xt_FLOWOFFLOAD.o
+ obj-$(CONFIG_NETFILTER_XT_TARGET_HL) += xt_HL.o
+ obj-$(CONFIG_NETFILTER_XT_TARGET_HMARK) += xt_HMARK.o
+ obj-$(CONFIG_NETFILTER_XT_TARGET_LED) += xt_LED.o
+diff --git a/net/netfilter/nf_conntrack_core.c b/net/netfilter/nf_conntrack_core.c
+index 706d180..65d16ef 100644
+--- a/net/netfilter/nf_conntrack_core.c
++++ b/net/netfilter/nf_conntrack_core.c
+@@ -864,9 +864,8 @@ nf_conntrack_hash_check_insert(struct nf_conn *ct)
+ }
+ EXPORT_SYMBOL_GPL(nf_conntrack_hash_check_insert);
+
+-static inline void nf_ct_acct_update(struct nf_conn *ct,
+- enum ip_conntrack_info ctinfo,
+- unsigned int len)
++void nf_ct_acct_add(struct nf_conn *ct, u32 dir, unsigned int packets,
++ unsigned int bytes)
+ {
+ struct nf_conn_acct *acct;
+
+@@ -874,10 +873,11 @@ static inline void nf_ct_acct_update(struct nf_conn *ct,
+ if (acct) {
+ struct nf_conn_counter *counter = acct->counter;
+
+- atomic64_inc(&counter[CTINFO2DIR(ctinfo)].packets);
+- atomic64_add(len, &counter[CTINFO2DIR(ctinfo)].bytes);
++ atomic64_add(packets, &counter[dir].packets);
++ atomic64_add(bytes, &counter[dir].bytes);
+ }
+ }
++EXPORT_SYMBOL_GPL(nf_ct_acct_add);
+
+ static void nf_ct_acct_merge(struct nf_conn *ct, enum ip_conntrack_info ctinfo,
+ const struct nf_conn *loser_ct)
+@@ -891,7 +891,7 @@ static void nf_ct_acct_merge(struct nf_conn *ct, enum ip_conntrack_info ctinfo,
+
+ /* u32 should be fine since we must have seen one packet. */
+ bytes = atomic64_read(&counter[CTINFO2DIR(ctinfo)].bytes);
+- nf_ct_acct_update(ct, ctinfo, bytes);
++ nf_ct_acct_update(ct, CTINFO2DIR(ctinfo), bytes);
+ }
+ }
+
+@@ -1238,8 +1238,10 @@ static void gc_worker(struct work_struct *work)
+
+ tmp = nf_ct_tuplehash_to_ctrack(h);
+
+- if (test_bit(IPS_OFFLOAD_BIT, &tmp->status))
++ if (test_bit(IPS_OFFLOAD_BIT, &tmp->status)) {
++ nf_ct_offload_timeout(tmp);
+ continue;
++ }
+
+ if (nf_ct_is_expired(tmp)) {
+ nf_ct_gc_expired(tmp);
+@@ -1763,7 +1765,7 @@ void __nf_ct_refresh_acct(struct nf_conn *ct,
+ WRITE_ONCE(ct->timeout, extra_jiffies);
+ acct:
+ if (do_acct)
+- nf_ct_acct_update(ct, ctinfo, skb->len);
++ nf_ct_acct_update(ct, CTINFO2DIR(ctinfo), skb->len);
+ }
+ EXPORT_SYMBOL_GPL(__nf_ct_refresh_acct);
+
+@@ -1771,7 +1773,7 @@ bool nf_ct_kill_acct(struct nf_conn *ct,
+ enum ip_conntrack_info ctinfo,
+ const struct sk_buff *skb)
+ {
+- nf_ct_acct_update(ct, ctinfo, skb->len);
++ nf_ct_acct_update(ct, CTINFO2DIR(ctinfo), skb->len);
+
+ return nf_ct_delete(ct, 0, 0);
+ }
+diff --git a/net/netfilter/nf_conntrack_proto_tcp.c b/net/netfilter/nf_conntrack_proto_tcp.c
+index e219b6f..65e406f 100644
+--- a/net/netfilter/nf_conntrack_proto_tcp.c
++++ b/net/netfilter/nf_conntrack_proto_tcp.c
+@@ -840,12 +840,6 @@ static noinline bool tcp_new(struct nf_conn *ct, const struct sk_buff *skb,
+ return true;
+ }
+
+-static bool nf_conntrack_tcp_established(const struct nf_conn *ct)
+-{
+- return ct->proto.tcp.state == TCP_CONNTRACK_ESTABLISHED &&
+- test_bit(IPS_ASSURED_BIT, &ct->status);
+-}
+-
+ static void nf_ct_tcp_state_reset(struct ip_ct_tcp_state *state)
+ {
+ state->td_end = 0;
+@@ -1463,6 +1457,10 @@ void nf_conntrack_tcp_init_net(struct net *net)
+ tn->tcp_loose = nf_ct_tcp_loose;
+ tn->tcp_be_liberal = nf_ct_tcp_be_liberal;
+ tn->tcp_max_retrans = nf_ct_tcp_max_retrans;
++
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++ tn->offload_timeout = 30 * HZ;
++#endif
+ }
+
+ const struct nf_conntrack_l4proto nf_conntrack_l4proto_tcp =
+diff --git a/net/netfilter/nf_conntrack_proto_udp.c b/net/netfilter/nf_conntrack_proto_udp.c
+index e3a2d01..a1579d6 100644
+--- a/net/netfilter/nf_conntrack_proto_udp.c
++++ b/net/netfilter/nf_conntrack_proto_udp.c
+@@ -267,6 +267,10 @@ void nf_conntrack_udp_init_net(struct net *net)
+
+ for (i = 0; i < UDP_CT_MAX; i++)
+ un->timeouts[i] = udp_timeouts[i];
++
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++ un->offload_timeout = 30 * HZ;
++#endif
+ }
+
+ const struct nf_conntrack_l4proto nf_conntrack_l4proto_udp =
+diff --git a/net/netfilter/nf_conntrack_standalone.c b/net/netfilter/nf_conntrack_standalone.c
+index 0b600b4..a2cfafa 100644
+--- a/net/netfilter/nf_conntrack_standalone.c
++++ b/net/netfilter/nf_conntrack_standalone.c
+@@ -353,7 +353,9 @@ static int ct_seq_show(struct seq_file *s, void *v)
+ if (seq_print_acct(s, ct, IP_CT_DIR_REPLY))
+ goto release;
+
+- if (test_bit(IPS_OFFLOAD_BIT, &ct->status))
++ if (test_bit(IPS_HW_OFFLOAD_BIT, &ct->status))
++ seq_puts(s, "[HW_OFFLOAD] ");
++ else if (test_bit(IPS_OFFLOAD_BIT, &ct->status))
+ seq_puts(s, "[OFFLOAD] ");
+ else if (test_bit(IPS_ASSURED_BIT, &ct->status))
+ seq_puts(s, "[ASSURED] ");
+@@ -620,11 +622,17 @@ enum nf_ct_sysctl_index {
+ NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_CLOSE,
+ NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_RETRANS,
+ NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_UNACK,
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++ NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_OFFLOAD,
++#endif
+ NF_SYSCTL_CT_PROTO_TCP_LOOSE,
+ NF_SYSCTL_CT_PROTO_TCP_LIBERAL,
+ NF_SYSCTL_CT_PROTO_TCP_MAX_RETRANS,
+ NF_SYSCTL_CT_PROTO_TIMEOUT_UDP,
+ NF_SYSCTL_CT_PROTO_TIMEOUT_UDP_STREAM,
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++ NF_SYSCTL_CT_PROTO_TIMEOUT_UDP_OFFLOAD,
++#endif
+ NF_SYSCTL_CT_PROTO_TIMEOUT_ICMP,
+ NF_SYSCTL_CT_PROTO_TIMEOUT_ICMPV6,
+ #ifdef CONFIG_NF_CT_PROTO_SCTP
+@@ -811,6 +819,14 @@ static struct ctl_table nf_ct_sysctl_table[] = {
+ .mode = 0644,
+ .proc_handler = proc_dointvec_jiffies,
+ },
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++ [NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_OFFLOAD] = {
++ .procname = "nf_flowtable_tcp_timeout",
++ .maxlen = sizeof(unsigned int),
++ .mode = 0644,
++ .proc_handler = proc_dointvec_jiffies,
++ },
++#endif
+ [NF_SYSCTL_CT_PROTO_TCP_LOOSE] = {
+ .procname = "nf_conntrack_tcp_loose",
+ .maxlen = sizeof(int),
+@@ -845,6 +861,14 @@ static struct ctl_table nf_ct_sysctl_table[] = {
+ .mode = 0644,
+ .proc_handler = proc_dointvec_jiffies,
+ },
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++ [NF_SYSCTL_CT_PROTO_TIMEOUT_UDP_OFFLOAD] = {
++ .procname = "nf_flowtable_udp_timeout",
++ .maxlen = sizeof(unsigned int),
++ .mode = 0644,
++ .proc_handler = proc_dointvec_jiffies,
++ },
++#endif
+ [NF_SYSCTL_CT_PROTO_TIMEOUT_ICMP] = {
+ .procname = "nf_conntrack_icmp_timeout",
+ .maxlen = sizeof(unsigned int),
+@@ -1021,6 +1045,11 @@ static void nf_conntrack_standalone_init_tcp_sysctl(struct net *net,
+ XASSIGN(LIBERAL, &tn->tcp_be_liberal);
+ XASSIGN(MAX_RETRANS, &tn->tcp_max_retrans);
+ #undef XASSIGN
++
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++ table[NF_SYSCTL_CT_PROTO_TIMEOUT_TCP_OFFLOAD].data = &tn->offload_timeout;
++#endif
++
+ }
+
+ static void nf_conntrack_standalone_init_sctp_sysctl(struct net *net,
+@@ -1107,6 +1136,9 @@ static int nf_conntrack_standalone_init_sysctl(struct net *net)
+ table[NF_SYSCTL_CT_PROTO_TIMEOUT_ICMPV6].data = &nf_icmpv6_pernet(net)->timeout;
+ table[NF_SYSCTL_CT_PROTO_TIMEOUT_UDP].data = &un->timeouts[UDP_CT_UNREPLIED];
+ table[NF_SYSCTL_CT_PROTO_TIMEOUT_UDP_STREAM].data = &un->timeouts[UDP_CT_REPLIED];
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++ table[NF_SYSCTL_CT_PROTO_TIMEOUT_UDP_OFFLOAD].data = &un->offload_timeout;
++#endif
+
+ nf_conntrack_standalone_init_tcp_sysctl(net, table);
+ nf_conntrack_standalone_init_sctp_sysctl(net, table);
+diff --git a/net/netfilter/nf_flow_table_core.c b/net/netfilter/nf_flow_table_core.c
+index f212cec..a342d92 100644
+--- a/net/netfilter/nf_flow_table_core.c
++++ b/net/netfilter/nf_flow_table_core.c
+@@ -7,43 +7,21 @@
+ #include <linux/netdevice.h>
+ #include <net/ip.h>
+ #include <net/ip6_route.h>
+-#include <net/netfilter/nf_tables.h>
+ #include <net/netfilter/nf_flow_table.h>
+ #include <net/netfilter/nf_conntrack.h>
+ #include <net/netfilter/nf_conntrack_core.h>
+ #include <net/netfilter/nf_conntrack_l4proto.h>
+ #include <net/netfilter/nf_conntrack_tuple.h>
+
+-struct flow_offload_entry {
+- struct flow_offload flow;
+- struct nf_conn *ct;
+- struct rcu_head rcu_head;
+-};
+-
+ static DEFINE_MUTEX(flowtable_lock);
+ static LIST_HEAD(flowtables);
+
+-static u32 flow_offload_dst_cookie(struct flow_offload_tuple *flow_tuple)
+-{
+- const struct rt6_info *rt;
+-
+- if (flow_tuple->l3proto == NFPROTO_IPV6) {
+- rt = (const struct rt6_info *)flow_tuple->dst_cache;
+- return rt6_get_cookie(rt);
+- }
+-
+- return 0;
+-}
+-
+ static void
+-flow_offload_fill_dir(struct flow_offload *flow, struct nf_conn *ct,
+- struct nf_flow_route *route,
++flow_offload_fill_dir(struct flow_offload *flow,
+ enum flow_offload_tuple_dir dir)
+ {
+ struct flow_offload_tuple *ft = &flow->tuplehash[dir].tuple;
+- struct nf_conntrack_tuple *ctt = &ct->tuplehash[dir].tuple;
+- struct dst_entry *other_dst = route->tuple[!dir].dst;
+- struct dst_entry *dst = route->tuple[dir].dst;
++ struct nf_conntrack_tuple *ctt = &flow->ct->tuplehash[dir].tuple;
+
+ ft->dir = dir;
+
+@@ -51,12 +29,10 @@ flow_offload_fill_dir(struct flow_offload *flow, struct nf_conn *ct,
+ case NFPROTO_IPV4:
+ ft->src_v4 = ctt->src.u3.in;
+ ft->dst_v4 = ctt->dst.u3.in;
+- ft->mtu = ip_dst_mtu_maybe_forward(dst, true);
+ break;
+ case NFPROTO_IPV6:
+ ft->src_v6 = ctt->src.u3.in6;
+ ft->dst_v6 = ctt->dst.u3.in6;
+- ft->mtu = ip6_dst_mtu_forward(dst);
+ break;
+ }
+
+@@ -64,50 +40,32 @@ flow_offload_fill_dir(struct flow_offload *flow, struct nf_conn *ct,
+ ft->l4proto = ctt->dst.protonum;
+ ft->src_port = ctt->src.u.tcp.port;
+ ft->dst_port = ctt->dst.u.tcp.port;
+-
+- ft->iifidx = other_dst->dev->ifindex;
+- ft->dst_cache = dst;
+- ft->dst_cookie = flow_offload_dst_cookie(ft);
+ }
+
+-struct flow_offload *
+-flow_offload_alloc(struct nf_conn *ct, struct nf_flow_route *route)
++struct flow_offload *flow_offload_alloc(struct nf_conn *ct)
+ {
+- struct flow_offload_entry *entry;
+ struct flow_offload *flow;
+
+ if (unlikely(nf_ct_is_dying(ct) ||
+ !atomic_inc_not_zero(&ct->ct_general.use)))
+ return NULL;
+
+- entry = kzalloc(sizeof(*entry), GFP_ATOMIC);
+- if (!entry)
++ flow = kzalloc(sizeof(*flow), GFP_ATOMIC);
++ if (!flow)
+ goto err_ct_refcnt;
+
+- flow = &entry->flow;
+-
+- if (!dst_hold_safe(route->tuple[FLOW_OFFLOAD_DIR_ORIGINAL].dst))
+- goto err_dst_cache_original;
+-
+- if (!dst_hold_safe(route->tuple[FLOW_OFFLOAD_DIR_REPLY].dst))
+- goto err_dst_cache_reply;
++ flow->ct = ct;
+
+- entry->ct = ct;
+-
+- flow_offload_fill_dir(flow, ct, route, FLOW_OFFLOAD_DIR_ORIGINAL);
+- flow_offload_fill_dir(flow, ct, route, FLOW_OFFLOAD_DIR_REPLY);
++ flow_offload_fill_dir(flow, FLOW_OFFLOAD_DIR_ORIGINAL);
++ flow_offload_fill_dir(flow, FLOW_OFFLOAD_DIR_REPLY);
+
+ if (ct->status & IPS_SRC_NAT)
+- flow->flags |= FLOW_OFFLOAD_SNAT;
++ __set_bit(NF_FLOW_SNAT, &flow->flags);
+ if (ct->status & IPS_DST_NAT)
+- flow->flags |= FLOW_OFFLOAD_DNAT;
++ __set_bit(NF_FLOW_DNAT, &flow->flags);
+
+ return flow;
+
+-err_dst_cache_reply:
+- dst_release(route->tuple[FLOW_OFFLOAD_DIR_ORIGINAL].dst);
+-err_dst_cache_original:
+- kfree(entry);
+ err_ct_refcnt:
+ nf_ct_put(ct);
+
+@@ -115,65 +73,158 @@ flow_offload_alloc(struct nf_conn *ct, struct nf_flow_route *route)
+ }
+ EXPORT_SYMBOL_GPL(flow_offload_alloc);
+
+-static void flow_offload_fixup_tcp(struct ip_ct_tcp *tcp)
++static u32 flow_offload_dst_cookie(struct flow_offload_tuple *flow_tuple)
+ {
+- tcp->state = TCP_CONNTRACK_ESTABLISHED;
+- tcp->seen[0].td_maxwin = 0;
+- tcp->seen[1].td_maxwin = 0;
++ const struct rt6_info *rt;
++
++ if (flow_tuple->l3proto == NFPROTO_IPV6) {
++ rt = (const struct rt6_info *)flow_tuple->dst_cache;
++ return rt6_get_cookie(rt);
++ }
++
++ return 0;
+ }
+
+-#define NF_FLOWTABLE_TCP_PICKUP_TIMEOUT (120 * HZ)
+-#define NF_FLOWTABLE_UDP_PICKUP_TIMEOUT (30 * HZ)
++static int flow_offload_fill_route(struct flow_offload *flow,
++ const struct nf_flow_route *route,
++ enum flow_offload_tuple_dir dir)
++{
++ struct flow_offload_tuple *flow_tuple = &flow->tuplehash[dir].tuple;
++ struct dst_entry *dst = route->tuple[dir].dst;
++ int i, j = 0;
++
++ switch (flow_tuple->l3proto) {
++ case NFPROTO_IPV4:
++ flow_tuple->mtu = ip_dst_mtu_maybe_forward(dst, true);
++ break;
++ case NFPROTO_IPV6:
++ flow_tuple->mtu = ip6_dst_mtu_maybe_forward(dst, true);
++ break;
++ }
++
++ flow_tuple->iifidx = route->tuple[dir].in.ifindex;
++ for (i = route->tuple[dir].in.num_encaps - 1; i >= 0; i--) {
++ flow_tuple->encap[j].id = route->tuple[dir].in.encap[i].id;
++ flow_tuple->encap[j].proto = route->tuple[dir].in.encap[i].proto;
++ if (route->tuple[dir].in.ingress_vlans & BIT(i))
++ flow_tuple->in_vlan_ingress |= BIT(j);
++ j++;
++ }
++ flow_tuple->encap_num = route->tuple[dir].in.num_encaps;
++
++ switch (route->tuple[dir].xmit_type) {
++ case FLOW_OFFLOAD_XMIT_DIRECT:
++ memcpy(flow_tuple->out.h_dest, route->tuple[dir].out.h_dest,
++ ETH_ALEN);
++ memcpy(flow_tuple->out.h_source, route->tuple[dir].out.h_source,
++ ETH_ALEN);
++ flow_tuple->out.ifidx = route->tuple[dir].out.ifindex;
++ flow_tuple->out.hw_ifidx = route->tuple[dir].out.hw_ifindex;
++ break;
++ case FLOW_OFFLOAD_XMIT_XFRM:
++ case FLOW_OFFLOAD_XMIT_NEIGH:
++ if (!dst_hold_safe(route->tuple[dir].dst))
++ return -1;
++
++ flow_tuple->dst_cache = dst;
++ flow_tuple->dst_cookie = flow_offload_dst_cookie(flow_tuple);
++ break;
++ default:
++ WARN_ON_ONCE(1);
++ break;
++ }
++ flow_tuple->xmit_type = route->tuple[dir].xmit_type;
++
++ return 0;
++}
+
+-static inline __s32 nf_flow_timeout_delta(unsigned int timeout)
++static void nft_flow_dst_release(struct flow_offload *flow,
++ enum flow_offload_tuple_dir dir)
+ {
+- return (__s32)(timeout - (u32)jiffies);
++ if (flow->tuplehash[dir].tuple.xmit_type == FLOW_OFFLOAD_XMIT_NEIGH ||
++ flow->tuplehash[dir].tuple.xmit_type == FLOW_OFFLOAD_XMIT_XFRM)
++ dst_release(flow->tuplehash[dir].tuple.dst_cache);
+ }
+
+-static void flow_offload_fixup_ct_timeout(struct nf_conn *ct)
++int flow_offload_route_init(struct flow_offload *flow,
++ const struct nf_flow_route *route)
+ {
+- const struct nf_conntrack_l4proto *l4proto;
+- int l4num = nf_ct_protonum(ct);
+- unsigned int timeout;
++ int err;
+
+- l4proto = nf_ct_l4proto_find(l4num);
+- if (!l4proto)
+- return;
++ err = flow_offload_fill_route(flow, route, FLOW_OFFLOAD_DIR_ORIGINAL);
++ if (err < 0)
++ return err;
+
+- if (l4num == IPPROTO_TCP)
+- timeout = NF_FLOWTABLE_TCP_PICKUP_TIMEOUT;
+- else if (l4num == IPPROTO_UDP)
+- timeout = NF_FLOWTABLE_UDP_PICKUP_TIMEOUT;
+- else
+- return;
++ err = flow_offload_fill_route(flow, route, FLOW_OFFLOAD_DIR_REPLY);
++ if (err < 0)
++ goto err_route_reply;
++
++ flow->type = NF_FLOW_OFFLOAD_ROUTE;
++
++ return 0;
++
++err_route_reply:
++ nft_flow_dst_release(flow, FLOW_OFFLOAD_DIR_ORIGINAL);
+
+- if (nf_flow_timeout_delta(ct->timeout) > (__s32)timeout)
+- ct->timeout = nfct_time_stamp + timeout;
++ return err;
+ }
++EXPORT_SYMBOL_GPL(flow_offload_route_init);
+
+-static void flow_offload_fixup_ct_state(struct nf_conn *ct)
++static void flow_offload_fixup_tcp(struct ip_ct_tcp *tcp)
+ {
+- if (nf_ct_protonum(ct) == IPPROTO_TCP)
+- flow_offload_fixup_tcp(&ct->proto.tcp);
++ tcp->seen[0].td_maxwin = 0;
++ tcp->seen[1].td_maxwin = 0;
+ }
+
+ static void flow_offload_fixup_ct(struct nf_conn *ct)
+ {
+- flow_offload_fixup_ct_state(ct);
+- flow_offload_fixup_ct_timeout(ct);
++ struct net *net = nf_ct_net(ct);
++ int l4num = nf_ct_protonum(ct);
++ s32 timeout;
++
++ if (l4num == IPPROTO_TCP) {
++ struct nf_tcp_net *tn = nf_tcp_pernet(net);
++
++ flow_offload_fixup_tcp(&ct->proto.tcp);
++
++ timeout = tn->timeouts[ct->proto.tcp.state];
++ timeout -= tn->offload_timeout;
++ } else if (l4num == IPPROTO_UDP) {
++ struct nf_udp_net *tn = nf_udp_pernet(net);
++ enum udp_conntrack state =
++ test_bit(IPS_SEEN_REPLY_BIT, &ct->status) ?
++ UDP_CT_REPLIED : UDP_CT_UNREPLIED;
++
++ timeout = tn->timeouts[state];
++ timeout -= tn->offload_timeout;
++ } else {
++ return;
++ }
++
++ if (timeout < 0)
++ timeout = 0;
++
++ if (nf_flow_timeout_delta(READ_ONCE(ct->timeout)) > (__s32)timeout)
++ WRITE_ONCE(ct->timeout, nfct_time_stamp + timeout);
+ }
+
+-void flow_offload_free(struct flow_offload *flow)
++static void flow_offload_route_release(struct flow_offload *flow)
+ {
+- struct flow_offload_entry *e;
++ nft_flow_dst_release(flow, FLOW_OFFLOAD_DIR_ORIGINAL);
++ nft_flow_dst_release(flow, FLOW_OFFLOAD_DIR_REPLY);
++}
+
+- dst_release(flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_cache);
+- dst_release(flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_cache);
+- e = container_of(flow, struct flow_offload_entry, flow);
+- if (flow->flags & FLOW_OFFLOAD_DYING)
+- nf_ct_delete(e->ct, 0, 0);
+- nf_ct_put(e->ct);
+- kfree_rcu(e, rcu_head);
++void flow_offload_free(struct flow_offload *flow)
++{
++ switch (flow->type) {
++ case NF_FLOW_OFFLOAD_ROUTE:
++ flow_offload_route_release(flow);
++ break;
++ default:
++ break;
++ }
++ nf_ct_put(flow->ct);
++ kfree_rcu(flow, rcu_head);
+ }
+ EXPORT_SYMBOL_GPL(flow_offload_free);
+
+@@ -181,14 +232,14 @@ static u32 flow_offload_hash(const void *data, u32 len, u32 seed)
+ {
+ const struct flow_offload_tuple *tuple = data;
+
+- return jhash(tuple, offsetof(struct flow_offload_tuple, dir), seed);
++ return jhash(tuple, offsetof(struct flow_offload_tuple, __hash), seed);
+ }
+
+ static u32 flow_offload_hash_obj(const void *data, u32 len, u32 seed)
+ {
+ const struct flow_offload_tuple_rhash *tuplehash = data;
+
+- return jhash(&tuplehash->tuple, offsetof(struct flow_offload_tuple, dir), seed);
++ return jhash(&tuplehash->tuple, offsetof(struct flow_offload_tuple, __hash), seed);
+ }
+
+ static int flow_offload_hash_cmp(struct rhashtable_compare_arg *arg,
+@@ -197,7 +248,7 @@ static int flow_offload_hash_cmp(struct rhashtable_compare_arg *arg,
+ const struct flow_offload_tuple *tuple = arg->key;
+ const struct flow_offload_tuple_rhash *x = ptr;
+
+- if (memcmp(&x->tuple, tuple, offsetof(struct flow_offload_tuple, dir)))
++ if (memcmp(&x->tuple, tuple, offsetof(struct flow_offload_tuple, __hash)))
+ return 1;
+
+ return 0;
+@@ -211,30 +262,30 @@ static const struct rhashtable_params nf_flow_offload_rhash_params = {
+ .automatic_shrinking = true,
+ };
+
+-#define DAY (86400 * HZ)
+-
+-/* Set an arbitrary timeout large enough not to ever expire, this save
+- * us a check for the IPS_OFFLOAD_BIT from the packet path via
+- * nf_ct_is_expired().
+- */
+-static void nf_ct_offload_timeout(struct flow_offload *flow)
++unsigned long flow_offload_get_timeout(struct flow_offload *flow)
+ {
+- struct flow_offload_entry *entry;
+- struct nf_conn *ct;
++ unsigned long timeout = NF_FLOW_TIMEOUT;
++ struct net *net = nf_ct_net(flow->ct);
++ int l4num = nf_ct_protonum(flow->ct);
++
++ if (l4num == IPPROTO_TCP) {
++ struct nf_tcp_net *tn = nf_tcp_pernet(net);
+
+- entry = container_of(flow, struct flow_offload_entry, flow);
+- ct = entry->ct;
++ timeout = tn->offload_timeout;
++ } else if (l4num == IPPROTO_UDP) {
++ struct nf_udp_net *tn = nf_udp_pernet(net);
++
++ timeout = tn->offload_timeout;
++ }
+
+- if (nf_ct_expires(ct) < DAY / 2)
+- ct->timeout = nfct_time_stamp + DAY;
++ return timeout;
+ }
+
+ int flow_offload_add(struct nf_flowtable *flow_table, struct flow_offload *flow)
+ {
+ int err;
+
+- nf_ct_offload_timeout(flow);
+- flow->timeout = (u32)jiffies + NF_FLOW_TIMEOUT;
++ flow->timeout = nf_flowtable_time_stamp + flow_offload_get_timeout(flow);
+
+ err = rhashtable_insert_fast(&flow_table->rhashtable,
+ &flow->tuplehash[0].node,
+@@ -252,10 +303,35 @@ int flow_offload_add(struct nf_flowtable *flow_table, struct flow_offload *flow)
+ return err;
+ }
+
++ nf_ct_offload_timeout(flow->ct);
++
++ if (nf_flowtable_hw_offload(flow_table)) {
++ __set_bit(NF_FLOW_HW, &flow->flags);
++ nf_flow_offload_add(flow_table, flow);
++ }
++
+ return 0;
+ }
+ EXPORT_SYMBOL_GPL(flow_offload_add);
+
++void flow_offload_refresh(struct nf_flowtable *flow_table,
++ struct flow_offload *flow)
++{
++ u32 timeout;
++
++ timeout = nf_flowtable_time_stamp + flow_offload_get_timeout(flow);
++ if (timeout - READ_ONCE(flow->timeout) > HZ)
++ WRITE_ONCE(flow->timeout, timeout);
++ else
++ return;
++
++ if (likely(!nf_flowtable_hw_offload(flow_table)))
++ return;
++
++ nf_flow_offload_add(flow_table, flow);
++}
++EXPORT_SYMBOL_GPL(flow_offload_refresh);
++
+ static inline bool nf_flow_has_expired(const struct flow_offload *flow)
+ {
+ return nf_flow_timeout_delta(flow->timeout) <= 0;
+@@ -264,8 +340,6 @@ static inline bool nf_flow_has_expired(const struct flow_offload *flow)
+ static void flow_offload_del(struct nf_flowtable *flow_table,
+ struct flow_offload *flow)
+ {
+- struct flow_offload_entry *e;
+-
+ rhashtable_remove_fast(&flow_table->rhashtable,
+ &flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].node,
+ nf_flow_offload_rhash_params);
+@@ -273,28 +347,15 @@ static void flow_offload_del(struct nf_flowtable *flow_table,
+ &flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].node,
+ nf_flow_offload_rhash_params);
+
+- e = container_of(flow, struct flow_offload_entry, flow);
+- clear_bit(IPS_OFFLOAD_BIT, &e->ct->status);
+-
+- if (nf_flow_has_expired(flow))
+- flow_offload_fixup_ct(e->ct);
+- else if (flow->flags & FLOW_OFFLOAD_TEARDOWN)
+- flow_offload_fixup_ct_timeout(e->ct);
+-
+- if (!(flow->flags & FLOW_OFFLOAD_TEARDOWN))
+- flow_offload_fixup_ct_state(e->ct);
+-
+ flow_offload_free(flow);
+ }
+
+ void flow_offload_teardown(struct flow_offload *flow)
+ {
+- struct flow_offload_entry *e;
+-
+- flow->flags |= FLOW_OFFLOAD_TEARDOWN;
++ clear_bit(IPS_OFFLOAD_BIT, &flow->ct->status);
++ set_bit(NF_FLOW_TEARDOWN, &flow->flags);
+
+- e = container_of(flow, struct flow_offload_entry, flow);
+- flow_offload_fixup_ct_state(e->ct);
++ flow_offload_fixup_ct(flow->ct);
+ }
+ EXPORT_SYMBOL_GPL(flow_offload_teardown);
+
+@@ -304,7 +365,6 @@ flow_offload_lookup(struct nf_flowtable *flow_table,
+ {
+ struct flow_offload_tuple_rhash *tuplehash;
+ struct flow_offload *flow;
+- struct flow_offload_entry *e;
+ int dir;
+
+ tuplehash = rhashtable_lookup(&flow_table->rhashtable, tuple,
+@@ -314,19 +374,17 @@ flow_offload_lookup(struct nf_flowtable *flow_table,
+
+ dir = tuplehash->tuple.dir;
+ flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]);
+- if (flow->flags & (FLOW_OFFLOAD_DYING | FLOW_OFFLOAD_TEARDOWN))
++ if (test_bit(NF_FLOW_TEARDOWN, &flow->flags))
+ return NULL;
+
+- e = container_of(flow, struct flow_offload_entry, flow);
+- if (unlikely(nf_ct_is_dying(e->ct)))
++ if (unlikely(nf_ct_is_dying(flow->ct)))
+ return NULL;
+
+ return tuplehash;
+ }
+ EXPORT_SYMBOL_GPL(flow_offload_lookup);
+
+-static int
+-nf_flow_table_iterate(struct nf_flowtable *flow_table,
++int nf_flow_table_iterate(struct nf_flowtable *flow_table,
+ void (*iter)(struct flow_offload *flow, void *data),
+ void *data)
+ {
+@@ -339,7 +397,6 @@ nf_flow_table_iterate(struct nf_flowtable *flow_table,
+ rhashtable_walk_start(&hti);
+
+ while ((tuplehash = rhashtable_walk_next(&hti))) {
+-
+ if (IS_ERR(tuplehash)) {
+ if (PTR_ERR(tuplehash) != -EAGAIN) {
+ err = PTR_ERR(tuplehash);
+@@ -359,23 +416,52 @@ nf_flow_table_iterate(struct nf_flowtable *flow_table,
+
+ return err;
+ }
++EXPORT_SYMBOL_GPL(nf_flow_table_iterate);
+
+-static void nf_flow_offload_gc_step(struct flow_offload *flow, void *data)
++static bool flow_offload_stale_dst(struct flow_offload_tuple *tuple)
+ {
+- struct nf_flowtable *flow_table = data;
+- struct flow_offload_entry *e;
+- bool teardown;
++ struct dst_entry *dst;
+
+- e = container_of(flow, struct flow_offload_entry, flow);
++ if (tuple->xmit_type == FLOW_OFFLOAD_XMIT_NEIGH ||
++ tuple->xmit_type == FLOW_OFFLOAD_XMIT_XFRM) {
++ dst = tuple->dst_cache;
++ if (!dst_check(dst, tuple->dst_cookie))
++ return true;
++ }
+
+- teardown = flow->flags & (FLOW_OFFLOAD_DYING |
+- FLOW_OFFLOAD_TEARDOWN);
++ return false;
++}
+
+- if (!teardown)
+- nf_ct_offload_timeout(flow);
++static bool nf_flow_has_stale_dst(struct flow_offload *flow)
++{
++ return flow_offload_stale_dst(&flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple) ||
++ flow_offload_stale_dst(&flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple);
++}
+
+- if (nf_flow_has_expired(flow) || teardown)
+- flow_offload_del(flow_table, flow);
++static void nf_flow_offload_gc_step(struct flow_offload *flow, void *data)
++{
++ struct nf_flowtable *flow_table = data;
++
++ if (nf_flow_has_expired(flow) ||
++ nf_ct_is_dying(flow->ct) ||
++ nf_flow_has_stale_dst(flow))
++ flow_offload_teardown(flow);
++
++ if (test_bit(NF_FLOW_TEARDOWN, &flow->flags)) {
++ if (test_bit(NF_FLOW_HW, &flow->flags)) {
++ if (!test_and_set_bit(NF_FLOW_HW_ACCT_DYING, &flow->flags))
++ nf_flow_offload_stats(flow_table, flow, true);
++
++ if (!test_bit(NF_FLOW_HW_DYING, &flow->flags))
++ nf_flow_offload_del(flow_table, flow);
++ else if (test_bit(NF_FLOW_HW_DEAD, &flow->flags))
++ flow_offload_del(flow_table, flow);
++ } else {
++ flow_offload_del(flow_table, flow);
++ }
++ } else if (test_bit(NF_FLOW_HW, &flow->flags)) {
++ nf_flow_offload_stats(flow_table, flow, false);
++ }
+ }
+
+ static void nf_flow_offload_work_gc(struct work_struct *work)
+@@ -387,30 +473,20 @@ static void nf_flow_offload_work_gc(struct work_struct *work)
+ queue_delayed_work(system_power_efficient_wq, &flow_table->gc_work, HZ);
+ }
+
+-static int nf_flow_nat_port_tcp(struct sk_buff *skb, unsigned int thoff,
+- __be16 port, __be16 new_port)
++static void nf_flow_nat_port_tcp(struct sk_buff *skb, unsigned int thoff,
++ __be16 port, __be16 new_port)
+ {
+ struct tcphdr *tcph;
+
+- if (!pskb_may_pull(skb, thoff + sizeof(*tcph)) ||
+- skb_try_make_writable(skb, thoff + sizeof(*tcph)))
+- return -1;
+-
+ tcph = (void *)(skb_network_header(skb) + thoff);
+ inet_proto_csum_replace2(&tcph->check, skb, port, new_port, false);
+-
+- return 0;
+ }
+
+-static int nf_flow_nat_port_udp(struct sk_buff *skb, unsigned int thoff,
+- __be16 port, __be16 new_port)
++static void nf_flow_nat_port_udp(struct sk_buff *skb, unsigned int thoff,
++ __be16 port, __be16 new_port)
+ {
+ struct udphdr *udph;
+
+- if (!pskb_may_pull(skb, thoff + sizeof(*udph)) ||
+- skb_try_make_writable(skb, thoff + sizeof(*udph)))
+- return -1;
+-
+ udph = (void *)(skb_network_header(skb) + thoff);
+ if (udph->check || skb->ip_summed == CHECKSUM_PARTIAL) {
+ inet_proto_csum_replace2(&udph->check, skb, port,
+@@ -418,38 +494,28 @@ static int nf_flow_nat_port_udp(struct sk_buff *skb, unsigned int thoff,
+ if (!udph->check)
+ udph->check = CSUM_MANGLED_0;
+ }
+-
+- return 0;
+ }
+
+-static int nf_flow_nat_port(struct sk_buff *skb, unsigned int thoff,
+- u8 protocol, __be16 port, __be16 new_port)
++static void nf_flow_nat_port(struct sk_buff *skb, unsigned int thoff,
++ u8 protocol, __be16 port, __be16 new_port)
+ {
+ switch (protocol) {
+ case IPPROTO_TCP:
+- if (nf_flow_nat_port_tcp(skb, thoff, port, new_port) < 0)
+- return NF_DROP;
++ nf_flow_nat_port_tcp(skb, thoff, port, new_port);
+ break;
+ case IPPROTO_UDP:
+- if (nf_flow_nat_port_udp(skb, thoff, port, new_port) < 0)
+- return NF_DROP;
++ nf_flow_nat_port_udp(skb, thoff, port, new_port);
+ break;
+ }
+-
+- return 0;
+ }
+
+-int nf_flow_snat_port(const struct flow_offload *flow,
+- struct sk_buff *skb, unsigned int thoff,
+- u8 protocol, enum flow_offload_tuple_dir dir)
++void nf_flow_snat_port(const struct flow_offload *flow,
++ struct sk_buff *skb, unsigned int thoff,
++ u8 protocol, enum flow_offload_tuple_dir dir)
+ {
+ struct flow_ports *hdr;
+ __be16 port, new_port;
+
+- if (!pskb_may_pull(skb, thoff + sizeof(*hdr)) ||
+- skb_try_make_writable(skb, thoff + sizeof(*hdr)))
+- return -1;
+-
+ hdr = (void *)(skb_network_header(skb) + thoff);
+
+ switch (dir) {
+@@ -463,25 +529,19 @@ int nf_flow_snat_port(const struct flow_offload *flow,
+ new_port = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_port;
+ hdr->dest = new_port;
+ break;
+- default:
+- return -1;
+ }
+
+- return nf_flow_nat_port(skb, thoff, protocol, port, new_port);
++ nf_flow_nat_port(skb, thoff, protocol, port, new_port);
+ }
+ EXPORT_SYMBOL_GPL(nf_flow_snat_port);
+
+-int nf_flow_dnat_port(const struct flow_offload *flow,
+- struct sk_buff *skb, unsigned int thoff,
+- u8 protocol, enum flow_offload_tuple_dir dir)
++void nf_flow_dnat_port(const struct flow_offload *flow, struct sk_buff *skb,
++ unsigned int thoff, u8 protocol,
++ enum flow_offload_tuple_dir dir)
+ {
+ struct flow_ports *hdr;
+ __be16 port, new_port;
+
+- if (!pskb_may_pull(skb, thoff + sizeof(*hdr)) ||
+- skb_try_make_writable(skb, thoff + sizeof(*hdr)))
+- return -1;
+-
+ hdr = (void *)(skb_network_header(skb) + thoff);
+
+ switch (dir) {
+@@ -495,11 +555,9 @@ int nf_flow_dnat_port(const struct flow_offload *flow,
+ new_port = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_port;
+ hdr->source = new_port;
+ break;
+- default:
+- return -1;
+ }
+
+- return nf_flow_nat_port(skb, thoff, protocol, port, new_port);
++ nf_flow_nat_port(skb, thoff, protocol, port, new_port);
+ }
+ EXPORT_SYMBOL_GPL(nf_flow_dnat_port);
+
+@@ -507,7 +565,9 @@ int nf_flow_table_init(struct nf_flowtable *flowtable)
+ {
+ int err;
+
+- INIT_DEFERRABLE_WORK(&flowtable->gc_work, nf_flow_offload_work_gc);
++ INIT_DELAYED_WORK(&flowtable->gc_work, nf_flow_offload_work_gc);
++ flow_block_init(&flowtable->flow_block);
++ init_rwsem(&flowtable->flow_block_lock);
+
+ err = rhashtable_init(&flowtable->rhashtable,
+ &nf_flow_offload_rhash_params);
+@@ -528,25 +588,24 @@ EXPORT_SYMBOL_GPL(nf_flow_table_init);
+ static void nf_flow_table_do_cleanup(struct flow_offload *flow, void *data)
+ {
+ struct net_device *dev = data;
+- struct flow_offload_entry *e;
+-
+- e = container_of(flow, struct flow_offload_entry, flow);
+
+ if (!dev) {
+ flow_offload_teardown(flow);
+ return;
+ }
+- if (net_eq(nf_ct_net(e->ct), dev_net(dev)) &&
++
++ if (net_eq(nf_ct_net(flow->ct), dev_net(dev)) &&
+ (flow->tuplehash[0].tuple.iifidx == dev->ifindex ||
+ flow->tuplehash[1].tuple.iifidx == dev->ifindex))
+- flow_offload_dead(flow);
++ flow_offload_teardown(flow);
+ }
+
+-static void nf_flow_table_iterate_cleanup(struct nf_flowtable *flowtable,
+- struct net_device *dev)
++void nf_flow_table_gc_cleanup(struct nf_flowtable *flowtable,
++ struct net_device *dev)
+ {
+ nf_flow_table_iterate(flowtable, nf_flow_table_do_cleanup, dev);
+ flush_delayed_work(&flowtable->gc_work);
++ nf_flow_table_offload_flush(flowtable);
+ }
+
+ void nf_flow_table_cleanup(struct net_device *dev)
+@@ -555,7 +614,7 @@ void nf_flow_table_cleanup(struct net_device *dev)
+
+ mutex_lock(&flowtable_lock);
+ list_for_each_entry(flowtable, &flowtables, list)
+- nf_flow_table_iterate_cleanup(flowtable, dev);
++ nf_flow_table_gc_cleanup(flowtable, dev);
+ mutex_unlock(&flowtable_lock);
+ }
+ EXPORT_SYMBOL_GPL(nf_flow_table_cleanup);
+@@ -565,9 +624,14 @@ void nf_flow_table_free(struct nf_flowtable *flow_table)
+ mutex_lock(&flowtable_lock);
+ list_del(&flow_table->list);
+ mutex_unlock(&flowtable_lock);
++
+ cancel_delayed_work_sync(&flow_table->gc_work);
+ nf_flow_table_iterate(flow_table, nf_flow_table_do_cleanup, NULL);
+ nf_flow_table_iterate(flow_table, nf_flow_offload_gc_step, flow_table);
++ nf_flow_table_offload_flush(flow_table);
++ if (nf_flowtable_hw_offload(flow_table))
++ nf_flow_table_iterate(flow_table, nf_flow_offload_gc_step,
++ flow_table);
+ rhashtable_destroy(&flow_table->rhashtable);
+ }
+ EXPORT_SYMBOL_GPL(nf_flow_table_free);
+@@ -591,12 +655,23 @@ static struct notifier_block flow_offload_netdev_notifier = {
+
+ static int __init nf_flow_table_module_init(void)
+ {
+- return register_netdevice_notifier(&flow_offload_netdev_notifier);
++ int ret;
++
++ ret = nf_flow_table_offload_init();
++ if (ret)
++ return ret;
++
++ ret = register_netdevice_notifier(&flow_offload_netdev_notifier);
++ if (ret)
++ nf_flow_table_offload_exit();
++
++ return ret;
+ }
+
+ static void __exit nf_flow_table_module_exit(void)
+ {
+ unregister_netdevice_notifier(&flow_offload_netdev_notifier);
++ nf_flow_table_offload_exit();
+ }
+
+ module_init(nf_flow_table_module_init);
+@@ -604,3 +679,4 @@ module_exit(nf_flow_table_module_exit);
+
+ MODULE_LICENSE("GPL");
+ MODULE_AUTHOR("Pablo Neira Ayuso <pablo@netfilter.org>");
++MODULE_DESCRIPTION("Netfilter flow table module");
+diff --git a/net/netfilter/nf_flow_table_ip.c b/net/netfilter/nf_flow_table_ip.c
+index 397129b..6257d87 100644
+--- a/net/netfilter/nf_flow_table_ip.c
++++ b/net/netfilter/nf_flow_table_ip.c
+@@ -7,11 +7,13 @@
+ #include <linux/ip.h>
+ #include <linux/ipv6.h>
+ #include <linux/netdevice.h>
++#include <linux/if_ether.h>
+ #include <net/ip.h>
+ #include <net/ipv6.h>
+ #include <net/ip6_route.h>
+ #include <net/neighbour.h>
+ #include <net/netfilter/nf_flow_table.h>
++#include <net/netfilter/nf_conntrack_acct.h>
+ /* For layer 4 checksum field offset. */
+ #include <linux/tcp.h>
+ #include <linux/udp.h>
+@@ -24,9 +26,6 @@ static int nf_flow_state_check(struct flow_offload *flow, int proto,
+ if (proto != IPPROTO_TCP)
+ return 0;
+
+- if (!pskb_may_pull(skb, thoff + sizeof(*tcph)))
+- return -1;
+-
+ tcph = (void *)(skb_network_header(skb) + thoff);
+ if (unlikely(tcph->fin || tcph->rst)) {
+ flow_offload_teardown(flow);
+@@ -36,30 +35,20 @@ static int nf_flow_state_check(struct flow_offload *flow, int proto,
+ return 0;
+ }
+
+-static int nf_flow_nat_ip_tcp(struct sk_buff *skb, unsigned int thoff,
+- __be32 addr, __be32 new_addr)
++static void nf_flow_nat_ip_tcp(struct sk_buff *skb, unsigned int thoff,
++ __be32 addr, __be32 new_addr)
+ {
+ struct tcphdr *tcph;
+
+- if (!pskb_may_pull(skb, thoff + sizeof(*tcph)) ||
+- skb_try_make_writable(skb, thoff + sizeof(*tcph)))
+- return -1;
+-
+ tcph = (void *)(skb_network_header(skb) + thoff);
+ inet_proto_csum_replace4(&tcph->check, skb, addr, new_addr, true);
+-
+- return 0;
+ }
+
+-static int nf_flow_nat_ip_udp(struct sk_buff *skb, unsigned int thoff,
+- __be32 addr, __be32 new_addr)
++static void nf_flow_nat_ip_udp(struct sk_buff *skb, unsigned int thoff,
++ __be32 addr, __be32 new_addr)
+ {
+ struct udphdr *udph;
+
+- if (!pskb_may_pull(skb, thoff + sizeof(*udph)) ||
+- skb_try_make_writable(skb, thoff + sizeof(*udph)))
+- return -1;
+-
+ udph = (void *)(skb_network_header(skb) + thoff);
+ if (udph->check || skb->ip_summed == CHECKSUM_PARTIAL) {
+ inet_proto_csum_replace4(&udph->check, skb, addr,
+@@ -67,31 +56,25 @@ static int nf_flow_nat_ip_udp(struct sk_buff *skb, unsigned int thoff,
+ if (!udph->check)
+ udph->check = CSUM_MANGLED_0;
+ }
+-
+- return 0;
+ }
+
+-static int nf_flow_nat_ip_l4proto(struct sk_buff *skb, struct iphdr *iph,
+- unsigned int thoff, __be32 addr,
+- __be32 new_addr)
++static void nf_flow_nat_ip_l4proto(struct sk_buff *skb, struct iphdr *iph,
++ unsigned int thoff, __be32 addr,
++ __be32 new_addr)
+ {
+ switch (iph->protocol) {
+ case IPPROTO_TCP:
+- if (nf_flow_nat_ip_tcp(skb, thoff, addr, new_addr) < 0)
+- return NF_DROP;
++ nf_flow_nat_ip_tcp(skb, thoff, addr, new_addr);
+ break;
+ case IPPROTO_UDP:
+- if (nf_flow_nat_ip_udp(skb, thoff, addr, new_addr) < 0)
+- return NF_DROP;
++ nf_flow_nat_ip_udp(skb, thoff, addr, new_addr);
+ break;
+ }
+-
+- return 0;
+ }
+
+-static int nf_flow_snat_ip(const struct flow_offload *flow, struct sk_buff *skb,
+- struct iphdr *iph, unsigned int thoff,
+- enum flow_offload_tuple_dir dir)
++static void nf_flow_snat_ip(const struct flow_offload *flow,
++ struct sk_buff *skb, struct iphdr *iph,
++ unsigned int thoff, enum flow_offload_tuple_dir dir)
+ {
+ __be32 addr, new_addr;
+
+@@ -106,17 +89,15 @@ static int nf_flow_snat_ip(const struct flow_offload *flow, struct sk_buff *skb,
+ new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v4.s_addr;
+ iph->daddr = new_addr;
+ break;
+- default:
+- return -1;
+ }
+ csum_replace4(&iph->check, addr, new_addr);
+
+- return nf_flow_nat_ip_l4proto(skb, iph, thoff, addr, new_addr);
++ nf_flow_nat_ip_l4proto(skb, iph, thoff, addr, new_addr);
+ }
+
+-static int nf_flow_dnat_ip(const struct flow_offload *flow, struct sk_buff *skb,
+- struct iphdr *iph, unsigned int thoff,
+- enum flow_offload_tuple_dir dir)
++static void nf_flow_dnat_ip(const struct flow_offload *flow,
++ struct sk_buff *skb, struct iphdr *iph,
++ unsigned int thoff, enum flow_offload_tuple_dir dir)
+ {
+ __be32 addr, new_addr;
+
+@@ -131,29 +112,24 @@ static int nf_flow_dnat_ip(const struct flow_offload *flow, struct sk_buff *skb,
+ new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v4.s_addr;
+ iph->saddr = new_addr;
+ break;
+- default:
+- return -1;
+ }
+ csum_replace4(&iph->check, addr, new_addr);
+
+- return nf_flow_nat_ip_l4proto(skb, iph, thoff, addr, new_addr);
++ nf_flow_nat_ip_l4proto(skb, iph, thoff, addr, new_addr);
+ }
+
+-static int nf_flow_nat_ip(const struct flow_offload *flow, struct sk_buff *skb,
+- unsigned int thoff, enum flow_offload_tuple_dir dir)
++static void nf_flow_nat_ip(const struct flow_offload *flow, struct sk_buff *skb,
++ unsigned int thoff, enum flow_offload_tuple_dir dir,
++ struct iphdr *iph)
+ {
+- struct iphdr *iph = ip_hdr(skb);
+-
+- if (flow->flags & FLOW_OFFLOAD_SNAT &&
+- (nf_flow_snat_port(flow, skb, thoff, iph->protocol, dir) < 0 ||
+- nf_flow_snat_ip(flow, skb, iph, thoff, dir) < 0))
+- return -1;
+- if (flow->flags & FLOW_OFFLOAD_DNAT &&
+- (nf_flow_dnat_port(flow, skb, thoff, iph->protocol, dir) < 0 ||
+- nf_flow_dnat_ip(flow, skb, iph, thoff, dir) < 0))
+- return -1;
+-
+- return 0;
++ if (test_bit(NF_FLOW_SNAT, &flow->flags)) {
++ nf_flow_snat_port(flow, skb, thoff, iph->protocol, dir);
++ nf_flow_snat_ip(flow, skb, iph, thoff, dir);
++ }
++ if (test_bit(NF_FLOW_DNAT, &flow->flags)) {
++ nf_flow_dnat_port(flow, skb, thoff, iph->protocol, dir);
++ nf_flow_dnat_ip(flow, skb, iph, thoff, dir);
++ }
+ }
+
+ static bool ip_has_options(unsigned int thoff)
+@@ -161,35 +137,70 @@ static bool ip_has_options(unsigned int thoff)
+ return thoff != sizeof(struct iphdr);
+ }
+
++static void nf_flow_tuple_encap(struct sk_buff *skb,
++ struct flow_offload_tuple *tuple)
++{
++ struct vlan_ethhdr *veth;
++ struct pppoe_hdr *phdr;
++ int i = 0;
++
++ if (skb_vlan_tag_present(skb)) {
++ tuple->encap[i].id = skb_vlan_tag_get(skb);
++ tuple->encap[i].proto = skb->vlan_proto;
++ i++;
++ }
++ switch (skb->protocol) {
++ case htons(ETH_P_8021Q):
++ veth = (struct vlan_ethhdr *)skb_mac_header(skb);
++ tuple->encap[i].id = ntohs(veth->h_vlan_TCI);
++ tuple->encap[i].proto = skb->protocol;
++ break;
++ case htons(ETH_P_PPP_SES):
++ phdr = (struct pppoe_hdr *)skb_mac_header(skb);
++ tuple->encap[i].id = ntohs(phdr->sid);
++ tuple->encap[i].proto = skb->protocol;
++ break;
++ }
++}
++
+ static int nf_flow_tuple_ip(struct sk_buff *skb, const struct net_device *dev,
+- struct flow_offload_tuple *tuple)
++ struct flow_offload_tuple *tuple, u32 *hdrsize,
++ u32 offset)
+ {
+ struct flow_ports *ports;
+ unsigned int thoff;
+ struct iphdr *iph;
+
+- if (!pskb_may_pull(skb, sizeof(*iph)))
++ if (!pskb_may_pull(skb, sizeof(*iph) + offset))
+ return -1;
+
+- iph = ip_hdr(skb);
+- thoff = iph->ihl * 4;
++ iph = (struct iphdr *)(skb_network_header(skb) + offset);
++ thoff = (iph->ihl * 4);
+
+ if (ip_is_fragment(iph) ||
+ unlikely(ip_has_options(thoff)))
+ return -1;
+
+- if (iph->protocol != IPPROTO_TCP &&
+- iph->protocol != IPPROTO_UDP)
++ thoff += offset;
++
++ switch (iph->protocol) {
++ case IPPROTO_TCP:
++ *hdrsize = sizeof(struct tcphdr);
++ break;
++ case IPPROTO_UDP:
++ *hdrsize = sizeof(struct udphdr);
++ break;
++ default:
+ return -1;
++ }
+
+ if (iph->ttl <= 1)
+ return -1;
+
+- thoff = iph->ihl * 4;
+- if (!pskb_may_pull(skb, thoff + sizeof(*ports)))
++ if (!pskb_may_pull(skb, thoff + *hdrsize))
+ return -1;
+
+- iph = ip_hdr(skb);
++ iph = (struct iphdr *)(skb_network_header(skb) + offset);
+ ports = (struct flow_ports *)(skb_network_header(skb) + thoff);
+
+ tuple->src_v4.s_addr = iph->saddr;
+@@ -199,6 +210,7 @@ static int nf_flow_tuple_ip(struct sk_buff *skb, const struct net_device *dev,
+ tuple->l3proto = AF_INET;
+ tuple->l4proto = iph->protocol;
+ tuple->iifidx = dev->ifindex;
++ nf_flow_tuple_encap(skb, tuple);
+
+ return 0;
+ }
+@@ -225,6 +237,75 @@ static unsigned int nf_flow_xmit_xfrm(struct sk_buff *skb,
+ return NF_STOLEN;
+ }
+
++static bool nf_flow_skb_encap_protocol(const struct sk_buff *skb, __be16 proto,
++ u32 *offset)
++{
++ struct vlan_ethhdr *veth;
++
++ switch (skb->protocol) {
++ case htons(ETH_P_8021Q):
++ veth = (struct vlan_ethhdr *)skb_mac_header(skb);
++ if (veth->h_vlan_encapsulated_proto == proto) {
++ *offset += VLAN_HLEN;
++ return true;
++ }
++ break;
++ case htons(ETH_P_PPP_SES):
++ if (nf_flow_pppoe_proto(skb) == proto) {
++ *offset += PPPOE_SES_HLEN;
++ return true;
++ }
++ break;
++ }
++
++ return false;
++}
++
++static void nf_flow_encap_pop(struct sk_buff *skb,
++ struct flow_offload_tuple_rhash *tuplehash)
++{
++ struct vlan_hdr *vlan_hdr;
++ int i;
++
++ for (i = 0; i < tuplehash->tuple.encap_num; i++) {
++ if (skb_vlan_tag_present(skb)) {
++ __vlan_hwaccel_clear_tag(skb);
++ continue;
++ }
++ switch (skb->protocol) {
++ case htons(ETH_P_8021Q):
++ vlan_hdr = (struct vlan_hdr *)skb->data;
++ __skb_pull(skb, VLAN_HLEN);
++ vlan_set_encap_proto(skb, vlan_hdr);
++ skb_reset_network_header(skb);
++ break;
++ case htons(ETH_P_PPP_SES):
++ skb->protocol = nf_flow_pppoe_proto(skb);
++ skb_pull(skb, PPPOE_SES_HLEN);
++ skb_reset_network_header(skb);
++ break;
++ }
++ }
++}
++
++static unsigned int nf_flow_queue_xmit(struct net *net, struct sk_buff *skb,
++ const struct flow_offload_tuple_rhash *tuplehash,
++ unsigned short type)
++{
++ struct net_device *outdev;
++
++ outdev = dev_get_by_index_rcu(net, tuplehash->tuple.out.ifidx);
++ if (!outdev)
++ return NF_DROP;
++
++ skb->dev = outdev;
++ dev_hard_header(skb, skb->dev, type, tuplehash->tuple.out.h_dest,
++ tuplehash->tuple.out.h_source, skb->len);
++ dev_queue_xmit(skb);
++
++ return NF_STOLEN;
++}
++
+ unsigned int
+ nf_flow_offload_ip_hook(void *priv, struct sk_buff *skb,
+ const struct nf_hook_state *state)
+@@ -235,15 +316,18 @@ nf_flow_offload_ip_hook(void *priv, struct sk_buff *skb,
+ enum flow_offload_tuple_dir dir;
+ struct flow_offload *flow;
+ struct net_device *outdev;
++ u32 hdrsize, offset = 0;
++ unsigned int thoff, mtu;
+ struct rtable *rt;
+- unsigned int thoff;
+ struct iphdr *iph;
+ __be32 nexthop;
++ int ret;
+
+- if (skb->protocol != htons(ETH_P_IP))
++ if (skb->protocol != htons(ETH_P_IP) &&
++ !nf_flow_skb_encap_protocol(skb, htons(ETH_P_IP), &offset))
+ return NF_ACCEPT;
+
+- if (nf_flow_tuple_ip(skb, state->in, &tuple) < 0)
++ if (nf_flow_tuple_ip(skb, state->in, &tuple, &hdrsize, offset) < 0)
+ return NF_ACCEPT;
+
+ tuplehash = flow_offload_lookup(flow_table, &tuple);
+@@ -252,75 +336,80 @@ nf_flow_offload_ip_hook(void *priv, struct sk_buff *skb,
+
+ dir = tuplehash->tuple.dir;
+ flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]);
+- rt = (struct rtable *)flow->tuplehash[dir].tuple.dst_cache;
+- outdev = rt->dst.dev;
+-
+- if (unlikely(nf_flow_exceeds_mtu(skb, flow->tuplehash[dir].tuple.mtu)))
+- return NF_ACCEPT;
+
+- if (skb_try_make_writable(skb, sizeof(*iph)))
+- return NF_DROP;
+-
+- thoff = ip_hdr(skb)->ihl * 4;
+- if (nf_flow_state_check(flow, ip_hdr(skb)->protocol, skb, thoff))
++ mtu = flow->tuplehash[dir].tuple.mtu + offset;
++ if (unlikely(nf_flow_exceeds_mtu(skb, mtu)))
+ return NF_ACCEPT;
+
+- if (!dst_check(&rt->dst, 0)) {
+- flow_offload_teardown(flow);
++ iph = (struct iphdr *)(skb_network_header(skb) + offset);
++ thoff = (iph->ihl * 4) + offset;
++ if (nf_flow_state_check(flow, iph->protocol, skb, thoff))
+ return NF_ACCEPT;
+- }
+
+- if (nf_flow_nat_ip(flow, skb, thoff, dir) < 0)
++ if (skb_try_make_writable(skb, thoff + hdrsize))
+ return NF_DROP;
+
+- flow->timeout = (u32)jiffies + NF_FLOW_TIMEOUT;
++ flow_offload_refresh(flow_table, flow);
++
++ nf_flow_encap_pop(skb, tuplehash);
++ thoff -= offset;
++
+ iph = ip_hdr(skb);
++ nf_flow_nat_ip(flow, skb, thoff, dir, iph);
++
+ ip_decrease_ttl(iph);
+ skb->tstamp = 0;
+
+- if (unlikely(dst_xfrm(&rt->dst))) {
++ if (flow_table->flags & NF_FLOWTABLE_COUNTER)
++ nf_ct_acct_update(flow->ct, tuplehash->tuple.dir, skb->len);
++
++ if (unlikely(tuplehash->tuple.xmit_type == FLOW_OFFLOAD_XMIT_XFRM)) {
++ rt = (struct rtable *)tuplehash->tuple.dst_cache;
+ memset(skb->cb, 0, sizeof(struct inet_skb_parm));
+ IPCB(skb)->iif = skb->dev->ifindex;
+ IPCB(skb)->flags = IPSKB_FORWARDED;
+ return nf_flow_xmit_xfrm(skb, state, &rt->dst);
+ }
+
+- skb->dev = outdev;
+- nexthop = rt_nexthop(rt, flow->tuplehash[!dir].tuple.src_v4.s_addr);
+- skb_dst_set_noref(skb, &rt->dst);
+- neigh_xmit(NEIGH_ARP_TABLE, outdev, &nexthop, skb);
++ switch (tuplehash->tuple.xmit_type) {
++ case FLOW_OFFLOAD_XMIT_NEIGH:
++ rt = (struct rtable *)tuplehash->tuple.dst_cache;
++ outdev = rt->dst.dev;
++ skb->dev = outdev;
++ nexthop = rt_nexthop(rt, flow->tuplehash[!dir].tuple.src_v4.s_addr);
++ skb_dst_set_noref(skb, &rt->dst);
++ neigh_xmit(NEIGH_ARP_TABLE, outdev, &nexthop, skb);
++ ret = NF_STOLEN;
++ break;
++ case FLOW_OFFLOAD_XMIT_DIRECT:
++ ret = nf_flow_queue_xmit(state->net, skb, tuplehash, ETH_P_IP);
++ if (ret == NF_DROP)
++ flow_offload_teardown(flow);
++ break;
++ }
+
+- return NF_STOLEN;
++ return ret;
+ }
+ EXPORT_SYMBOL_GPL(nf_flow_offload_ip_hook);
+
+-static int nf_flow_nat_ipv6_tcp(struct sk_buff *skb, unsigned int thoff,
+- struct in6_addr *addr,
+- struct in6_addr *new_addr)
++static void nf_flow_nat_ipv6_tcp(struct sk_buff *skb, unsigned int thoff,
++ struct in6_addr *addr,
++ struct in6_addr *new_addr,
++ struct ipv6hdr *ip6h)
+ {
+ struct tcphdr *tcph;
+
+- if (!pskb_may_pull(skb, thoff + sizeof(*tcph)) ||
+- skb_try_make_writable(skb, thoff + sizeof(*tcph)))
+- return -1;
+-
+ tcph = (void *)(skb_network_header(skb) + thoff);
+ inet_proto_csum_replace16(&tcph->check, skb, addr->s6_addr32,
+ new_addr->s6_addr32, true);
+-
+- return 0;
+ }
+
+-static int nf_flow_nat_ipv6_udp(struct sk_buff *skb, unsigned int thoff,
+- struct in6_addr *addr,
+- struct in6_addr *new_addr)
++static void nf_flow_nat_ipv6_udp(struct sk_buff *skb, unsigned int thoff,
++ struct in6_addr *addr,
++ struct in6_addr *new_addr)
+ {
+ struct udphdr *udph;
+
+- if (!pskb_may_pull(skb, thoff + sizeof(*udph)) ||
+- skb_try_make_writable(skb, thoff + sizeof(*udph)))
+- return -1;
+-
+ udph = (void *)(skb_network_header(skb) + thoff);
+ if (udph->check || skb->ip_summed == CHECKSUM_PARTIAL) {
+ inet_proto_csum_replace16(&udph->check, skb, addr->s6_addr32,
+@@ -328,32 +417,26 @@ static int nf_flow_nat_ipv6_udp(struct sk_buff *skb, unsigned int thoff,
+ if (!udph->check)
+ udph->check = CSUM_MANGLED_0;
+ }
+-
+- return 0;
+ }
+
+-static int nf_flow_nat_ipv6_l4proto(struct sk_buff *skb, struct ipv6hdr *ip6h,
+- unsigned int thoff, struct in6_addr *addr,
+- struct in6_addr *new_addr)
++static void nf_flow_nat_ipv6_l4proto(struct sk_buff *skb, struct ipv6hdr *ip6h,
++ unsigned int thoff, struct in6_addr *addr,
++ struct in6_addr *new_addr)
+ {
+ switch (ip6h->nexthdr) {
+ case IPPROTO_TCP:
+- if (nf_flow_nat_ipv6_tcp(skb, thoff, addr, new_addr) < 0)
+- return NF_DROP;
++ nf_flow_nat_ipv6_tcp(skb, thoff, addr, new_addr, ip6h);
+ break;
+ case IPPROTO_UDP:
+- if (nf_flow_nat_ipv6_udp(skb, thoff, addr, new_addr) < 0)
+- return NF_DROP;
++ nf_flow_nat_ipv6_udp(skb, thoff, addr, new_addr);
+ break;
+ }
+-
+- return 0;
+ }
+
+-static int nf_flow_snat_ipv6(const struct flow_offload *flow,
+- struct sk_buff *skb, struct ipv6hdr *ip6h,
+- unsigned int thoff,
+- enum flow_offload_tuple_dir dir)
++static void nf_flow_snat_ipv6(const struct flow_offload *flow,
++ struct sk_buff *skb, struct ipv6hdr *ip6h,
++ unsigned int thoff,
++ enum flow_offload_tuple_dir dir)
+ {
+ struct in6_addr addr, new_addr;
+
+@@ -368,17 +451,15 @@ static int nf_flow_snat_ipv6(const struct flow_offload *flow,
+ new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v6;
+ ip6h->daddr = new_addr;
+ break;
+- default:
+- return -1;
+ }
+
+- return nf_flow_nat_ipv6_l4proto(skb, ip6h, thoff, &addr, &new_addr);
++ nf_flow_nat_ipv6_l4proto(skb, ip6h, thoff, &addr, &new_addr);
+ }
+
+-static int nf_flow_dnat_ipv6(const struct flow_offload *flow,
+- struct sk_buff *skb, struct ipv6hdr *ip6h,
+- unsigned int thoff,
+- enum flow_offload_tuple_dir dir)
++static void nf_flow_dnat_ipv6(const struct flow_offload *flow,
++ struct sk_buff *skb, struct ipv6hdr *ip6h,
++ unsigned int thoff,
++ enum flow_offload_tuple_dir dir)
+ {
+ struct in6_addr addr, new_addr;
+
+@@ -393,56 +474,60 @@ static int nf_flow_dnat_ipv6(const struct flow_offload *flow,
+ new_addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v6;
+ ip6h->saddr = new_addr;
+ break;
+- default:
+- return -1;
+ }
+
+- return nf_flow_nat_ipv6_l4proto(skb, ip6h, thoff, &addr, &new_addr);
++ nf_flow_nat_ipv6_l4proto(skb, ip6h, thoff, &addr, &new_addr);
+ }
+
+-static int nf_flow_nat_ipv6(const struct flow_offload *flow,
+- struct sk_buff *skb,
+- enum flow_offload_tuple_dir dir)
++static void nf_flow_nat_ipv6(const struct flow_offload *flow,
++ struct sk_buff *skb,
++ enum flow_offload_tuple_dir dir,
++ struct ipv6hdr *ip6h)
+ {
+- struct ipv6hdr *ip6h = ipv6_hdr(skb);
+ unsigned int thoff = sizeof(*ip6h);
+
+- if (flow->flags & FLOW_OFFLOAD_SNAT &&
+- (nf_flow_snat_port(flow, skb, thoff, ip6h->nexthdr, dir) < 0 ||
+- nf_flow_snat_ipv6(flow, skb, ip6h, thoff, dir) < 0))
+- return -1;
+- if (flow->flags & FLOW_OFFLOAD_DNAT &&
+- (nf_flow_dnat_port(flow, skb, thoff, ip6h->nexthdr, dir) < 0 ||
+- nf_flow_dnat_ipv6(flow, skb, ip6h, thoff, dir) < 0))
+- return -1;
+-
+- return 0;
++ if (test_bit(NF_FLOW_SNAT, &flow->flags)) {
++ nf_flow_snat_port(flow, skb, thoff, ip6h->nexthdr, dir);
++ nf_flow_snat_ipv6(flow, skb, ip6h, thoff, dir);
++ }
++ if (test_bit(NF_FLOW_DNAT, &flow->flags)) {
++ nf_flow_dnat_port(flow, skb, thoff, ip6h->nexthdr, dir);
++ nf_flow_dnat_ipv6(flow, skb, ip6h, thoff, dir);
++ }
+ }
+
+ static int nf_flow_tuple_ipv6(struct sk_buff *skb, const struct net_device *dev,
+- struct flow_offload_tuple *tuple)
++ struct flow_offload_tuple *tuple, u32 *hdrsize,
++ u32 offset)
+ {
+ struct flow_ports *ports;
+ struct ipv6hdr *ip6h;
+ unsigned int thoff;
+
+- if (!pskb_may_pull(skb, sizeof(*ip6h)))
++ thoff = sizeof(*ip6h) + offset;
++ if (!pskb_may_pull(skb, thoff))
+ return -1;
+
+- ip6h = ipv6_hdr(skb);
++ ip6h = (struct ipv6hdr *)(skb_network_header(skb) + offset);
+
+- if (ip6h->nexthdr != IPPROTO_TCP &&
+- ip6h->nexthdr != IPPROTO_UDP)
++ switch (ip6h->nexthdr) {
++ case IPPROTO_TCP:
++ *hdrsize = sizeof(struct tcphdr);
++ break;
++ case IPPROTO_UDP:
++ *hdrsize = sizeof(struct udphdr);
++ break;
++ default:
+ return -1;
++ }
+
+ if (ip6h->hop_limit <= 1)
+ return -1;
+
+- thoff = sizeof(*ip6h);
+- if (!pskb_may_pull(skb, thoff + sizeof(*ports)))
++ if (!pskb_may_pull(skb, thoff + *hdrsize))
+ return -1;
+
+- ip6h = ipv6_hdr(skb);
++ ip6h = (struct ipv6hdr *)(skb_network_header(skb) + offset);
+ ports = (struct flow_ports *)(skb_network_header(skb) + thoff);
+
+ tuple->src_v6 = ip6h->saddr;
+@@ -452,6 +537,7 @@ static int nf_flow_tuple_ipv6(struct sk_buff *skb, const struct net_device *dev,
+ tuple->l3proto = AF_INET6;
+ tuple->l4proto = ip6h->nexthdr;
+ tuple->iifidx = dev->ifindex;
++ nf_flow_tuple_encap(skb, tuple);
+
+ return 0;
+ }
+@@ -467,13 +553,17 @@ nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb,
+ const struct in6_addr *nexthop;
+ struct flow_offload *flow;
+ struct net_device *outdev;
++ unsigned int thoff, mtu;
++ u32 hdrsize, offset = 0;
+ struct ipv6hdr *ip6h;
+ struct rt6_info *rt;
++ int ret;
+
+- if (skb->protocol != htons(ETH_P_IPV6))
++ if (skb->protocol != htons(ETH_P_IPV6) &&
++ !nf_flow_skb_encap_protocol(skb, htons(ETH_P_IPV6), &offset))
+ return NF_ACCEPT;
+
+- if (nf_flow_tuple_ipv6(skb, state->in, &tuple) < 0)
++ if (nf_flow_tuple_ipv6(skb, state->in, &tuple, &hdrsize, offset) < 0)
+ return NF_ACCEPT;
+
+ tuplehash = flow_offload_lookup(flow_table, &tuple);
+@@ -482,44 +572,57 @@ nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb,
+
+ dir = tuplehash->tuple.dir;
+ flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]);
+- rt = (struct rt6_info *)flow->tuplehash[dir].tuple.dst_cache;
+- outdev = rt->dst.dev;
+
+- if (unlikely(nf_flow_exceeds_mtu(skb, flow->tuplehash[dir].tuple.mtu)))
++ mtu = flow->tuplehash[dir].tuple.mtu + offset;
++ if (unlikely(nf_flow_exceeds_mtu(skb, mtu)))
+ return NF_ACCEPT;
+
+- if (nf_flow_state_check(flow, ipv6_hdr(skb)->nexthdr, skb,
+- sizeof(*ip6h)))
++ ip6h = (struct ipv6hdr *)(skb_network_header(skb) + offset);
++ thoff = sizeof(*ip6h) + offset;
++ if (nf_flow_state_check(flow, ip6h->nexthdr, skb, thoff))
+ return NF_ACCEPT;
+
+- if (!dst_check(&rt->dst, tuplehash->tuple.dst_cookie)) {
+- flow_offload_teardown(flow);
+- return NF_ACCEPT;
+- }
+-
+- if (skb_try_make_writable(skb, sizeof(*ip6h)))
++ if (skb_try_make_writable(skb, thoff + hdrsize))
+ return NF_DROP;
+
+- if (nf_flow_nat_ipv6(flow, skb, dir) < 0)
+- return NF_DROP;
++ flow_offload_refresh(flow_table, flow);
++
++ nf_flow_encap_pop(skb, tuplehash);
+
+- flow->timeout = (u32)jiffies + NF_FLOW_TIMEOUT;
+ ip6h = ipv6_hdr(skb);
++ nf_flow_nat_ipv6(flow, skb, dir, ip6h);
++
+ ip6h->hop_limit--;
+ skb->tstamp = 0;
+
+- if (unlikely(dst_xfrm(&rt->dst))) {
++ if (flow_table->flags & NF_FLOWTABLE_COUNTER)
++ nf_ct_acct_update(flow->ct, tuplehash->tuple.dir, skb->len);
++
++ if (unlikely(tuplehash->tuple.xmit_type == FLOW_OFFLOAD_XMIT_XFRM)) {
++ rt = (struct rt6_info *)tuplehash->tuple.dst_cache;
+ memset(skb->cb, 0, sizeof(struct inet6_skb_parm));
+ IP6CB(skb)->iif = skb->dev->ifindex;
+ IP6CB(skb)->flags = IP6SKB_FORWARDED;
+ return nf_flow_xmit_xfrm(skb, state, &rt->dst);
+ }
+
+- skb->dev = outdev;
+- nexthop = rt6_nexthop(rt, &flow->tuplehash[!dir].tuple.src_v6);
+- skb_dst_set_noref(skb, &rt->dst);
+- neigh_xmit(NEIGH_ND_TABLE, outdev, nexthop, skb);
++ switch (tuplehash->tuple.xmit_type) {
++ case FLOW_OFFLOAD_XMIT_NEIGH:
++ rt = (struct rt6_info *)tuplehash->tuple.dst_cache;
++ outdev = rt->dst.dev;
++ skb->dev = outdev;
++ nexthop = rt6_nexthop(rt, &flow->tuplehash[!dir].tuple.src_v6);
++ skb_dst_set_noref(skb, &rt->dst);
++ neigh_xmit(NEIGH_ND_TABLE, outdev, nexthop, skb);
++ ret = NF_STOLEN;
++ break;
++ case FLOW_OFFLOAD_XMIT_DIRECT:
++ ret = nf_flow_queue_xmit(state->net, skb, tuplehash, ETH_P_IPV6);
++ if (ret == NF_DROP)
++ flow_offload_teardown(flow);
++ break;
++ }
+
+- return NF_STOLEN;
++ return ret;
+ }
+ EXPORT_SYMBOL_GPL(nf_flow_offload_ipv6_hook);
+diff --git a/net/netfilter/nf_flow_table_offload.c b/net/netfilter/nf_flow_table_offload.c
+new file mode 100644
+index 0000000..50f2f2e
+--- /dev/null
++++ b/net/netfilter/nf_flow_table_offload.c
+@@ -0,0 +1,1199 @@
++#include <linux/kernel.h>
++#include <linux/init.h>
++#include <linux/module.h>
++#include <linux/netfilter.h>
++#include <linux/rhashtable.h>
++#include <linux/netdevice.h>
++#include <linux/tc_act/tc_csum.h>
++#include <net/flow_offload.h>
++#include <net/netfilter/nf_flow_table.h>
++#include <net/netfilter/nf_tables.h>
++#include <net/netfilter/nf_conntrack.h>
++#include <net/netfilter/nf_conntrack_acct.h>
++#include <net/netfilter/nf_conntrack_core.h>
++#include <net/netfilter/nf_conntrack_tuple.h>
++
++static struct workqueue_struct *nf_flow_offload_add_wq;
++static struct workqueue_struct *nf_flow_offload_del_wq;
++static struct workqueue_struct *nf_flow_offload_stats_wq;
++
++struct flow_offload_work {
++ struct list_head list;
++ enum flow_cls_command cmd;
++ int priority;
++ struct nf_flowtable *flowtable;
++ struct flow_offload *flow;
++ struct work_struct work;
++};
++
++#define NF_FLOW_DISSECTOR(__match, __type, __field) \
++ (__match)->dissector.offset[__type] = \
++ offsetof(struct nf_flow_key, __field)
++
++static void nf_flow_rule_lwt_match(struct nf_flow_match *match,
++ struct ip_tunnel_info *tun_info)
++{
++ struct nf_flow_key *mask = &match->mask;
++ struct nf_flow_key *key = &match->key;
++ unsigned int enc_keys;
++
++ if (!tun_info || !(tun_info->mode & IP_TUNNEL_INFO_TX))
++ return;
++
++ NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_ENC_CONTROL, enc_control);
++ NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_ENC_KEYID, enc_key_id);
++ key->enc_key_id.keyid = tunnel_id_to_key32(tun_info->key.tun_id);
++ mask->enc_key_id.keyid = 0xffffffff;
++ enc_keys = BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) |
++ BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL);
++
++ if (ip_tunnel_info_af(tun_info) == AF_INET) {
++ NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS,
++ enc_ipv4);
++ key->enc_ipv4.src = tun_info->key.u.ipv4.dst;
++ key->enc_ipv4.dst = tun_info->key.u.ipv4.src;
++ if (key->enc_ipv4.src)
++ mask->enc_ipv4.src = 0xffffffff;
++ if (key->enc_ipv4.dst)
++ mask->enc_ipv4.dst = 0xffffffff;
++ enc_keys |= BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS);
++ key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
++ } else {
++ memcpy(&key->enc_ipv6.src, &tun_info->key.u.ipv6.dst,
++ sizeof(struct in6_addr));
++ memcpy(&key->enc_ipv6.dst, &tun_info->key.u.ipv6.src,
++ sizeof(struct in6_addr));
++ if (memcmp(&key->enc_ipv6.src, &in6addr_any,
++ sizeof(struct in6_addr)))
++ memset(&mask->enc_ipv6.src, 0xff,
++ sizeof(struct in6_addr));
++ if (memcmp(&key->enc_ipv6.dst, &in6addr_any,
++ sizeof(struct in6_addr)))
++ memset(&mask->enc_ipv6.dst, 0xff,
++ sizeof(struct in6_addr));
++ enc_keys |= BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS);
++ key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
++ }
++
++ match->dissector.used_keys |= enc_keys;
++}
++
++static void nf_flow_rule_vlan_match(struct flow_dissector_key_vlan *key,
++ struct flow_dissector_key_vlan *mask,
++ u16 vlan_id, __be16 proto)
++{
++ key->vlan_id = vlan_id;
++ mask->vlan_id = VLAN_VID_MASK;
++ key->vlan_tpid = proto;
++ mask->vlan_tpid = 0xffff;
++}
++
++static int nf_flow_rule_match(struct nf_flow_match *match,
++ const struct flow_offload_tuple *tuple,
++ struct dst_entry *other_dst)
++{
++ struct nf_flow_key *mask = &match->mask;
++ struct nf_flow_key *key = &match->key;
++ struct ip_tunnel_info *tun_info;
++ bool vlan_encap = false;
++
++ NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_META, meta);
++ NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_CONTROL, control);
++ NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_BASIC, basic);
++ NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4);
++ NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6);
++ NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_TCP, tcp);
++ NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_PORTS, tp);
++
++ if (other_dst && other_dst->lwtstate) {
++ tun_info = lwt_tun_info(other_dst->lwtstate);
++ nf_flow_rule_lwt_match(match, tun_info);
++ }
++
++ key->meta.ingress_ifindex = tuple->iifidx;
++ mask->meta.ingress_ifindex = 0xffffffff;
++
++ if (tuple->encap_num > 0 && !(tuple->in_vlan_ingress & BIT(0)) &&
++ tuple->encap[0].proto == htons(ETH_P_8021Q)) {
++ NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_VLAN, vlan);
++ nf_flow_rule_vlan_match(&key->vlan, &mask->vlan,
++ tuple->encap[0].id,
++ tuple->encap[0].proto);
++ vlan_encap = true;
++ }
++
++ if (tuple->encap_num > 1 && !(tuple->in_vlan_ingress & BIT(1)) &&
++ tuple->encap[1].proto == htons(ETH_P_8021Q)) {
++ if (vlan_encap) {
++ NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_CVLAN,
++ cvlan);
++ nf_flow_rule_vlan_match(&key->cvlan, &mask->cvlan,
++ tuple->encap[1].id,
++ tuple->encap[1].proto);
++ } else {
++ NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_VLAN,
++ vlan);
++ nf_flow_rule_vlan_match(&key->vlan, &mask->vlan,
++ tuple->encap[1].id,
++ tuple->encap[1].proto);
++ }
++ }
++
++ switch (tuple->l3proto) {
++ case AF_INET:
++ key->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
++ key->basic.n_proto = htons(ETH_P_IP);
++ key->ipv4.src = tuple->src_v4.s_addr;
++ mask->ipv4.src = 0xffffffff;
++ key->ipv4.dst = tuple->dst_v4.s_addr;
++ mask->ipv4.dst = 0xffffffff;
++ break;
++ case AF_INET6:
++ key->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
++ key->basic.n_proto = htons(ETH_P_IPV6);
++ key->ipv6.src = tuple->src_v6;
++ memset(&mask->ipv6.src, 0xff, sizeof(mask->ipv6.src));
++ key->ipv6.dst = tuple->dst_v6;
++ memset(&mask->ipv6.dst, 0xff, sizeof(mask->ipv6.dst));
++ break;
++ default:
++ return -EOPNOTSUPP;
++ }
++ mask->control.addr_type = 0xffff;
++ match->dissector.used_keys |= BIT(key->control.addr_type);
++ mask->basic.n_proto = 0xffff;
++
++ switch (tuple->l4proto) {
++ case IPPROTO_TCP:
++ key->tcp.flags = 0;
++ mask->tcp.flags = cpu_to_be16(be32_to_cpu(TCP_FLAG_RST | TCP_FLAG_FIN) >> 16);
++ match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_TCP);
++ break;
++ case IPPROTO_UDP:
++ break;
++ default:
++ return -EOPNOTSUPP;
++ }
++
++ key->basic.ip_proto = tuple->l4proto;
++ mask->basic.ip_proto = 0xff;
++
++ key->tp.src = tuple->src_port;
++ mask->tp.src = 0xffff;
++ key->tp.dst = tuple->dst_port;
++ mask->tp.dst = 0xffff;
++
++ match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_META) |
++ BIT(FLOW_DISSECTOR_KEY_CONTROL) |
++ BIT(FLOW_DISSECTOR_KEY_BASIC) |
++ BIT(FLOW_DISSECTOR_KEY_PORTS);
++ return 0;
++}
++
++static void flow_offload_mangle(struct flow_action_entry *entry,
++ enum flow_action_mangle_base htype, u32 offset,
++ const __be32 *value, const __be32 *mask)
++{
++ entry->id = FLOW_ACTION_MANGLE;
++ entry->mangle.htype = htype;
++ entry->mangle.offset = offset;
++ memcpy(&entry->mangle.mask, mask, sizeof(u32));
++ memcpy(&entry->mangle.val, value, sizeof(u32));
++}
++
++static inline struct flow_action_entry *
++flow_action_entry_next(struct nf_flow_rule *flow_rule)
++{
++ int i = flow_rule->rule->action.num_entries++;
++
++ return &flow_rule->rule->action.entries[i];
++}
++
++static int flow_offload_eth_src(struct net *net,
++ const struct flow_offload *flow,
++ enum flow_offload_tuple_dir dir,
++ struct nf_flow_rule *flow_rule)
++{
++ struct flow_action_entry *entry0 = flow_action_entry_next(flow_rule);
++ struct flow_action_entry *entry1 = flow_action_entry_next(flow_rule);
++ const struct flow_offload_tuple *other_tuple, *this_tuple;
++ struct net_device *dev = NULL;
++ const unsigned char *addr;
++ u32 mask, val;
++ u16 val16;
++
++ this_tuple = &flow->tuplehash[dir].tuple;
++
++ switch (this_tuple->xmit_type) {
++ case FLOW_OFFLOAD_XMIT_DIRECT:
++ addr = this_tuple->out.h_source;
++ break;
++ case FLOW_OFFLOAD_XMIT_NEIGH:
++ other_tuple = &flow->tuplehash[!dir].tuple;
++ dev = dev_get_by_index(net, other_tuple->iifidx);
++ if (!dev)
++ return -ENOENT;
++
++ addr = dev->dev_addr;
++ break;
++ default:
++ return -EOPNOTSUPP;
++ }
++
++ mask = ~0xffff0000;
++ memcpy(&val16, addr, 2);
++ val = val16 << 16;
++ flow_offload_mangle(entry0, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 4,
++ &val, &mask);
++
++ mask = ~0xffffffff;
++ memcpy(&val, addr + 2, 4);
++ flow_offload_mangle(entry1, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 8,
++ &val, &mask);
++
++ if (dev)
++ dev_put(dev);
++
++ return 0;
++}
++
++static int flow_offload_eth_dst(struct net *net,
++ const struct flow_offload *flow,
++ enum flow_offload_tuple_dir dir,
++ struct nf_flow_rule *flow_rule)
++{
++ struct flow_action_entry *entry0 = flow_action_entry_next(flow_rule);
++ struct flow_action_entry *entry1 = flow_action_entry_next(flow_rule);
++ const struct flow_offload_tuple *other_tuple, *this_tuple;
++ const struct dst_entry *dst_cache;
++ unsigned char ha[ETH_ALEN];
++ struct neighbour *n;
++ const void *daddr;
++ u32 mask, val;
++ u8 nud_state;
++ u16 val16;
++
++ this_tuple = &flow->tuplehash[dir].tuple;
++
++ switch (this_tuple->xmit_type) {
++ case FLOW_OFFLOAD_XMIT_DIRECT:
++ ether_addr_copy(ha, this_tuple->out.h_dest);
++ break;
++ case FLOW_OFFLOAD_XMIT_NEIGH:
++ other_tuple = &flow->tuplehash[!dir].tuple;
++ daddr = &other_tuple->src_v4;
++ dst_cache = this_tuple->dst_cache;
++ n = dst_neigh_lookup(dst_cache, daddr);
++ if (!n)
++ return -ENOENT;
++
++ read_lock_bh(&n->lock);
++ nud_state = n->nud_state;
++ ether_addr_copy(ha, n->ha);
++ read_unlock_bh(&n->lock);
++ neigh_release(n);
++
++ if (!(nud_state & NUD_VALID))
++ return -ENOENT;
++ break;
++ default:
++ return -EOPNOTSUPP;
++ }
++
++ mask = ~0xffffffff;
++ memcpy(&val, ha, 4);
++ flow_offload_mangle(entry0, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 0,
++ &val, &mask);
++
++ mask = ~0x0000ffff;
++ memcpy(&val16, ha + 4, 2);
++ val = val16;
++ flow_offload_mangle(entry1, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 4,
++ &val, &mask);
++
++ return 0;
++}
++
++static void flow_offload_ipv4_snat(struct net *net,
++ const struct flow_offload *flow,
++ enum flow_offload_tuple_dir dir,
++ struct nf_flow_rule *flow_rule)
++{
++ struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
++ u32 mask = ~htonl(0xffffffff);
++ __be32 addr;
++ u32 offset;
++
++ switch (dir) {
++ case FLOW_OFFLOAD_DIR_ORIGINAL:
++ addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v4.s_addr;
++ offset = offsetof(struct iphdr, saddr);
++ break;
++ case FLOW_OFFLOAD_DIR_REPLY:
++ addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v4.s_addr;
++ offset = offsetof(struct iphdr, daddr);
++ break;
++ default:
++ return;
++ }
++
++ flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP4, offset,
++ &addr, &mask);
++}
++
++static void flow_offload_ipv4_dnat(struct net *net,
++ const struct flow_offload *flow,
++ enum flow_offload_tuple_dir dir,
++ struct nf_flow_rule *flow_rule)
++{
++ struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
++ u32 mask = ~htonl(0xffffffff);
++ __be32 addr;
++ u32 offset;
++
++ switch (dir) {
++ case FLOW_OFFLOAD_DIR_ORIGINAL:
++ addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v4.s_addr;
++ offset = offsetof(struct iphdr, daddr);
++ break;
++ case FLOW_OFFLOAD_DIR_REPLY:
++ addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v4.s_addr;
++ offset = offsetof(struct iphdr, saddr);
++ break;
++ default:
++ return;
++ }
++
++ flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP4, offset,
++ &addr, &mask);
++}
++
++static void flow_offload_ipv6_mangle(struct nf_flow_rule *flow_rule,
++ unsigned int offset,
++ const __be32 *addr, const __be32 *mask)
++{
++ struct flow_action_entry *entry;
++ int i, j;
++
++ for (i = 0, j = 0; i < sizeof(struct in6_addr) / sizeof(u32); i += sizeof(u32), j++) {
++ entry = flow_action_entry_next(flow_rule);
++ flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP6,
++ offset + i, &addr[j], mask);
++ }
++}
++
++static void flow_offload_ipv6_snat(struct net *net,
++ const struct flow_offload *flow,
++ enum flow_offload_tuple_dir dir,
++ struct nf_flow_rule *flow_rule)
++{
++ u32 mask = ~htonl(0xffffffff);
++ const __be32 *addr;
++ u32 offset;
++
++ switch (dir) {
++ case FLOW_OFFLOAD_DIR_ORIGINAL:
++ addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v6.s6_addr32;
++ offset = offsetof(struct ipv6hdr, saddr);
++ break;
++ case FLOW_OFFLOAD_DIR_REPLY:
++ addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v6.s6_addr32;
++ offset = offsetof(struct ipv6hdr, daddr);
++ break;
++ default:
++ return;
++ }
++
++ flow_offload_ipv6_mangle(flow_rule, offset, addr, &mask);
++}
++
++static void flow_offload_ipv6_dnat(struct net *net,
++ const struct flow_offload *flow,
++ enum flow_offload_tuple_dir dir,
++ struct nf_flow_rule *flow_rule)
++{
++ u32 mask = ~htonl(0xffffffff);
++ const __be32 *addr;
++ u32 offset;
++
++ switch (dir) {
++ case FLOW_OFFLOAD_DIR_ORIGINAL:
++ addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v6.s6_addr32;
++ offset = offsetof(struct ipv6hdr, daddr);
++ break;
++ case FLOW_OFFLOAD_DIR_REPLY:
++ addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v6.s6_addr32;
++ offset = offsetof(struct ipv6hdr, saddr);
++ break;
++ default:
++ return;
++ }
++
++ flow_offload_ipv6_mangle(flow_rule, offset, addr, &mask);
++}
++
++static int flow_offload_l4proto(const struct flow_offload *flow)
++{
++ u8 protonum = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.l4proto;
++ u8 type = 0;
++
++ switch (protonum) {
++ case IPPROTO_TCP:
++ type = FLOW_ACT_MANGLE_HDR_TYPE_TCP;
++ break;
++ case IPPROTO_UDP:
++ type = FLOW_ACT_MANGLE_HDR_TYPE_UDP;
++ break;
++ default:
++ break;
++ }
++
++ return type;
++}
++
++static void flow_offload_port_snat(struct net *net,
++ const struct flow_offload *flow,
++ enum flow_offload_tuple_dir dir,
++ struct nf_flow_rule *flow_rule)
++{
++ struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
++ u32 mask, port;
++ u32 offset;
++
++ switch (dir) {
++ case FLOW_OFFLOAD_DIR_ORIGINAL:
++ port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_port);
++ offset = 0; /* offsetof(struct tcphdr, source); */
++ port = htonl(port << 16);
++ mask = ~htonl(0xffff0000);
++ break;
++ case FLOW_OFFLOAD_DIR_REPLY:
++ port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_port);
++ offset = 0; /* offsetof(struct tcphdr, dest); */
++ port = htonl(port);
++ mask = ~htonl(0xffff);
++ break;
++ default:
++ return;
++ }
++
++ flow_offload_mangle(entry, flow_offload_l4proto(flow), offset,
++ &port, &mask);
++}
++
++static void flow_offload_port_dnat(struct net *net,
++ const struct flow_offload *flow,
++ enum flow_offload_tuple_dir dir,
++ struct nf_flow_rule *flow_rule)
++{
++ struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
++ u32 mask, port;
++ u32 offset;
++
++ switch (dir) {
++ case FLOW_OFFLOAD_DIR_ORIGINAL:
++ port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_port);
++ offset = 0; /* offsetof(struct tcphdr, dest); */
++ port = htonl(port);
++ mask = ~htonl(0xffff);
++ break;
++ case FLOW_OFFLOAD_DIR_REPLY:
++ port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_port);
++ offset = 0; /* offsetof(struct tcphdr, source); */
++ port = htonl(port << 16);
++ mask = ~htonl(0xffff0000);
++ break;
++ default:
++ return;
++ }
++
++ flow_offload_mangle(entry, flow_offload_l4proto(flow), offset,
++ &port, &mask);
++}
++
++static void flow_offload_ipv4_checksum(struct net *net,
++ const struct flow_offload *flow,
++ struct nf_flow_rule *flow_rule)
++{
++ u8 protonum = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.l4proto;
++ struct flow_action_entry *entry = flow_action_entry_next(flow_rule);
++
++ entry->id = FLOW_ACTION_CSUM;
++ entry->csum_flags = TCA_CSUM_UPDATE_FLAG_IPV4HDR;
++
++ switch (protonum) {
++ case IPPROTO_TCP:
++ entry->csum_flags |= TCA_CSUM_UPDATE_FLAG_TCP;
++ break;
++ case IPPROTO_UDP:
++ entry->csum_flags |= TCA_CSUM_UPDATE_FLAG_UDP;
++ break;
++ }
++}
++
++static void flow_offload_redirect(struct net *net,
++ const struct flow_offload *flow,
++ enum flow_offload_tuple_dir dir,
++ struct nf_flow_rule *flow_rule)
++{
++ const struct flow_offload_tuple *this_tuple, *other_tuple;
++ struct flow_action_entry *entry;
++ struct net_device *dev;
++ int ifindex;
++
++ this_tuple = &flow->tuplehash[dir].tuple;
++ switch (this_tuple->xmit_type) {
++ case FLOW_OFFLOAD_XMIT_DIRECT:
++ this_tuple = &flow->tuplehash[dir].tuple;
++ ifindex = this_tuple->out.hw_ifidx;
++ break;
++ case FLOW_OFFLOAD_XMIT_NEIGH:
++ other_tuple = &flow->tuplehash[!dir].tuple;
++ ifindex = other_tuple->iifidx;
++ break;
++ default:
++ return;
++ }
++
++ dev = dev_get_by_index(net, ifindex);
++ if (!dev)
++ return;
++
++ entry = flow_action_entry_next(flow_rule);
++ entry->id = FLOW_ACTION_REDIRECT;
++ entry->dev = dev;
++}
++
++static void flow_offload_encap_tunnel(const struct flow_offload *flow,
++ enum flow_offload_tuple_dir dir,
++ struct nf_flow_rule *flow_rule)
++{
++ const struct flow_offload_tuple *this_tuple;
++ struct flow_action_entry *entry;
++ struct dst_entry *dst;
++
++ this_tuple = &flow->tuplehash[dir].tuple;
++ if (this_tuple->xmit_type == FLOW_OFFLOAD_XMIT_DIRECT)
++ return;
++
++ dst = this_tuple->dst_cache;
++ if (dst && dst->lwtstate) {
++ struct ip_tunnel_info *tun_info;
++
++ tun_info = lwt_tun_info(dst->lwtstate);
++ if (tun_info && (tun_info->mode & IP_TUNNEL_INFO_TX)) {
++ entry = flow_action_entry_next(flow_rule);
++ entry->id = FLOW_ACTION_TUNNEL_ENCAP;
++ entry->tunnel = tun_info;
++ }
++ }
++}
++
++static void flow_offload_decap_tunnel(const struct flow_offload *flow,
++ enum flow_offload_tuple_dir dir,
++ struct nf_flow_rule *flow_rule)
++{
++ const struct flow_offload_tuple *other_tuple;
++ struct flow_action_entry *entry;
++ struct dst_entry *dst;
++
++ other_tuple = &flow->tuplehash[!dir].tuple;
++ if (other_tuple->xmit_type == FLOW_OFFLOAD_XMIT_DIRECT)
++ return;
++
++ dst = other_tuple->dst_cache;
++ if (dst && dst->lwtstate) {
++ struct ip_tunnel_info *tun_info;
++
++ tun_info = lwt_tun_info(dst->lwtstate);
++ if (tun_info && (tun_info->mode & IP_TUNNEL_INFO_TX)) {
++ entry = flow_action_entry_next(flow_rule);
++ entry->id = FLOW_ACTION_TUNNEL_DECAP;
++ }
++ }
++}
++
++static int
++nf_flow_rule_route_common(struct net *net, const struct flow_offload *flow,
++ enum flow_offload_tuple_dir dir,
++ struct nf_flow_rule *flow_rule)
++{
++ const struct flow_offload_tuple *other_tuple;
++ const struct flow_offload_tuple *tuple;
++ int i;
++
++ flow_offload_decap_tunnel(flow, dir, flow_rule);
++ flow_offload_encap_tunnel(flow, dir, flow_rule);
++
++ if (flow_offload_eth_src(net, flow, dir, flow_rule) < 0 ||
++ flow_offload_eth_dst(net, flow, dir, flow_rule) < 0)
++ return -1;
++
++ tuple = &flow->tuplehash[dir].tuple;
++
++ for (i = 0; i < tuple->encap_num; i++) {
++ struct flow_action_entry *entry;
++
++ if (tuple->in_vlan_ingress & BIT(i))
++ continue;
++
++ if (tuple->encap[i].proto == htons(ETH_P_8021Q)) {
++ entry = flow_action_entry_next(flow_rule);
++ entry->id = FLOW_ACTION_VLAN_POP;
++ }
++ }
++
++ other_tuple = &flow->tuplehash[!dir].tuple;
++
++ for (i = 0; i < other_tuple->encap_num; i++) {
++ struct flow_action_entry *entry;
++
++ if (other_tuple->in_vlan_ingress & BIT(i))
++ continue;
++
++ entry = flow_action_entry_next(flow_rule);
++
++ switch (other_tuple->encap[i].proto) {
++ case htons(ETH_P_PPP_SES):
++ entry->id = FLOW_ACTION_PPPOE_PUSH;
++ entry->pppoe.sid = other_tuple->encap[i].id;
++ break;
++ case htons(ETH_P_8021Q):
++ entry->id = FLOW_ACTION_VLAN_PUSH;
++ entry->vlan.vid = other_tuple->encap[i].id;
++ entry->vlan.proto = other_tuple->encap[i].proto;
++ break;
++ }
++ }
++
++ return 0;
++}
++
++int nf_flow_rule_route_ipv4(struct net *net, const struct flow_offload *flow,
++ enum flow_offload_tuple_dir dir,
++ struct nf_flow_rule *flow_rule)
++{
++ if (nf_flow_rule_route_common(net, flow, dir, flow_rule) < 0)
++ return -1;
++
++ if (test_bit(NF_FLOW_SNAT, &flow->flags)) {
++ flow_offload_ipv4_snat(net, flow, dir, flow_rule);
++ flow_offload_port_snat(net, flow, dir, flow_rule);
++ }
++ if (test_bit(NF_FLOW_DNAT, &flow->flags)) {
++ flow_offload_ipv4_dnat(net, flow, dir, flow_rule);
++ flow_offload_port_dnat(net, flow, dir, flow_rule);
++ }
++ if (test_bit(NF_FLOW_SNAT, &flow->flags) ||
++ test_bit(NF_FLOW_DNAT, &flow->flags))
++ flow_offload_ipv4_checksum(net, flow, flow_rule);
++
++ flow_offload_redirect(net, flow, dir, flow_rule);
++
++ return 0;
++}
++EXPORT_SYMBOL_GPL(nf_flow_rule_route_ipv4);
++
++int nf_flow_rule_route_ipv6(struct net *net, const struct flow_offload *flow,
++ enum flow_offload_tuple_dir dir,
++ struct nf_flow_rule *flow_rule)
++{
++ if (nf_flow_rule_route_common(net, flow, dir, flow_rule) < 0)
++ return -1;
++
++ if (test_bit(NF_FLOW_SNAT, &flow->flags)) {
++ flow_offload_ipv6_snat(net, flow, dir, flow_rule);
++ flow_offload_port_snat(net, flow, dir, flow_rule);
++ }
++ if (test_bit(NF_FLOW_DNAT, &flow->flags)) {
++ flow_offload_ipv6_dnat(net, flow, dir, flow_rule);
++ flow_offload_port_dnat(net, flow, dir, flow_rule);
++ }
++
++ flow_offload_redirect(net, flow, dir, flow_rule);
++
++ return 0;
++}
++EXPORT_SYMBOL_GPL(nf_flow_rule_route_ipv6);
++
++#define NF_FLOW_RULE_ACTION_MAX 16
++
++static struct nf_flow_rule *
++nf_flow_offload_rule_alloc(struct net *net,
++ const struct flow_offload_work *offload,
++ enum flow_offload_tuple_dir dir)
++{
++ const struct nf_flowtable *flowtable = offload->flowtable;
++ const struct flow_offload_tuple *tuple, *other_tuple;
++ const struct flow_offload *flow = offload->flow;
++ struct dst_entry *other_dst = NULL;
++ struct nf_flow_rule *flow_rule;
++ int err = -ENOMEM;
++
++ flow_rule = kzalloc(sizeof(*flow_rule), GFP_KERNEL);
++ if (!flow_rule)
++ goto err_flow;
++
++ flow_rule->rule = flow_rule_alloc(NF_FLOW_RULE_ACTION_MAX);
++ if (!flow_rule->rule)
++ goto err_flow_rule;
++
++ flow_rule->rule->match.dissector = &flow_rule->match.dissector;
++ flow_rule->rule->match.mask = &flow_rule->match.mask;
++ flow_rule->rule->match.key = &flow_rule->match.key;
++
++ tuple = &flow->tuplehash[dir].tuple;
++ other_tuple = &flow->tuplehash[!dir].tuple;
++ if (other_tuple->xmit_type == FLOW_OFFLOAD_XMIT_NEIGH)
++ other_dst = other_tuple->dst_cache;
++
++ err = nf_flow_rule_match(&flow_rule->match, tuple, other_dst);
++ if (err < 0)
++ goto err_flow_match;
++
++ flow_rule->rule->action.num_entries = 0;
++ if (flowtable->type->action(net, flow, dir, flow_rule) < 0)
++ goto err_flow_match;
++
++ return flow_rule;
++
++err_flow_match:
++ kfree(flow_rule->rule);
++err_flow_rule:
++ kfree(flow_rule);
++err_flow:
++ return NULL;
++}
++
++static void __nf_flow_offload_destroy(struct nf_flow_rule *flow_rule)
++{
++ struct flow_action_entry *entry;
++ int i;
++
++ for (i = 0; i < flow_rule->rule->action.num_entries; i++) {
++ entry = &flow_rule->rule->action.entries[i];
++ if (entry->id != FLOW_ACTION_REDIRECT)
++ continue;
++
++ dev_put(entry->dev);
++ }
++ kfree(flow_rule->rule);
++ kfree(flow_rule);
++}
++
++static void nf_flow_offload_destroy(struct nf_flow_rule *flow_rule[])
++{
++ int i;
++
++ for (i = 0; i < FLOW_OFFLOAD_DIR_MAX; i++)
++ __nf_flow_offload_destroy(flow_rule[i]);
++}
++
++static int nf_flow_offload_alloc(const struct flow_offload_work *offload,
++ struct nf_flow_rule *flow_rule[])
++{
++ struct net *net = read_pnet(&offload->flowtable->net);
++
++ flow_rule[0] = nf_flow_offload_rule_alloc(net, offload,
++ FLOW_OFFLOAD_DIR_ORIGINAL);
++ if (!flow_rule[0])
++ return -ENOMEM;
++
++ flow_rule[1] = nf_flow_offload_rule_alloc(net, offload,
++ FLOW_OFFLOAD_DIR_REPLY);
++ if (!flow_rule[1]) {
++ __nf_flow_offload_destroy(flow_rule[0]);
++ return -ENOMEM;
++ }
++
++ return 0;
++}
++
++static void nf_flow_offload_init(struct flow_cls_offload *cls_flow,
++ __be16 proto, int priority,
++ enum flow_cls_command cmd,
++ const struct flow_offload_tuple *tuple,
++ struct netlink_ext_ack *extack)
++{
++ cls_flow->common.protocol = proto;
++ cls_flow->common.prio = priority;
++ cls_flow->common.extack = extack;
++ cls_flow->command = cmd;
++ cls_flow->cookie = (unsigned long)tuple;
++}
++
++static int nf_flow_offload_tuple(struct nf_flowtable *flowtable,
++ struct flow_offload *flow,
++ struct nf_flow_rule *flow_rule,
++ enum flow_offload_tuple_dir dir,
++ int priority, int cmd,
++ struct flow_stats *stats,
++ struct list_head *block_cb_list)
++{
++ struct flow_cls_offload cls_flow = {};
++ struct flow_block_cb *block_cb;
++ struct netlink_ext_ack extack;
++ __be16 proto = ETH_P_ALL;
++ int err, i = 0;
++
++ nf_flow_offload_init(&cls_flow, proto, priority, cmd,
++ &flow->tuplehash[dir].tuple, &extack);
++ if (cmd == FLOW_CLS_REPLACE)
++ cls_flow.rule = flow_rule->rule;
++
++ down_read(&flowtable->flow_block_lock);
++ list_for_each_entry(block_cb, block_cb_list, list) {
++ err = block_cb->cb(TC_SETUP_CLSFLOWER, &cls_flow,
++ block_cb->cb_priv);
++ if (err < 0)
++ continue;
++
++ i++;
++ }
++ up_read(&flowtable->flow_block_lock);
++
++ if (cmd == FLOW_CLS_STATS)
++ memcpy(stats, &cls_flow.stats, sizeof(*stats));
++
++ return i;
++}
++
++static int flow_offload_tuple_add(struct flow_offload_work *offload,
++ struct nf_flow_rule *flow_rule,
++ enum flow_offload_tuple_dir dir)
++{
++ return nf_flow_offload_tuple(offload->flowtable, offload->flow,
++ flow_rule, dir, offload->priority,
++ FLOW_CLS_REPLACE, NULL,
++ &offload->flowtable->flow_block.cb_list);
++}
++
++static void flow_offload_tuple_del(struct flow_offload_work *offload,
++ enum flow_offload_tuple_dir dir)
++{
++ nf_flow_offload_tuple(offload->flowtable, offload->flow, NULL, dir,
++ offload->priority, FLOW_CLS_DESTROY, NULL,
++ &offload->flowtable->flow_block.cb_list);
++}
++
++static int flow_offload_rule_add(struct flow_offload_work *offload,
++ struct nf_flow_rule *flow_rule[])
++{
++ int ok_count = 0;
++
++ ok_count += flow_offload_tuple_add(offload, flow_rule[0],
++ FLOW_OFFLOAD_DIR_ORIGINAL);
++ ok_count += flow_offload_tuple_add(offload, flow_rule[1],
++ FLOW_OFFLOAD_DIR_REPLY);
++ if (ok_count == 0)
++ return -ENOENT;
++
++ return 0;
++}
++
++static void flow_offload_work_add(struct flow_offload_work *offload)
++{
++ struct nf_flow_rule *flow_rule[FLOW_OFFLOAD_DIR_MAX];
++ int err;
++
++ err = nf_flow_offload_alloc(offload, flow_rule);
++ if (err < 0)
++ return;
++
++ err = flow_offload_rule_add(offload, flow_rule);
++ if (err < 0)
++ goto out;
++
++ set_bit(IPS_HW_OFFLOAD_BIT, &offload->flow->ct->status);
++
++out:
++ nf_flow_offload_destroy(flow_rule);
++}
++
++static void flow_offload_work_del(struct flow_offload_work *offload)
++{
++ clear_bit(IPS_HW_OFFLOAD_BIT, &offload->flow->ct->status);
++ flow_offload_tuple_del(offload, FLOW_OFFLOAD_DIR_ORIGINAL);
++ flow_offload_tuple_del(offload, FLOW_OFFLOAD_DIR_REPLY);
++ set_bit(NF_FLOW_HW_DEAD, &offload->flow->flags);
++}
++
++static void flow_offload_tuple_stats(struct flow_offload_work *offload,
++ enum flow_offload_tuple_dir dir,
++ struct flow_stats *stats)
++{
++ nf_flow_offload_tuple(offload->flowtable, offload->flow, NULL, dir,
++ offload->priority, FLOW_CLS_STATS, stats,
++ &offload->flowtable->flow_block.cb_list);
++}
++
++static void flow_offload_work_stats(struct flow_offload_work *offload)
++{
++ struct flow_stats stats[FLOW_OFFLOAD_DIR_MAX] = {};
++ u64 lastused;
++
++ flow_offload_tuple_stats(offload, FLOW_OFFLOAD_DIR_ORIGINAL, &stats[0]);
++ flow_offload_tuple_stats(offload, FLOW_OFFLOAD_DIR_REPLY, &stats[1]);
++
++ lastused = max_t(u64, stats[0].lastused, stats[1].lastused);
++ offload->flow->timeout = max_t(u64, offload->flow->timeout,
++ lastused + flow_offload_get_timeout(offload->flow));
++
++ if (offload->flowtable->flags & NF_FLOWTABLE_COUNTER) {
++ if (stats[0].pkts)
++ nf_ct_acct_add(offload->flow->ct,
++ FLOW_OFFLOAD_DIR_ORIGINAL,
++ stats[0].pkts, stats[0].bytes);
++ if (stats[1].pkts)
++ nf_ct_acct_add(offload->flow->ct,
++ FLOW_OFFLOAD_DIR_REPLY,
++ stats[1].pkts, stats[1].bytes);
++ }
++}
++
++static void flow_offload_work_handler(struct work_struct *work)
++{
++ struct flow_offload_work *offload;
++
++ offload = container_of(work, struct flow_offload_work, work);
++ switch (offload->cmd) {
++ case FLOW_CLS_REPLACE:
++ flow_offload_work_add(offload);
++ break;
++ case FLOW_CLS_DESTROY:
++ flow_offload_work_del(offload);
++ break;
++ case FLOW_CLS_STATS:
++ flow_offload_work_stats(offload);
++ break;
++ default:
++ WARN_ON_ONCE(1);
++ }
++
++ clear_bit(NF_FLOW_HW_PENDING, &offload->flow->flags);
++ kfree(offload);
++}
++
++static void flow_offload_queue_work(struct flow_offload_work *offload)
++{
++ if (offload->cmd == FLOW_CLS_REPLACE)
++ queue_work(nf_flow_offload_add_wq, &offload->work);
++ else if (offload->cmd == FLOW_CLS_DESTROY)
++ queue_work(nf_flow_offload_del_wq, &offload->work);
++ else
++ queue_work(nf_flow_offload_stats_wq, &offload->work);
++}
++
++static struct flow_offload_work *
++nf_flow_offload_work_alloc(struct nf_flowtable *flowtable,
++ struct flow_offload *flow, unsigned int cmd)
++{
++ struct flow_offload_work *offload;
++
++ if (test_and_set_bit(NF_FLOW_HW_PENDING, &flow->flags))
++ return NULL;
++
++ offload = kmalloc(sizeof(struct flow_offload_work), GFP_ATOMIC);
++ if (!offload) {
++ clear_bit(NF_FLOW_HW_PENDING, &flow->flags);
++ return NULL;
++ }
++
++ offload->cmd = cmd;
++ offload->flow = flow;
++ offload->priority = flowtable->priority;
++ offload->flowtable = flowtable;
++ INIT_WORK(&offload->work, flow_offload_work_handler);
++
++ return offload;
++}
++
++
++void nf_flow_offload_add(struct nf_flowtable *flowtable,
++ struct flow_offload *flow)
++{
++ struct flow_offload_work *offload;
++
++ offload = nf_flow_offload_work_alloc(flowtable, flow, FLOW_CLS_REPLACE);
++ if (!offload)
++ return;
++
++ flow_offload_queue_work(offload);
++}
++
++void nf_flow_offload_del(struct nf_flowtable *flowtable,
++ struct flow_offload *flow)
++{
++ struct flow_offload_work *offload;
++
++ offload = nf_flow_offload_work_alloc(flowtable, flow, FLOW_CLS_DESTROY);
++ if (!offload)
++ return;
++
++ set_bit(NF_FLOW_HW_DYING, &flow->flags);
++ flow_offload_queue_work(offload);
++}
++
++void nf_flow_offload_stats(struct nf_flowtable *flowtable,
++ struct flow_offload *flow, bool force)
++{
++ struct flow_offload_work *offload;
++ __s32 delta;
++
++ if (!force) {
++ delta = nf_flow_timeout_delta(flow->timeout);
++ if ((delta >= (9 * flow_offload_get_timeout(flow)) / 10))
++ return;
++ }
++
++ offload = nf_flow_offload_work_alloc(flowtable, flow, FLOW_CLS_STATS);
++ if (!offload)
++ return;
++
++ flow_offload_queue_work(offload);
++}
++
++void nf_flow_table_offload_flush(struct nf_flowtable *flowtable)
++{
++ if (nf_flowtable_hw_offload(flowtable)) {
++ flush_workqueue(nf_flow_offload_add_wq);
++ flush_workqueue(nf_flow_offload_del_wq);
++ flush_workqueue(nf_flow_offload_stats_wq);
++ }
++}
++
++static int nf_flow_table_block_setup(struct nf_flowtable *flowtable,
++ struct flow_block_offload *bo,
++ enum flow_block_command cmd)
++{
++ struct flow_block_cb *block_cb, *next;
++ int err = 0;
++
++ down_write(&flowtable->flow_block_lock);
++
++ switch (cmd) {
++ case FLOW_BLOCK_BIND:
++ list_splice(&bo->cb_list, &flowtable->flow_block.cb_list);
++ break;
++ case FLOW_BLOCK_UNBIND:
++ list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) {
++ list_del(&block_cb->list);
++ flow_block_cb_free(block_cb);
++ }
++ break;
++ default:
++ WARN_ON_ONCE(1);
++ err = -EOPNOTSUPP;
++ }
++
++ up_write(&flowtable->flow_block_lock);
++
++ return err;
++}
++
++static void nf_flow_table_block_offload_init(struct flow_block_offload *bo,
++ struct net *net,
++ enum flow_block_command cmd,
++ struct nf_flowtable *flowtable,
++ struct netlink_ext_ack *extack)
++{
++ memset(bo, 0, sizeof(*bo));
++ bo->net = net;
++ bo->block = &flowtable->flow_block;
++ bo->command = cmd;
++ bo->binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS;
++ bo->extack = extack;
++ INIT_LIST_HEAD(&bo->cb_list);
++}
++
++static int nf_flow_table_indr_offload_cmd(struct flow_block_offload *bo,
++ struct nf_flowtable *flowtable,
++ struct net_device *dev,
++ enum flow_block_command cmd,
++ struct netlink_ext_ack *extack)
++{
++ nf_flow_table_block_offload_init(bo, dev_net(dev), cmd, flowtable,
++ extack);
++ flow_indr_block_call(dev, bo, cmd);
++
++ if (list_empty(&bo->cb_list))
++ return -EOPNOTSUPP;
++
++ return 0;
++}
++
++static int nf_flow_table_offload_cmd(struct flow_block_offload *bo,
++ struct nf_flowtable *flowtable,
++ struct net_device *dev,
++ enum flow_block_command cmd,
++ struct netlink_ext_ack *extack)
++{
++ int err;
++
++ nf_flow_table_block_offload_init(bo, dev_net(dev), cmd, flowtable,
++ extack);
++ down_write(&flowtable->flow_block_lock);
++ err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_FT, bo);
++ up_write(&flowtable->flow_block_lock);
++ if (err < 0)
++ return err;
++
++ return 0;
++}
++
++int nf_flow_table_offload_setup(struct nf_flowtable *flowtable,
++ struct net_device *dev,
++ enum flow_block_command cmd)
++{
++ struct netlink_ext_ack extack = {};
++ struct flow_block_offload bo;
++ int err;
++
++ if (!nf_flowtable_hw_offload(flowtable))
++ return 0;
++
++ if (dev->netdev_ops->ndo_setup_tc)
++ err = nf_flow_table_offload_cmd(&bo, flowtable, dev, cmd,
++ &extack);
++ else
++ err = nf_flow_table_indr_offload_cmd(&bo, flowtable, dev, cmd,
++ &extack);
++ if (err < 0)
++ return err;
++
++ return nf_flow_table_block_setup(flowtable, &bo, cmd);
++}
++EXPORT_SYMBOL_GPL(nf_flow_table_offload_setup);
++
++int nf_flow_table_offload_init(void)
++{
++ nf_flow_offload_add_wq = alloc_workqueue("nf_ft_offload_add",
++ WQ_UNBOUND | WQ_SYSFS, 0);
++ if (!nf_flow_offload_add_wq)
++ return -ENOMEM;
++
++ nf_flow_offload_del_wq = alloc_workqueue("nf_ft_offload_del",
++ WQ_UNBOUND | WQ_SYSFS, 0);
++ if (!nf_flow_offload_del_wq)
++ goto err_del_wq;
++
++ nf_flow_offload_stats_wq = alloc_workqueue("nf_ft_offload_stats",
++ WQ_UNBOUND | WQ_SYSFS, 0);
++ if (!nf_flow_offload_stats_wq)
++ goto err_stats_wq;
++
++ return 0;
++
++err_stats_wq:
++ destroy_workqueue(nf_flow_offload_del_wq);
++err_del_wq:
++ destroy_workqueue(nf_flow_offload_add_wq);
++ return -ENOMEM;
++}
++
++void nf_flow_table_offload_exit(void)
++{
++ destroy_workqueue(nf_flow_offload_add_wq);
++ destroy_workqueue(nf_flow_offload_del_wq);
++ destroy_workqueue(nf_flow_offload_stats_wq);
++}
+diff --git a/net/netfilter/xt_FLOWOFFLOAD.c b/net/netfilter/xt_FLOWOFFLOAD.c
+new file mode 100644
+index 0000000..9120c60
+--- /dev/null
++++ b/net/netfilter/xt_FLOWOFFLOAD.c
+@@ -0,0 +1,780 @@
++/*
++ * Copyright (C) 2018-2021 Felix Fietkau <nbd@nbd.name>
++ *
++ * This program is free software; you can redistribute it and/or modify
++ * it under the terms of the GNU General Public License version 2 as
++ * published by the Free Software Foundation.
++ */
++#include <linux/module.h>
++#include <linux/init.h>
++#include <linux/netfilter.h>
++#include <linux/netfilter/xt_FLOWOFFLOAD.h>
++#include <linux/if_vlan.h>
++#include <net/ip.h>
++#include <net/netfilter/nf_conntrack.h>
++#include <net/netfilter/nf_conntrack_core.h>
++#include <net/netfilter/nf_conntrack_extend.h>
++#include <net/netfilter/nf_conntrack_helper.h>
++#include <net/netfilter/nf_flow_table.h>
++
++struct xt_flowoffload_hook {
++ struct hlist_node list;
++ struct nf_hook_ops ops;
++ struct net *net;
++ bool registered;
++ bool used;
++};
++
++struct xt_flowoffload_table {
++ struct nf_flowtable ft;
++ struct hlist_head hooks;
++ struct delayed_work work;
++};
++
++struct nf_forward_info {
++ const struct net_device *indev;
++ const struct net_device *outdev;
++ const struct net_device *hw_outdev;
++ struct id {
++ __u16 id;
++ __be16 proto;
++ } encap[NF_FLOW_TABLE_ENCAP_MAX];
++ u8 num_encaps;
++ u8 ingress_vlans;
++ u8 h_source[ETH_ALEN];
++ u8 h_dest[ETH_ALEN];
++ enum flow_offload_xmit_type xmit_type;
++};
++
++static DEFINE_SPINLOCK(hooks_lock);
++
++struct xt_flowoffload_table flowtable[2];
++
++static unsigned int
++xt_flowoffload_net_hook(void *priv, struct sk_buff *skb,
++ const struct nf_hook_state *state)
++{
++ struct vlan_ethhdr *veth;
++ __be16 proto;
++
++ switch (skb->protocol) {
++ case htons(ETH_P_8021Q):
++ veth = (struct vlan_ethhdr *)skb_mac_header(skb);
++ proto = veth->h_vlan_encapsulated_proto;
++ break;
++ case htons(ETH_P_PPP_SES):
++ proto = nf_flow_pppoe_proto(skb);
++ break;
++ default:
++ proto = skb->protocol;
++ break;
++ }
++
++ switch (proto) {
++ case htons(ETH_P_IP):
++ return nf_flow_offload_ip_hook(priv, skb, state);
++ case htons(ETH_P_IPV6):
++ return nf_flow_offload_ipv6_hook(priv, skb, state);
++ }
++
++ return NF_ACCEPT;
++}
++
++static int
++xt_flowoffload_create_hook(struct xt_flowoffload_table *table,
++ struct net_device *dev)
++{
++ struct xt_flowoffload_hook *hook;
++ struct nf_hook_ops *ops;
++
++ hook = kzalloc(sizeof(*hook), GFP_ATOMIC);
++ if (!hook)
++ return -ENOMEM;
++
++ ops = &hook->ops;
++ ops->pf = NFPROTO_NETDEV;
++ ops->hooknum = NF_NETDEV_INGRESS;
++ ops->priority = 10;
++ ops->priv = &table->ft;
++ ops->hook = xt_flowoffload_net_hook;
++ ops->dev = dev;
++
++ hlist_add_head(&hook->list, &table->hooks);
++ mod_delayed_work(system_power_efficient_wq, &table->work, 0);
++
++ return 0;
++}
++
++static struct xt_flowoffload_hook *
++flow_offload_lookup_hook(struct xt_flowoffload_table *table,
++ struct net_device *dev)
++{
++ struct xt_flowoffload_hook *hook;
++
++ hlist_for_each_entry(hook, &table->hooks, list) {
++ if (hook->ops.dev == dev)
++ return hook;
++ }
++
++ return NULL;
++}
++
++static void
++xt_flowoffload_check_device(struct xt_flowoffload_table *table,
++ struct net_device *dev)
++{
++ struct xt_flowoffload_hook *hook;
++
++ if (!dev)
++ return;
++
++ spin_lock_bh(&hooks_lock);
++ hook = flow_offload_lookup_hook(table, dev);
++ if (hook)
++ hook->used = true;
++ else
++ xt_flowoffload_create_hook(table, dev);
++ spin_unlock_bh(&hooks_lock);
++}
++
++static void
++xt_flowoffload_register_hooks(struct xt_flowoffload_table *table)
++{
++ struct xt_flowoffload_hook *hook;
++
++restart:
++ hlist_for_each_entry(hook, &table->hooks, list) {
++ if (hook->registered)
++ continue;
++
++ hook->registered = true;
++ hook->net = dev_net(hook->ops.dev);
++ spin_unlock_bh(&hooks_lock);
++ nf_register_net_hook(hook->net, &hook->ops);
++ if (table->ft.flags & NF_FLOWTABLE_HW_OFFLOAD)
++ table->ft.type->setup(&table->ft, hook->ops.dev,
++ FLOW_BLOCK_BIND);
++ spin_lock_bh(&hooks_lock);
++ goto restart;
++ }
++
++}
++
++static bool
++xt_flowoffload_cleanup_hooks(struct xt_flowoffload_table *table)
++{
++ struct xt_flowoffload_hook *hook;
++ bool active = false;
++
++restart:
++ spin_lock_bh(&hooks_lock);
++ hlist_for_each_entry(hook, &table->hooks, list) {
++ if (hook->used || !hook->registered) {
++ active = true;
++ continue;
++ }
++
++ hlist_del(&hook->list);
++ spin_unlock_bh(&hooks_lock);
++ if (table->ft.flags & NF_FLOWTABLE_HW_OFFLOAD)
++ table->ft.type->setup(&table->ft, hook->ops.dev,
++ FLOW_BLOCK_UNBIND);
++ nf_unregister_net_hook(hook->net, &hook->ops);
++ kfree(hook);
++ goto restart;
++ }
++ spin_unlock_bh(&hooks_lock);
++
++ return active;
++}
++
++static void
++xt_flowoffload_check_hook(struct flow_offload *flow, void *data)
++{
++ struct xt_flowoffload_table *table = data;
++ struct flow_offload_tuple *tuple0 = &flow->tuplehash[0].tuple;
++ struct flow_offload_tuple *tuple1 = &flow->tuplehash[1].tuple;
++ struct xt_flowoffload_hook *hook;
++
++ spin_lock_bh(&hooks_lock);
++ hlist_for_each_entry(hook, &table->hooks, list) {
++ if (hook->ops.dev->ifindex != tuple0->iifidx &&
++ hook->ops.dev->ifindex != tuple1->iifidx)
++ continue;
++
++ hook->used = true;
++ }
++ spin_unlock_bh(&hooks_lock);
++}
++
++static void
++xt_flowoffload_hook_work(struct work_struct *work)
++{
++ struct xt_flowoffload_table *table;
++ struct xt_flowoffload_hook *hook;
++ int err;
++
++ table = container_of(work, struct xt_flowoffload_table, work.work);
++
++ spin_lock_bh(&hooks_lock);
++ xt_flowoffload_register_hooks(table);
++ hlist_for_each_entry(hook, &table->hooks, list)
++ hook->used = false;
++ spin_unlock_bh(&hooks_lock);
++
++ err = nf_flow_table_iterate(&table->ft, xt_flowoffload_check_hook,
++ table);
++ if (err && err != -EAGAIN)
++ goto out;
++
++ if (!xt_flowoffload_cleanup_hooks(table))
++ return;
++
++out:
++ queue_delayed_work(system_power_efficient_wq, &table->work, HZ);
++}
++
++static bool
++xt_flowoffload_skip(struct sk_buff *skb, int family)
++{
++ if (skb_sec_path(skb))
++ return true;
++
++ if (family == NFPROTO_IPV4) {
++ const struct ip_options *opt = &(IPCB(skb)->opt);
++
++ if (unlikely(opt->optlen))
++ return true;
++ }
++
++ return false;
++}
++
++static enum flow_offload_xmit_type nf_xmit_type(struct dst_entry *dst)
++{
++ if (dst_xfrm(dst))
++ return FLOW_OFFLOAD_XMIT_XFRM;
++
++ return FLOW_OFFLOAD_XMIT_NEIGH;
++}
++
++static void nf_default_forward_path(struct nf_flow_route *route,
++ struct dst_entry *dst_cache,
++ enum ip_conntrack_dir dir,
++ struct net_device **dev)
++{
++ route->tuple[!dir].in.ifindex = dst_cache->dev->ifindex;
++ route->tuple[dir].dst = dst_cache;
++ route->tuple[dir].xmit_type = nf_xmit_type(dst_cache);
++}
++
++static bool nf_is_valid_ether_device(const struct net_device *dev)
++{
++ if (!dev || (dev->flags & IFF_LOOPBACK) || dev->type != ARPHRD_ETHER ||
++ dev->addr_len != ETH_ALEN || !is_valid_ether_addr(dev->dev_addr))
++ return false;
++
++ return true;
++}
++
++static void nf_dev_path_info(const struct net_device_path_stack *stack,
++ struct nf_forward_info *info,
++ unsigned char *ha)
++{
++ const struct net_device_path *path;
++ int i;
++
++ memcpy(info->h_dest, ha, ETH_ALEN);
++
++ for (i = 0; i < stack->num_paths; i++) {
++ path = &stack->path[i];
++
++ info->indev = path->dev;
++
++ switch (path->type) {
++ case DEV_PATH_ETHERNET:
++ case DEV_PATH_DSA:
++ case DEV_PATH_VLAN:
++ case DEV_PATH_PPPOE:
++ if (is_zero_ether_addr(info->h_source))
++ memcpy(info->h_source, path->dev->dev_addr, ETH_ALEN);
++
++ if (path->type == DEV_PATH_ETHERNET)
++ break;
++ if (path->type == DEV_PATH_DSA) {
++ i = stack->num_paths;
++ break;
++ }
++
++ /* DEV_PATH_VLAN and DEV_PATH_PPPOE */
++ if (info->num_encaps >= NF_FLOW_TABLE_ENCAP_MAX) {
++ info->indev = NULL;
++ break;
++ }
++ if (!info->outdev)
++ info->outdev = path->dev;
++ info->encap[info->num_encaps].id = path->encap.id;
++ info->encap[info->num_encaps].proto = path->encap.proto;
++ info->num_encaps++;
++ if (path->type == DEV_PATH_PPPOE)
++ memcpy(info->h_dest, path->encap.h_dest, ETH_ALEN);
++ break;
++ case DEV_PATH_BRIDGE:
++ if (is_zero_ether_addr(info->h_source))
++ memcpy(info->h_source, path->dev->dev_addr, ETH_ALEN);
++
++ switch (path->bridge.vlan_mode) {
++ case DEV_PATH_BR_VLAN_UNTAG_HW:
++ info->ingress_vlans |= BIT(info->num_encaps - 1);
++ break;
++ case DEV_PATH_BR_VLAN_TAG:
++ info->encap[info->num_encaps].id = path->bridge.vlan_id;
++ info->encap[info->num_encaps].proto = path->bridge.vlan_proto;
++ info->num_encaps++;
++ break;
++ case DEV_PATH_BR_VLAN_UNTAG:
++ info->num_encaps--;
++ break;
++ case DEV_PATH_BR_VLAN_KEEP:
++ break;
++ }
++ break;
++ default:
++ break;
++ }
++ }
++ if (!info->outdev)
++ info->outdev = info->indev;
++
++ info->hw_outdev = info->indev;
++
++ if (nf_is_valid_ether_device(info->indev))
++ info->xmit_type = FLOW_OFFLOAD_XMIT_DIRECT;
++}
++
++static int nf_dev_fill_forward_path(const struct nf_flow_route *route,
++ const struct dst_entry *dst_cache,
++ const struct nf_conn *ct,
++ enum ip_conntrack_dir dir, u8 *ha,
++ struct net_device_path_stack *stack)
++{
++ const void *daddr = &ct->tuplehash[!dir].tuple.src.u3;
++ struct net_device *dev = dst_cache->dev;
++ struct neighbour *n;
++ u8 nud_state;
++
++ if (!nf_is_valid_ether_device(dev))
++ goto out;
++
++ n = dst_neigh_lookup(dst_cache, daddr);
++ if (!n)
++ return -1;
++
++ read_lock_bh(&n->lock);
++ nud_state = n->nud_state;
++ ether_addr_copy(ha, n->ha);
++ read_unlock_bh(&n->lock);
++ neigh_release(n);
++
++ if (!(nud_state & NUD_VALID))
++ return -1;
++
++out:
++ return dev_fill_forward_path(dev, ha, stack);
++}
++
++static int nf_dev_forward_path(struct nf_flow_route *route,
++ const struct nf_conn *ct,
++ enum ip_conntrack_dir dir,
++ struct net_device **devs)
++{
++ const struct dst_entry *dst = route->tuple[dir].dst;
++ struct net_device_path_stack stack;
++ struct nf_forward_info info = {};
++ unsigned char ha[ETH_ALEN];
++ int i;
++
++ if (nf_dev_fill_forward_path(route, dst, ct, dir, ha, &stack) >= 0)
++ nf_dev_path_info(&stack, &info, ha);
++
++ devs[!dir] = (struct net_device *)info.indev;
++ if (!info.indev)
++ return -1;
++
++ route->tuple[!dir].in.ifindex = info.indev->ifindex;
++ for (i = 0; i < info.num_encaps; i++) {
++ route->tuple[!dir].in.encap[i].id = info.encap[i].id;
++ route->tuple[!dir].in.encap[i].proto = info.encap[i].proto;
++ }
++ route->tuple[!dir].in.num_encaps = info.num_encaps;
++ route->tuple[!dir].in.ingress_vlans = info.ingress_vlans;
++
++ if (info.xmit_type == FLOW_OFFLOAD_XMIT_DIRECT) {
++ memcpy(route->tuple[dir].out.h_source, info.h_source, ETH_ALEN);
++ memcpy(route->tuple[dir].out.h_dest, info.h_dest, ETH_ALEN);
++ route->tuple[dir].out.ifindex = info.outdev->ifindex;
++ route->tuple[dir].out.hw_ifindex = info.hw_outdev->ifindex;
++ route->tuple[dir].xmit_type = info.xmit_type;
++ }
++
++ return 0;
++}
++
++static int
++xt_flowoffload_route_dir(struct nf_flow_route *route, const struct nf_conn *ct,
++ enum ip_conntrack_dir dir,
++ const struct xt_action_param *par, int ifindex,
++ struct net_device **devs)
++{
++ struct dst_entry *dst = NULL;
++ struct flowi fl;
++
++ memset(&fl, 0, sizeof(fl));
++ switch (xt_family(par)) {
++ case NFPROTO_IPV4:
++ fl.u.ip4.daddr = ct->tuplehash[!dir].tuple.src.u3.ip;
++ fl.u.ip4.flowi4_oif = ifindex;
++ break;
++ case NFPROTO_IPV6:
++ fl.u.ip6.saddr = ct->tuplehash[!dir].tuple.dst.u3.in6;
++ fl.u.ip6.daddr = ct->tuplehash[!dir].tuple.src.u3.in6;
++ fl.u.ip6.flowi6_oif = ifindex;
++ break;
++ }
++
++ nf_route(xt_net(par), &dst, &fl, false, xt_family(par));
++ if (!dst)
++ return -ENOENT;
++
++ nf_default_forward_path(route, dst, dir, devs);
++
++ return 0;
++}
++
++static int
++xt_flowoffload_route_nat(struct sk_buff *skb, const struct nf_conn *ct,
++ const struct xt_action_param *par,
++ struct nf_flow_route *route, enum ip_conntrack_dir dir,
++ struct net_device **devs)
++{
++ struct dst_entry *this_dst = skb_dst(skb);
++ struct dst_entry *other_dst = NULL;
++ struct flowi fl;
++
++ memset(&fl, 0, sizeof(fl));
++ switch (xt_family(par)) {
++ case NFPROTO_IPV4:
++ fl.u.ip4.daddr = ct->tuplehash[dir].tuple.src.u3.ip;
++ fl.u.ip4.flowi4_oif = xt_in(par)->ifindex;
++ break;
++ case NFPROTO_IPV6:
++ fl.u.ip6.saddr = ct->tuplehash[!dir].tuple.dst.u3.in6;
++ fl.u.ip6.daddr = ct->tuplehash[dir].tuple.src.u3.in6;
++ fl.u.ip6.flowi6_oif = xt_in(par)->ifindex;
++ break;
++ }
++
++ nf_route(xt_net(par), &other_dst, &fl, false, xt_family(par));
++ if (!other_dst)
++ return -ENOENT;
++
++ nf_default_forward_path(route, this_dst, dir, devs);
++ nf_default_forward_path(route, other_dst, !dir, devs);
++
++ if (route->tuple[dir].xmit_type == FLOW_OFFLOAD_XMIT_NEIGH &&
++ route->tuple[!dir].xmit_type == FLOW_OFFLOAD_XMIT_NEIGH) {
++ if (nf_dev_forward_path(route, ct, dir, devs))
++ return -1;
++ if (nf_dev_forward_path(route, ct, !dir, devs))
++ return -1;
++ }
++
++ return 0;
++}
++
++static int
++xt_flowoffload_route_bridge(struct sk_buff *skb, const struct nf_conn *ct,
++ const struct xt_action_param *par,
++ struct nf_flow_route *route, enum ip_conntrack_dir dir,
++ struct net_device **devs)
++{
++ int ret;
++
++ ret = xt_flowoffload_route_dir(route, ct, dir, par,
++ devs[dir]->ifindex,
++ devs);
++ if (ret)
++ return ret;
++
++ ret = xt_flowoffload_route_dir(route, ct, !dir, par,
++ devs[!dir]->ifindex,
++ devs);
++ if (ret)
++ goto err_route_dir1;
++
++ if (route->tuple[dir].xmit_type == FLOW_OFFLOAD_XMIT_NEIGH &&
++ route->tuple[!dir].xmit_type == FLOW_OFFLOAD_XMIT_NEIGH) {
++ if (nf_dev_forward_path(route, ct, dir, devs) ||
++ nf_dev_forward_path(route, ct, !dir, devs)) {
++ ret = -1;
++ goto err_route_dir2;
++ }
++ }
++
++ return 0;
++
++err_route_dir2:
++ dst_release(route->tuple[!dir].dst);
++err_route_dir1:
++ dst_release(route->tuple[dir].dst);
++ return ret;
++}
++
++static unsigned int
++flowoffload_tg(struct sk_buff *skb, const struct xt_action_param *par)
++{
++ struct xt_flowoffload_table *table;
++ const struct xt_flowoffload_target_info *info = par->targinfo;
++ struct tcphdr _tcph, *tcph = NULL;
++ enum ip_conntrack_info ctinfo;
++ enum ip_conntrack_dir dir;
++ struct nf_flow_route route = {};
++ struct flow_offload *flow = NULL;
++ struct net_device *devs[2] = {};
++ struct nf_conn *ct;
++ struct net *net;
++
++ if (xt_flowoffload_skip(skb, xt_family(par)))
++ return XT_CONTINUE;
++
++ ct = nf_ct_get(skb, &ctinfo);
++ if (ct == NULL)
++ return XT_CONTINUE;
++
++ switch (ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.dst.protonum) {
++ case IPPROTO_TCP:
++ tcph = skb_header_pointer(skb, par->thoff,
++ sizeof(_tcph), &_tcph);
++ if (unlikely(!tcph || tcph->fin || tcph->rst ||
++ !nf_conntrack_tcp_established(ct)))
++ return XT_CONTINUE;
++ break;
++ case IPPROTO_UDP:
++ break;
++ default:
++ return XT_CONTINUE;
++ }
++
++ if (nf_ct_ext_exist(ct, NF_CT_EXT_HELPER) ||
++ ct->status & IPS_SEQ_ADJUST)
++ return XT_CONTINUE;
++
++ if (!nf_ct_is_confirmed(ct))
++ return XT_CONTINUE;
++
++ devs[dir] = xt_out(par);
++ devs[!dir] = xt_in(par);
++
++ if (!devs[dir] || !devs[!dir])
++ return XT_CONTINUE;
++
++ if (test_and_set_bit(IPS_OFFLOAD_BIT, &ct->status))
++ return XT_CONTINUE;
++
++ dir = CTINFO2DIR(ctinfo);
++
++ if (skb->protocol == htons(ETH_P_IPV6))
++ ct->inet6_mode = CT_INET_MODE_IPV6;
++ else
++ ct->inet6_mode = 0;
++
++ if (ct->status & IPS_NAT_MASK) {
++ if (xt_flowoffload_route_nat(skb, ct, par, &route, dir, devs) < 0)
++ goto err_flow_route;
++ } else {
++ if (xt_flowoffload_route_bridge(skb, ct, par, &route, dir, devs) < 0)
++ goto err_flow_route;
++ }
++
++ flow = flow_offload_alloc(ct);
++ if (!flow)
++ goto err_flow_alloc;
++
++ if (flow_offload_route_init(flow, &route) < 0)
++ goto err_flow_add;
++
++ if (tcph) {
++ ct->proto.tcp.seen[0].flags |= IP_CT_TCP_FLAG_BE_LIBERAL;
++ ct->proto.tcp.seen[1].flags |= IP_CT_TCP_FLAG_BE_LIBERAL;
++ }
++
++ table = &flowtable[!!(info->flags & XT_FLOWOFFLOAD_HW)];
++
++ net = read_pnet(&table->ft.net);
++ if (!net)
++ write_pnet(&table->ft.net, xt_net(par));
++
++ if (flow_offload_add(&table->ft, flow) < 0)
++ goto err_flow_add;
++
++ xt_flowoffload_check_device(table, devs[0]);
++ xt_flowoffload_check_device(table, devs[1]);
++
++ if (!(ct->status & IPS_NAT_MASK))
++ dst_release(route.tuple[dir].dst);
++ dst_release(route.tuple[!dir].dst);
++
++ return XT_CONTINUE;
++
++err_flow_add:
++ flow_offload_free(flow);
++err_flow_alloc:
++ if (!(ct->status & IPS_NAT_MASK))
++ dst_release(route.tuple[dir].dst);
++ dst_release(route.tuple[!dir].dst);
++err_flow_route:
++ clear_bit(IPS_OFFLOAD_BIT, &ct->status);
++
++ return XT_CONTINUE;
++}
++
++static int flowoffload_chk(const struct xt_tgchk_param *par)
++{
++ struct xt_flowoffload_target_info *info = par->targinfo;
++
++ if (info->flags & ~XT_FLOWOFFLOAD_MASK)
++ return -EINVAL;
++
++ return 0;
++}
++
++static struct xt_target offload_tg_reg __read_mostly = {
++ .family = NFPROTO_UNSPEC,
++ .name = "FLOWOFFLOAD",
++ .revision = 0,
++ .targetsize = sizeof(struct xt_flowoffload_target_info),
++ .usersize = sizeof(struct xt_flowoffload_target_info),
++ .checkentry = flowoffload_chk,
++ .target = flowoffload_tg,
++ .me = THIS_MODULE,
++};
++
++static int flow_offload_netdev_event(struct notifier_block *this,
++ unsigned long event, void *ptr)
++{
++ struct xt_flowoffload_hook *hook0, *hook1;
++ struct net_device *dev = netdev_notifier_info_to_dev(ptr);
++
++ if (event != NETDEV_UNREGISTER)
++ return NOTIFY_DONE;
++
++ spin_lock_bh(&hooks_lock);
++ hook0 = flow_offload_lookup_hook(&flowtable[0], dev);
++ if (hook0)
++ hlist_del(&hook0->list);
++
++ hook1 = flow_offload_lookup_hook(&flowtable[1], dev);
++ if (hook1)
++ hlist_del(&hook1->list);
++ spin_unlock_bh(&hooks_lock);
++
++ if (hook0) {
++ nf_unregister_net_hook(hook0->net, &hook0->ops);
++ kfree(hook0);
++ }
++
++ if (hook1) {
++ nf_unregister_net_hook(hook1->net, &hook1->ops);
++ kfree(hook1);
++ }
++
++ nf_flow_table_cleanup(dev);
++
++ return NOTIFY_DONE;
++}
++
++static struct notifier_block flow_offload_netdev_notifier = {
++ .notifier_call = flow_offload_netdev_event,
++};
++
++static int nf_flow_rule_route_inet(struct net *net,
++ const struct flow_offload *flow,
++ enum flow_offload_tuple_dir dir,
++ struct nf_flow_rule *flow_rule)
++{
++ const struct flow_offload_tuple *flow_tuple = &flow->tuplehash[dir].tuple;
++ int err;
++
++ switch (flow_tuple->l3proto) {
++ case NFPROTO_IPV4:
++ err = nf_flow_rule_route_ipv4(net, flow, dir, flow_rule);
++ break;
++ case NFPROTO_IPV6:
++ err = nf_flow_rule_route_ipv6(net, flow, dir, flow_rule);
++ break;
++ default:
++ err = -1;
++ break;
++ }
++
++ return err;
++}
++
++static struct nf_flowtable_type flowtable_inet = {
++ .family = NFPROTO_INET,
++ .init = nf_flow_table_init,
++ .setup = nf_flow_table_offload_setup,
++ .action = nf_flow_rule_route_inet,
++ .free = nf_flow_table_free,
++ .hook = xt_flowoffload_net_hook,
++ .owner = THIS_MODULE,
++};
++
++static int init_flowtable(struct xt_flowoffload_table *tbl)
++{
++ INIT_DELAYED_WORK(&tbl->work, xt_flowoffload_hook_work);
++ tbl->ft.type = &flowtable_inet;
++
++ return nf_flow_table_init(&tbl->ft);
++}
++
++static int __init xt_flowoffload_tg_init(void)
++{
++ int ret;
++
++ register_netdevice_notifier(&flow_offload_netdev_notifier);
++
++ ret = init_flowtable(&flowtable[0]);
++ if (ret)
++ return ret;
++
++ ret = init_flowtable(&flowtable[1]);
++ if (ret)
++ goto cleanup;
++
++ flowtable[1].ft.flags = NF_FLOWTABLE_HW_OFFLOAD;
++
++ ret = xt_register_target(&offload_tg_reg);
++ if (ret)
++ goto cleanup2;
++
++ return 0;
++
++cleanup2:
++ nf_flow_table_free(&flowtable[1].ft);
++cleanup:
++ nf_flow_table_free(&flowtable[0].ft);
++ return ret;
++}
++
++static void __exit xt_flowoffload_tg_exit(void)
++{
++ xt_unregister_target(&offload_tg_reg);
++ unregister_netdevice_notifier(&flow_offload_netdev_notifier);
++ nf_flow_table_free(&flowtable[0].ft);
++ nf_flow_table_free(&flowtable[1].ft);
++}
++
++MODULE_LICENSE("GPL");
++module_init(xt_flowoffload_tg_init);
++module_exit(xt_flowoffload_tg_exit);
+--
+2.45.2
+
diff --git a/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-2101-Add-trngv2-driver-support.patch b/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-2101-Add-trngv2-driver-support.patch
index 2169d20..9b11cd0 100644
--- a/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-2101-Add-trngv2-driver-support.patch
+++ b/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-2101-Add-trngv2-driver-support.patch
@@ -164,7 +164,7 @@
+};
+
+static const struct mtk_rng_of_data mt7986_rng_data = {
-+ .rng_version = 1,
++ .rng_version = 2,
+};
+
+static const struct mtk_rng_of_data mt7623_rng_data = {
diff --git a/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-2701-netfilter-nf_flow_table-support-hw-offload-through-v.patch b/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-2701-netfilter-nf_flow_table-support-hw-offload-through-v.patch
new file mode 100644
index 0000000..eb4c823
--- /dev/null
+++ b/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-2701-netfilter-nf_flow_table-support-hw-offload-through-v.patch
@@ -0,0 +1,69 @@
+From: Felix Fietkau <nbd@nbd.name>
+Date: Thu, 15 Mar 2018 20:46:31 +0100
+Subject: [PATCH] netfilter: nf_flow_table: support hw offload through
+ virtual interfaces
+
+There are hardware offload devices that support offloading VLANs and
+PPPoE devices. Additionally, it is useful to be able to offload packets
+routed through bridge interfaces as well.
+Add support for finding the path to the offload device through these
+virtual interfaces, while collecting useful parameters for the offload
+device, like VLAN ID/protocol, PPPoE session and Ethernet MAC address.
+
+Signed-off-by: Felix Fietkau <nbd@nbd.name>
+---
+
+diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
+index 2d1aa35..b60b506 100644
+--- a/include/linux/netdevice.h
++++ b/include/linux/netdevice.h
+@@ -989,6 +989,7 @@ struct dev_ifalias {
+ struct devlink;
+ struct tlsdev_ops;
+
++struct flow_offload_hw_path;
+
+ /*
+ * This structure defines the management hooks for network devices.
+@@ -1222,6 +1223,11 @@ struct tlsdev_ops;
+ * int (*ndo_bridge_dellink)(struct net_device *dev, struct nlmsghdr *nlh,
+ * u16 flags);
+ *
++ * int (*ndo_flow_offload_check)(struct flow_offload_hw_path *path);
++ * For virtual devices like bridges, vlan, and pppoe, fill in the
++ * underlying network device that can be used for offloading connections.
++ * Return an error if offloading is not supported.
++ *
+ * int (*ndo_change_carrier)(struct net_device *dev, bool new_carrier);
+ * Called to change device carrier. Soft-devices (like dummy, team, etc)
+ * which do not represent real hardware may define this to allow their
+@@ -1471,6 +1477,7 @@ struct net_device_ops {
+ int (*ndo_bridge_dellink)(struct net_device *dev,
+ struct nlmsghdr *nlh,
+ u16 flags);
++ int (*ndo_flow_offload_check)(struct flow_offload_hw_path *path);
+ int (*ndo_change_carrier)(struct net_device *dev,
+ bool new_carrier);
+ int (*ndo_get_phys_port_id)(struct net_device *dev,
+diff --git a/include/net/netfilter/nf_flow_table.h b/include/net/netfilter/nf_flow_table.h
+index 7374cb2..e0c0a80 100644
+--- a/include/net/netfilter/nf_flow_table.h
++++ b/include/net/netfilter/nf_flow_table.h
+@@ -179,6 +179,17 @@ struct flow_offload {
+ struct rcu_head rcu_head;
+ };
+
++struct flow_offload_hw_path {
++ struct net_device *dev;
++ u32 flags;
++
++ u8 eth_src[ETH_ALEN];
++ u8 eth_dest[ETH_ALEN];
++ u16 vlan_proto;
++ u16 vlan_id;
++ u16 pppoe_sid;
++};
++
+ #define NF_FLOW_TIMEOUT (30 * HZ)
+ #define nf_flowtable_time_stamp (u32)jiffies
+
diff --git a/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-2702-net-8021q-support-hardware-flow-table-offload.patch b/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-2702-net-8021q-support-hardware-flow-table-offload.patch
new file mode 100644
index 0000000..6fa10d4
--- /dev/null
+++ b/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-2702-net-8021q-support-hardware-flow-table-offload.patch
@@ -0,0 +1,63 @@
+From: Felix Fietkau <nbd@nbd.name>
+Date: Thu, 15 Mar 2018 20:49:58 +0100
+Subject: [PATCH] net: 8021q: support hardware flow table offload
+
+Add the VLAN ID and protocol information
+
+Signed-off-by: Felix Fietkau <nbd@nbd.name>
+---
+
+diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c
+index c373f1d..f45abd5 100644
+--- a/net/8021q/vlan_dev.c
++++ b/net/8021q/vlan_dev.c
+@@ -27,6 +27,11 @@
+ #include <linux/phy.h>
+ #include <net/arp.h>
+
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++#include <linux/netfilter.h>
++#include <net/netfilter/nf_flow_table.h>
++#endif
++
+ #include "vlan.h"
+ #include "vlanproc.h"
+ #include <linux/if_vlan.h>
+@@ -791,6 +796,27 @@ static int vlan_dev_fill_forward_path(struct net_device_path_ctx *ctx,
+ return 0;
+ }
+
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++static int vlan_dev_flow_offload_check(struct flow_offload_hw_path *path)
++{
++ struct net_device *dev = path->dev;
++ struct vlan_dev_priv *vlan = vlan_dev_priv(dev);
++
++ if (path->flags & BIT(DEV_PATH_VLAN))
++ return -EEXIST;
++
++ path->flags |= BIT(DEV_PATH_VLAN);
++ path->vlan_proto = vlan->vlan_proto;
++ path->vlan_id = vlan->vlan_id;
++ path->dev = vlan->real_dev;
++
++ if (vlan->real_dev->netdev_ops->ndo_flow_offload_check)
++ return vlan->real_dev->netdev_ops->ndo_flow_offload_check(path);
++
++ return 0;
++}
++#endif /* CONFIG_NF_FLOW_TABLE */
++
+ static const struct ethtool_ops vlan_ethtool_ops = {
+ .get_link_ksettings = vlan_ethtool_get_link_ksettings,
+ .get_drvinfo = vlan_ethtool_get_drvinfo,
+@@ -830,6 +856,9 @@ static const struct net_device_ops vlan_netdev_ops = {
+ .ndo_fix_features = vlan_dev_fix_features,
+ .ndo_get_iflink = vlan_dev_get_iflink,
+ .ndo_fill_forward_path = vlan_dev_fill_forward_path,
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++ .ndo_flow_offload_check = vlan_dev_flow_offload_check,
++#endif
+ };
+
+ static void vlan_dev_free(struct net_device *dev)
diff --git a/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-2703-net-bridge-support-hardware-flow-table-offload.patch b/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-2703-net-bridge-support-hardware-flow-table-offload.patch
new file mode 100644
index 0000000..6ec4d74
--- /dev/null
+++ b/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-2703-net-bridge-support-hardware-flow-table-offload.patch
@@ -0,0 +1,63 @@
+From: Felix Fietkau <nbd@nbd.name>
+Date: Thu, 15 Mar 2018 20:50:37 +0100
+Subject: [PATCH] net: bridge: support hardware flow table offload
+
+Look up the real device and pass it on
+
+Signed-off-by: Felix Fietkau <nbd@nbd.name>
+---
+
+diff --git a/net/bridge/br_device.c b/net/bridge/br_device.c
+index 03934dd..094b2b1 100644
+--- a/net/bridge/br_device.c
++++ b/net/bridge/br_device.c
+@@ -14,6 +14,10 @@
+ #include <linux/ethtool.h>
+ #include <linux/list.h>
+ #include <linux/netfilter_bridge.h>
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++#include <linux/netfilter.h>
++#include <net/netfilter/nf_flow_table.h>
++#endif
+
+ #include <linux/uaccess.h>
+ #include "br_private.h"
+@@ -440,6 +444,28 @@ static const struct ethtool_ops br_ethtool_ops = {
+ .get_link = ethtool_op_get_link,
+ };
+
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++static int br_flow_offload_check(struct flow_offload_hw_path *path)
++{
++ struct net_device *dev = path->dev;
++ struct net_bridge *br = netdev_priv(dev);
++ struct net_bridge_fdb_entry *dst;
++
++ if (!(path->flags & BIT(DEV_PATH_ETHERNET)))
++ return -EINVAL;
++
++ dst = br_fdb_find_rcu(br, path->eth_dest, path->vlan_id);
++ if (!dst || !dst->dst)
++ return -ENOENT;
++
++ path->dev = dst->dst->dev;
++ if (path->dev->netdev_ops->ndo_flow_offload_check)
++ return path->dev->netdev_ops->ndo_flow_offload_check(path);
++
++ return 0;
++}
++#endif /* CONFIG_NF_FLOW_TABLE */
++
+ static const struct net_device_ops br_netdev_ops = {
+ .ndo_open = br_dev_open,
+ .ndo_stop = br_dev_stop,
+@@ -469,6 +495,9 @@ static const struct net_device_ops br_netdev_ops = {
+ .ndo_bridge_dellink = br_dellink,
+ .ndo_features_check = passthru_features_check,
+ .ndo_fill_forward_path = br_fill_forward_path,
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++ .ndo_flow_offload_check = br_flow_offload_check,
++#endif
+ };
+
+ static struct device_type br_type = {
diff --git a/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-2704-net-pppoe-support-hardware-flow-table-offload.patch b/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-2704-net-pppoe-support-hardware-flow-table-offload.patch
new file mode 100644
index 0000000..d287e52
--- /dev/null
+++ b/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-2704-net-pppoe-support-hardware-flow-table-offload.patch
@@ -0,0 +1,132 @@
+From: Felix Fietkau <nbd@nbd.name>
+Date: Thu, 15 Mar 2018 21:15:00 +0100
+Subject: [PATCH] net: pppoe: support hardware flow table offload
+
+Pass on the PPPoE session ID and the remote MAC address
+
+Signed-off-by: Felix Fietkau <nbd@nbd.name>
+---
+
+diff --git a/drivers/net/ppp/ppp_generic.c b/drivers/net/ppp/ppp_generic.c
+index ec2fbd1..f361b52 100644
+--- a/drivers/net/ppp/ppp_generic.c
++++ b/drivers/net/ppp/ppp_generic.c
+@@ -53,6 +53,11 @@
+ #include <net/net_namespace.h>
+ #include <net/netns/generic.h>
+
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++#include <linux/netfilter.h>
++#include <net/netfilter/nf_flow_table.h>
++#endif
++
+ #define PPP_VERSION "2.4.2"
+
+ /*
+@@ -1416,6 +1421,28 @@ static int ppp_fill_forward_path(struct net_device_path_ctx *ctx,
+ return chan->ops->fill_forward_path(ctx, path, chan);
+ }
+
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++static int ppp_flow_offload_check(struct flow_offload_hw_path *path)
++{
++ struct ppp *ppp = netdev_priv(path->dev);
++ struct ppp_channel *chan;
++ struct channel *pch;
++
++ if (ppp->flags & SC_MULTILINK)
++ return -EOPNOTSUPP;
++
++ if (list_empty(&ppp->channels))
++ return -ENODEV;
++
++ pch = list_first_entry(&ppp->channels, struct channel, clist);
++ chan = pch->chan;
++ if (!chan->ops->flow_offload_check)
++ return -EOPNOTSUPP;
++
++ return chan->ops->flow_offload_check(chan, path);
++}
++#endif /* CONFIG_NF_FLOW_TABLE */
++
+ static const struct net_device_ops ppp_netdev_ops = {
+ .ndo_init = ppp_dev_init,
+ .ndo_uninit = ppp_dev_uninit,
+@@ -1423,6 +1450,9 @@ static const struct net_device_ops ppp_netdev_ops = {
+ .ndo_do_ioctl = ppp_net_ioctl,
+ .ndo_get_stats64 = ppp_get_stats64,
+ .ndo_fill_forward_path = ppp_fill_forward_path,
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++ .ndo_flow_offload_check = ppp_flow_offload_check,
++#endif
+ };
+
+ static struct device_type ppp_type = {
+diff --git a/drivers/net/ppp/pppoe.c b/drivers/net/ppp/pppoe.c
+index 7a8c246..1018464 100644
+--- a/drivers/net/ppp/pppoe.c
++++ b/drivers/net/ppp/pppoe.c
+@@ -73,6 +73,11 @@
+ #include <linux/proc_fs.h>
+ #include <linux/seq_file.h>
+
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++#include <linux/netfilter.h>
++#include <net/netfilter/nf_flow_table.h>
++#endif
++
+ #include <linux/nsproxy.h>
+ #include <net/net_namespace.h>
+ #include <net/netns/generic.h>
+@@ -997,9 +1002,37 @@ static int pppoe_fill_forward_path(struct net_device_path_ctx *ctx,
+ return 0;
+ }
+
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++static int pppoe_flow_offload_check(struct ppp_channel *chan,
++ struct flow_offload_hw_path *path)
++{
++ struct sock *sk = (struct sock *)chan->private;
++ struct pppox_sock *po = pppox_sk(sk);
++ struct net_device *dev = po->pppoe_dev;
++
++ if (sock_flag(sk, SOCK_DEAD) ||
++ !(sk->sk_state & PPPOX_CONNECTED) || !dev)
++ return -ENODEV;
++
++ path->dev = po->pppoe_dev;
++ path->flags |= BIT(DEV_PATH_PPPOE);
++ memcpy(path->eth_src, po->pppoe_dev->dev_addr, ETH_ALEN);
++ memcpy(path->eth_dest, po->pppoe_pa.remote, ETH_ALEN);
++ path->pppoe_sid = be16_to_cpu(po->num);
++
++ if (path->dev->netdev_ops->ndo_flow_offload_check)
++ return path->dev->netdev_ops->ndo_flow_offload_check(path);
++
++ return 0;
++}
++#endif /* CONFIG_NF_FLOW_TABLE */
++
+ static const struct ppp_channel_ops pppoe_chan_ops = {
+ .start_xmit = pppoe_xmit,
+ .fill_forward_path = pppoe_fill_forward_path,
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++ .flow_offload_check = pppoe_flow_offload_check,
++#endif
+ };
+
+ static int pppoe_recvmsg(struct socket *sock, struct msghdr *m,
+diff --git a/include/linux/ppp_channel.h b/include/linux/ppp_channel.h
+index 91f9a92..4a1729b 100644
+--- a/include/linux/ppp_channel.h
++++ b/include/linux/ppp_channel.h
+@@ -31,6 +31,9 @@ struct ppp_channel_ops {
+ int (*fill_forward_path)(struct net_device_path_ctx *,
+ struct net_device_path *,
+ const struct ppp_channel *);
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++ int (*flow_offload_check)(struct ppp_channel *, struct flow_offload_hw_path *);
++#endif
+ };
+
+ struct ppp_channel {
diff --git a/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-2705-net-dsa-support-hardware-flow-table-offload.patch b/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-2705-net-dsa-support-hardware-flow-table-offload.patch
new file mode 100644
index 0000000..54fd0a4
--- /dev/null
+++ b/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-2705-net-dsa-support-hardware-flow-table-offload.patch
@@ -0,0 +1,74 @@
+From: Felix Fietkau <nbd@nbd.name>
+Date: Thu, 17 Sep 2020 18:41:23 +0200
+Subject: [PATCH] net: dsa: support hardware flow table offload
+
+Look up the master device and the port id
+
+Signed-off-by: Felix Fietkau <nbd@nbd.name>
+---
+
+diff --git a/include/net/netfilter/nf_flow_table.h b/include/net/netfilter/nf_flow_table.h
+index e0c0a80..a1b4ab5 100644
+--- a/include/net/netfilter/nf_flow_table.h
++++ b/include/net/netfilter/nf_flow_table.h
+@@ -188,6 +188,7 @@ struct flow_offload_hw_path {
+ u16 vlan_proto;
+ u16 vlan_id;
+ u16 pppoe_sid;
++ u16 dsa_port;
+ };
+
+ #define NF_FLOW_TIMEOUT (30 * HZ)
+diff --git a/net/dsa/slave.c b/net/dsa/slave.c
+index 2ea9ec1..eab4e4a 100644
+--- a/net/dsa/slave.c
++++ b/net/dsa/slave.c
+@@ -19,6 +19,10 @@
+ #include <linux/if_bridge.h>
+ #include <linux/netpoll.h>
+ #include <linux/ptp_classify.h>
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++#include <linux/netfilter.h>
++#include <net/netfilter/nf_flow_table.h>
++#endif
+
+ #include "dsa_priv.h"
+
+@@ -1257,6 +1261,27 @@ static int dsa_slave_fill_forward_path(struct net_device_path_ctx *ctx,
+ return 0;
+ }
+
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++static int dsa_flow_offload_check(struct flow_offload_hw_path *path)
++{
++ struct net_device *dev = path->dev;
++ struct dsa_port *dp;
++
++ if (!(path->flags & BIT(DEV_PATH_ETHERNET)))
++ return -EINVAL;
++
++ dp = dsa_slave_to_port(dev);
++ path->dsa_port = dp->index;
++ path->dev = dsa_slave_to_master(dev);
++ path->flags |= BIT(DEV_PATH_DSA);
++
++ if (path->dev->netdev_ops->ndo_flow_offload_check)
++ return path->dev->netdev_ops->ndo_flow_offload_check(path);
++
++ return 0;
++}
++#endif /* CONFIG_NF_FLOW_TABLE */
++
+ static const struct net_device_ops dsa_slave_netdev_ops = {
+ .ndo_open = dsa_slave_open,
+ .ndo_stop = dsa_slave_close,
+@@ -1282,6 +1307,9 @@ static const struct net_device_ops dsa_slave_netdev_ops = {
+ .ndo_vlan_rx_kill_vid = dsa_slave_vlan_rx_kill_vid,
+ .ndo_get_devlink_port = dsa_slave_get_devlink_port,
+ .ndo_fill_forward_path = dsa_slave_fill_forward_path,
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++ .ndo_flow_offload_check = dsa_flow_offload_check,
++#endif
+ };
+
+ static struct device_type dsa_type = {
diff --git a/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-2708-mtkhnat-add-support-for-virtual-interface-acceleration.patch b/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-2708-mtkhnat-add-support-for-virtual-interface-acceleration.patch
new file mode 100644
index 0000000..8cc6f3d
--- /dev/null
+++ b/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-2708-mtkhnat-add-support-for-virtual-interface-acceleration.patch
@@ -0,0 +1,152 @@
+From fd6e50fdeb1d943b889a5aa093790a798ae598d3 Mon Sep 17 00:00:00 2001
+From: Sam Shih <sam.shih@mediatek.com>
+Date: Fri, 2 Jun 2023 13:06:29 +0800
+Subject: [PATCH]
+ [networking][999-2708-mtkhnat-add-support-for-virtual-interface-acceleration.patch]
+
+---
+ include/linux/netdevice.h | 1 +
+ include/net/netfilter/nf_flow_table.h | 1 +
+ net/8021q/vlan_dev.c | 1 +
+ net/ipv6/ip6_tunnel.c | 24 ++++++++++++++++++++++++
+ net/ipv6/sit.c | 24 ++++++++++++++++++++++++
+ 5 files changed, 51 insertions(+)
+
+diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
+index b60b506..c30952f 100644
+--- a/include/linux/netdevice.h
++++ b/include/linux/netdevice.h
+@@ -849,6 +849,8 @@ enum net_device_path_type {
+ DEV_PATH_BRIDGE,
+ DEV_PATH_PPPOE,
+ DEV_PATH_DSA,
++ DEV_PATH_DSLITE,
++ DEV_PATH_6RD,
+ };
+
+ struct net_device_path {
+diff --git a/include/net/netfilter/nf_flow_table.h b/include/net/netfilter/nf_flow_table.h
+index a1b4ab5..7a2945e 100644
+--- a/include/net/netfilter/nf_flow_table.h
++++ b/include/net/netfilter/nf_flow_table.h
+@@ -181,6 +181,7 @@ struct flow_offload {
+
+ struct flow_offload_hw_path {
+ struct net_device *dev;
++ struct net_device *virt_dev;
+ u32 flags;
+
+ u8 eth_src[ETH_ALEN];
+diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c
+index f45abd5..db5867d 100644
+--- a/net/8021q/vlan_dev.c
++++ b/net/8021q/vlan_dev.c
+@@ -808,6 +808,7 @@ static int vlan_dev_flow_offload_check(struct flow_offload_hw_path *path)
+ path->flags |= BIT(DEV_PATH_VLAN);
+ path->vlan_proto = vlan->vlan_proto;
+ path->vlan_id = vlan->vlan_id;
++ path->virt_dev = dev;
+ path->dev = vlan->real_dev;
+
+ if (vlan->real_dev->netdev_ops->ndo_flow_offload_check)
+diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c
+index 2e8a528..0bf25b0 100644
+--- a/net/ipv6/ip6_tunnel.c
++++ b/net/ipv6/ip6_tunnel.c
+@@ -57,6 +57,11 @@
+ #include <net/netns/generic.h>
+ #include <net/dst_metadata.h>
+
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++#include <linux/netfilter.h>
++#include <net/netfilter/nf_flow_table.h>
++#endif
++
+ MODULE_AUTHOR("Ville Nuorvala");
+ MODULE_DESCRIPTION("IPv6 tunneling device");
+ MODULE_LICENSE("GPL");
+@@ -1889,6 +1894,22 @@ int ip6_tnl_get_iflink(const struct net_device *dev)
+ }
+ EXPORT_SYMBOL(ip6_tnl_get_iflink);
+
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++static int ipip6_dev_flow_offload_check(struct flow_offload_hw_path *path)
++{
++ struct net_device *dev = path->dev;
++ struct ip6_tnl *tnl = netdev_priv(dev);
++
++ if (path->flags & BIT(DEV_PATH_DSLITE))
++ return -EEXIST;
++
++ path->flags |= BIT(DEV_PATH_DSLITE);
++ path->dev = tnl->dev;
++
++ return 0;
++}
++#endif /* CONFIG_NF_FLOW_TABLE */
++
+ int ip6_tnl_encap_add_ops(const struct ip6_tnl_encap_ops *ops,
+ unsigned int num)
+ {
+@@ -1950,6 +1971,9 @@ static const struct net_device_ops ip6_tnl_netdev_ops = {
+ .ndo_change_mtu = ip6_tnl_change_mtu,
+ .ndo_get_stats = ip6_get_stats,
+ .ndo_get_iflink = ip6_tnl_get_iflink,
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++ .ndo_flow_offload_check = ipip6_dev_flow_offload_check,
++#endif
+ };
+
+ #define IPXIPX_FEATURES (NETIF_F_SG | \
+diff --git a/net/ipv6/sit.c b/net/ipv6/sit.c
+index 8d704ea..c50bbef 100644
+--- a/net/ipv6/sit.c
++++ b/net/ipv6/sit.c
+@@ -52,6 +52,11 @@
+ #include <net/net_namespace.h>
+ #include <net/netns/generic.h>
+
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++#include <linux/netfilter.h>
++#include <net/netfilter/nf_flow_table.h>
++#endif
++
+ /*
+ This version of net/ipv6/sit.c is cloned of net/ipv4/ip_gre.c
+
+@@ -1344,6 +1349,22 @@ ipip6_tunnel_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
+ return err;
+ }
+
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++static int ipip6_dev_flow_offload_check(struct flow_offload_hw_path *path)
++{
++ struct net_device *dev = path->dev;
++ struct ip_tunnel *tnl = netdev_priv(dev);
++
++ if (path->flags & BIT(DEV_PATH_6RD))
++ return -EEXIST;
++
++ path->flags |= BIT(DEV_PATH_6RD);
++ path->dev = tnl->dev;
++
++ return 0;
++}
++#endif /* CONFIG_NF_FLOW_TABLE */
++
+ static const struct net_device_ops ipip6_netdev_ops = {
+ .ndo_init = ipip6_tunnel_init,
+ .ndo_uninit = ipip6_tunnel_uninit,
+@@ -1351,6 +1372,9 @@ static const struct net_device_ops ipip6_netdev_ops = {
+ .ndo_do_ioctl = ipip6_tunnel_ioctl,
+ .ndo_get_stats64 = ip_tunnel_get_stats64,
+ .ndo_get_iflink = ip_tunnel_get_iflink,
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++ .ndo_flow_offload_check = ipip6_dev_flow_offload_check,
++#endif
+ };
+
+ static void ipip6_dev_free(struct net_device *dev)
+--
+2.34.1
+
diff --git a/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-2726-mtkhnat-tnl-interface-offload-check.patch b/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-2726-mtkhnat-tnl-interface-offload-check.patch
new file mode 100644
index 0000000..5f8bf62
--- /dev/null
+++ b/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/999-2726-mtkhnat-tnl-interface-offload-check.patch
@@ -0,0 +1,66 @@
+diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
+index c30952f..6c0860b 100644
+--- a/include/linux/netdevice.h
++++ b/include/linux/netdevice.h
+@@ -851,6 +851,7 @@ enum net_device_path_type {
+ DEV_PATH_DSA,
+ DEV_PATH_DSLITE,
+ DEV_PATH_6RD,
++ DEV_PATH_TNL,
+ };
+
+ struct net_device_path {
+diff --git a/net/l2tp/l2tp_ppp.c b/net/l2tp/l2tp_ppp.c
+index 7d3c782..d7a5a9a 100644
+--- a/net/l2tp/l2tp_ppp.c
++++ b/net/l2tp/l2tp_ppp.c
+@@ -89,6 +89,7 @@
+ #include <linux/nsproxy.h>
+ #include <net/net_namespace.h>
+ #include <net/netns/generic.h>
++#include <net/netfilter/nf_flow_table.h>
+ #include <net/ip.h>
+ #include <net/udp.h>
+ #include <net/inet_common.h>
+@@ -124,9 +125,14 @@ struct pppol2tp_session {
+ };
+
+ static int pppol2tp_xmit(struct ppp_channel *chan, struct sk_buff *skb);
++static int l2tp_ppp_flow_offload_check(struct ppp_channel *chan,
++ struct flow_offload_hw_path *path);
+
+ static const struct ppp_channel_ops pppol2tp_chan_ops = {
+ .start_xmit = pppol2tp_xmit,
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++ .flow_offload_check = l2tp_ppp_flow_offload_check,
++#endif /* IS_ENABLED(CONFIG_NF_FLOW_TABLE) */
+ };
+
+ static const struct proto_ops pppol2tp_ops;
+@@ -335,6 +341,26 @@ static int pppol2tp_sendmsg(struct socket *sock, struct msghdr *m,
+ return error;
+ }
+
++#if IS_ENABLED(CONFIG_NF_FLOW_TABLE)
++static int l2tp_ppp_flow_offload_check(struct ppp_channel *chan,
++ struct flow_offload_hw_path *path)
++{
++ struct sock *sk = (struct sock *)chan->private;
++ struct l2tp_session *session;
++
++ if (path->flags & BIT(DEV_PATH_TNL))
++ return -EEXIST;
++
++ session = pppol2tp_sock_to_session(sk);
++ if (!session)
++ return -EINVAL;
++
++ path->flags |= BIT(DEV_PATH_TNL);
++
++ return 0;
++}
++#endif /* IS_ENABLED(CONFIG_NF_FLOW_TABLE) */
++
+ /* Transmit function called by generic PPP driver. Sends PPP frame
+ * over PPPoL2TP socket.
+ *
diff --git a/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/patches-5.4.inc b/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/patches-5.4.inc
index 47dfb49..a4ede55 100644
--- a/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/patches-5.4.inc
+++ b/recipes-kernel/linux/linux-mediatek-5.4/mediatek/patches-5.4/patches-5.4.inc
@@ -57,6 +57,7 @@
file://999-1715-v6.2-net-dsa-add-set-queue-mapping.patch \
file://999-1716-v6.6-net-phy-add-phylink-pcs_enable-and-pcs_disable.patch;apply=no \
file://999-1717-v5.12-net-phy-sfp-add-debugfs-support.patch \
+ file://999-1718-v5.15-net-netfilter-add-nf-hw-offload.patch \
file://999-1719-v6.2-net-ptp-introduce-adjust-by-scaled-ppm.patch \
file://999-1750-v5.18-net-macsec-get-ready-to-backport-from-5-18.patch \
file://999-1751-01-v5.18-net-macsec-move-some-definitions-in-a-dedicated-header.patch \
@@ -188,10 +189,16 @@
file://999-2621-xHCI-MT7986-USB-2.0-USBIF-compliance-toolkit.patch \
file://999-2622-usb-add-embedded-Host-feature-support.patch \
file://999-2700-netfilter_optional_tcp_window_check.patch \
+ file://999-2701-netfilter-nf_flow_table-support-hw-offload-through-v.patch \
+ file://999-2702-net-8021q-support-hardware-flow-table-offload.patch \
+ file://999-2703-net-bridge-support-hardware-flow-table-offload.patch \
file://999-2704-en8811h-2p5gphy-support.patch \
+ file://999-2704-net-pppoe-support-hardware-flow-table-offload.patch \
file://999-2705-hwnat_Kconfig_Makefile.patch \
+ file://999-2705-net-dsa-support-hardware-flow-table-offload.patch \
file://999-2706-crypto-add-eip197-inside-secure-support.patch \
file://999-2707-mtkhnat-ipv6-fix-pskb-expand-head-limitation.patch \
+ file://999-2708-mtkhnat-add-support-for-virtual-interface-acceleration.patch \
file://999-2709-fix-race-inside-napi-enable.patch \
file://999-2710-net-make-napi-disable-symmetric-with-enable.patch \
file://999-2711-net-fix-premature-exit-from-napi-state-polling-in-napi-disable-v2.patch \
@@ -207,6 +214,7 @@
file://999-2723-dt-bindings-phy-Add-PHY_TYPE_XPCS-definition.patch \
file://999-2724-dt-bindings-phy-Add-DT-bindings-for-Xilinx-ZynqMP-PS.patch \
file://999-2725-iwconfig-wireless-rate-fix.patch;apply=no \
+ file://999-2726-mtkhnat-tnl-interface-offload-check.patch \
file://999-2727-net-phy-sfp-add-debug-info.patch.patch \
file://999-2728-net-phy-aquantia-add-mib-read.patch \
file://999-2730-net-phy-sfp-change-shared-mod-def0.patch \