--- a/net/xfrm/xfrm_output.c
+++ b/net/xfrm/xfrm_output.c
@@ -410,7 +410,7 @@ static int xfrm_output_one(struct sk_buf
 	struct xfrm_state *x = dst->xfrm;
 	struct net *net = xs_net(x);
 
-	if (err <= 0 || x->xso.type == XFRM_DEV_OFFLOAD_PACKET)
+	if (err <= 0)
 		goto resume;
 
 	do {
@@ -570,12 +570,10 @@ int xfrm_output(struct sock *sk, struct
 
 	if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET) {
 		if (!xfrm_dev_offload_ok(skb, x)) {
-			XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR);
-			kfree_skb(skb);
-			return -EHOSTUNREACH;
+			secpath_reset(skb);
+			goto sw_path;
 		}
-
-		return xfrm_output_resume(skb, 0);
+		return 0;
 	}
 
 	secpath_reset(skb);
@@ -606,6 +604,7 @@ int xfrm_output(struct sock *sk, struct
 		if (x->xso.dev && x->xso.dev->features & NETIF_F_HW_ESP_TX_CSUM)
 			goto out;
 	} else {
+sw_path:
 		if (skb_is_gso(skb))
 			return xfrm_output_gso(net, sk, skb);
 	}
--- a/net/xfrm/xfrm_policy.c
+++ b/net/xfrm/xfrm_policy.c
@@ -3703,6 +3703,10 @@ int __xfrm_policy_check(struct sock *sk,
 		}
 	}
 #endif
+	/* Inbound HW offload packets, pass the check directly */
+	if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET &&
+	    (pol->xdo.dir == XFRM_DEV_OFFLOAD_IN || pol->xdo.dir == XFRM_DEV_OFFLOAD_FWD))
+		return 1;
 
 	if (pol->action == XFRM_POLICY_ALLOW) {
 		static struct sec_path dummy;
@@ -3712,6 +3716,14 @@ int __xfrm_policy_check(struct sock *sk,
 		int ti = 0;
 		int i, k;
 
+		/* Strongswan install FWD policy for inbound HW offload
+		 * packets. But cannot find corresponding packet offload
+		 * state here and will be drop. So, we bypass following
+		 * check for FWD policy with acction allow.
+		 */
+		if (dir == XFRM_POLICY_FWD)
+			return 1;
+
 		sp = skb_sec_path(skb);
 		if (!sp)
 			sp = &dummy;