Merge pull request #76 from apernet/fix-enobufs

fix: engine exit with "netlink receive: recvmsg: no buffer space available" when too many packets hit NFQUEUE

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/stretchr/testify v1.8.4
 	go.uber.org/zap v1.26.0
 	golang.org/x/crypto v0.19.0
+	golang.org/x/sys v0.17.0
 	google.golang.org/protobuf v1.31.0
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -43,7 +44,6 @@ require (
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
 	golang.org/x/net v0.19.0 // indirect
 	golang.org/x/sync v0.5.0 // indirect
-	golang.org/x/sys v0.17.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 )

io/nfqueue.go

@@ -12,6 +12,7 @@ import (
 	"github.com/coreos/go-iptables/iptables"
 	"github.com/florianl/go-nfqueue"
 	"github.com/mdlayher/netlink"
+	"golang.org/x/sys/unix"
 )
 
 const (
@@ -138,9 +139,10 @@ func NewNFQueuePacketIO(config NFQueuePacketIOConfig) (PacketIO, error) {
 func (n *nfqueuePacketIO) Register(ctx context.Context, cb PacketCallback) error {
 	err := n.n.RegisterWithErrorFunc(ctx,
 		func(a nfqueue.Attribute) int {
-			if a.PacketID == nil || a.Ct == nil || a.Payload == nil || len(*a.Payload) < 20 {
-				// Invalid packet, ignore
-				// 20 is the minimum possible size of an IP packet
+			if ok, verdict := n.packetAttributeSanityCheck(a); !ok {
+				if a.PacketID != nil {
+					_ = n.n.SetVerdict(*a.PacketID, verdict)
+				}
 				return 0
 			}
 			p := &nfqueuePacket{
@@ -151,6 +153,12 @@ func (n *nfqueuePacketIO) Register(ctx context.Context, cb PacketCallback) error
 			return okBoolToInt(cb(p, nil))
 		},
 		func(e error) int {
+			if opErr := (*netlink.OpError)(nil); errors.As(e, &opErr) {
+				if errors.Is(opErr.Err, unix.ENOBUFS) {
+					// Kernel buffer temporarily full, ignore
+					return 0
+				}
+			}
 			return okBoolToInt(cb(nil, e))
 		})
 	if err != nil {
@@ -170,6 +178,25 @@ func (n *nfqueuePacketIO) Register(ctx context.Context, cb PacketCallback) error
 	return nil
 }
 
+func (n *nfqueuePacketIO) packetAttributeSanityCheck(a nfqueue.Attribute) (ok bool, verdict int) {
+	if a.PacketID == nil {
+		// Re-inject to NFQUEUE is actually not possible in this condition
+		return false, -1
+	}
+	if a.Payload == nil || len(*a.Payload) < 20 {
+		// 20 is the minimum possible size of an IP packet
+		return false, nfqueue.NfDrop
+	}
+	if a.Ct == nil {
+		// Multicast packets may not have a conntrack, but only appear in local mode
+		if n.local {
+			return false, nfqueue.NfAccept
+		}
+		return false, nfqueue.NfDrop
+	}
+	return true, -1
+}
+
 func (n *nfqueuePacketIO) SetVerdict(p Packet, v Verdict, newPacket []byte) error {
 	nP, ok := p.(*nfqueuePacket)
 	if !ok {