diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c
index 94cacae76aca4..4c11b810a4477 100644
--- a/net/ipv4/ip_output.c
+++ b/net/ipv4/ip_output.c
@@ -1090,7 +1090,8 @@ static int __ip_append_data(struct sock *sk,
 		length -= copy;
 	}
 
-	refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc);
+	if (wmem_alloc_delta)
+		refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc);
 	return 0;
 
 error_efault:
diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c
index 66a768b7b8fb7..b8ee50e94af38 100644
--- a/net/ipv6/ip6_output.c
+++ b/net/ipv6/ip6_output.c
@@ -1545,7 +1545,8 @@ static int __ip6_append_data(struct sock *sk,
 		length -= copy;
 	}
 
-	refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc);
+	if (wmem_alloc_delta)
+		refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc);
 	return 0;
 
 error_efault: