Merge branch 'ynl-small-recv'

Jakub Kicinski says:

====================
tools: ynl: add --dbg-small-recv for easier kernel testing

When testing netlink dumps I usually hack some user space up
to constrain its user space buffer size (iproute2, ethtool or ynl).
Netlink will try to fill the messages up, so since these apps use
large buffers by default, the dumps are rarely fragmented.

I was hoping to figure out a way to create a selftest for dump
testing, but so far I have no idea how to do that in a useful
and generic way.

Until someone does that, make manual dump testing easier with YNL.
Create a special option for limiting the buffer size, so I don't
have to make the same edits each time, and maybe others will benefit,
too :)

Example:

  $ ./cli.py [...] --dbg-small-recv >/dev/null
  Recv: read 3712 bytes, 29 messages
     nl_len = 128 (112) nl_flags = 0x0 nl_type = 19
    [...]
     nl_len = 128 (112) nl_flags = 0x0 nl_type = 19
  Recv: read 3968 bytes, 31 messages
     nl_len = 128 (112) nl_flags = 0x0 nl_type = 19
    [...]
     nl_len = 128 (112) nl_flags = 0x0 nl_type = 19
  Recv: read 532 bytes, 5 messages
     nl_len = 128 (112) nl_flags = 0x0 nl_type = 19
    [...]
     nl_len = 128 (112) nl_flags = 0x0 nl_type = 19
     nl_len = 20 (4) nl_flags = 0x2 nl_type = 3

Now let's make the DONE not fit in the last message:

  $ ./cli.py [...] --dbg-small-recv 4499 >/dev/null
  Recv: read 3712 bytes, 29 messages
     nl_len = 128 (112) nl_flags = 0x0 nl_type = 19
    [...]
     nl_len = 128 (112) nl_flags = 0x0 nl_type = 19
  Recv: read 4480 bytes, 35 messages
     nl_len = 128 (112) nl_flags = 0x0 nl_type = 19
    [...]
     nl_len = 128 (112) nl_flags = 0x0 nl_type = 19
  Recv: read 20 bytes, 1 messages
     nl_len = 20 (4) nl_flags = 0x2 nl_type = 3

A real test would also have to check the messages are complete
and not duplicated. That part has to be done manually right now.

Note that the first message is always conservatively sized by the kernel.
Still, I think this is good enough to be useful.

v2:
 - patch 2:
   - move the recv_size setting up
   - change the default to 0 so that cli.py doesn't have to worry
     what the "unset" value is
v1: https://lore.kernel.org/all/20240301230542.116823-1-kuba@kernel.org/
====================

Signed-off-by: David S. Miller <davem@davemloft.net>

tools/net/ynl/cli.py

@@ -38,6 +38,8 @@ def main():
                         const=Netlink.NLM_F_APPEND)
     parser.add_argument('--process-unknown', action=argparse.BooleanOptionalAction)
     parser.add_argument('--output-json', action='store_true')
+    parser.add_argument('--dbg-small-recv', default=0, const=4000,
+                        action='store', nargs='?', type=int)
     args = parser.parse_args()
 
     def output(msg):
@@ -53,7 +55,10 @@ def output(msg):
     if args.json_text:
         attrs = json.loads(args.json_text)
 
-    ynl = YnlFamily(args.spec, args.schema, args.process_unknown)
+    ynl = YnlFamily(args.spec, args.schema, args.process_unknown,
+                    recv_size=args.dbg_small_recv)
+    if args.dbg_small_recv:
+        ynl.set_recv_dbg(True)
 
     if args.ntf:
         ynl.ntf_subscribe(args.ntf)

tools/net/ynl/lib/ynl.py

@@ -7,6 +7,7 @@
 import socket
 import struct
 from struct import Struct
+import sys
 import yaml
 import ipaddress
 import uuid
@@ -84,6 +85,10 @@ def __str__(self):
     return f"Netlink error: {os.strerror(-self.nl_msg.error)}\n{self.nl_msg}"
 
 
+class ConfigError(Exception):
+    pass
+
+
 class NlAttr:
     ScalarFormat = namedtuple('ScalarFormat', ['native', 'big', 'little'])
     type_formats = {
@@ -213,11 +218,11 @@ def cmd(self):
         return self.nl_type
 
     def __repr__(self):
-        msg = f"nl_len = {self.nl_len} ({len(self.raw)}) nl_flags = 0x{self.nl_flags:x} nl_type = {self.nl_type}\n"
+        msg = f"nl_len = {self.nl_len} ({len(self.raw)}) nl_flags = 0x{self.nl_flags:x} nl_type = {self.nl_type}"
         if self.error:
-            msg += '\terror: ' + str(self.error)
+            msg += '\n\terror: ' + str(self.error)
         if self.extack:
-            msg += '\textack: ' + repr(self.extack)
+            msg += '\n\textack: ' + repr(self.extack)
         return msg
 
 
@@ -400,7 +405,8 @@ def lookup(self, name):
 
 
 class YnlFamily(SpecFamily):
-    def __init__(self, def_path, schema=None, process_unknown=False):
+    def __init__(self, def_path, schema=None, process_unknown=False,
+                 recv_size=0):
         super().__init__(def_path, schema)
 
         self.include_raw = False
@@ -415,6 +421,17 @@ def __init__(self, def_path, schema=None, process_unknown=False):
         except KeyError:
             raise Exception(f"Family '{self.yaml['name']}' not supported by the kernel")
 
+        self._recv_dbg = False
+        # Note that netlink will use conservative (min) message size for
+        # the first dump recv() on the socket, our setting will only matter
+        # from the second recv() on.
+        self._recv_size = recv_size if recv_size else 131072
+        # Netlink will always allocate at least PAGE_SIZE - sizeof(skb_shinfo)
+        # for a message, so smaller receive sizes will lead to truncation.
+        # Note that the min size for other families may be larger than 4k!
+        if self._recv_size < 4000:
+            raise ConfigError()
+
         self.sock = socket.socket(socket.AF_NETLINK, socket.SOCK_RAW, self.nlproto.proto_num)
         self.sock.setsockopt(Netlink.SOL_NETLINK, Netlink.NETLINK_CAP_ACK, 1)
         self.sock.setsockopt(Netlink.SOL_NETLINK, Netlink.NETLINK_EXT_ACK, 1)
@@ -438,6 +455,17 @@ def ntf_subscribe(self, mcast_name):
         self.sock.setsockopt(Netlink.SOL_NETLINK, Netlink.NETLINK_ADD_MEMBERSHIP,
                              mcast_id)
 
+    def set_recv_dbg(self, enabled):
+        self._recv_dbg = enabled
+
+    def _recv_dbg_print(self, reply, nl_msgs):
+        if not self._recv_dbg:
+            return
+        print("Recv: read", len(reply), "bytes,",
+              len(nl_msgs.msgs), "messages", file=sys.stderr)
+        for nl_msg in nl_msgs:
+            print("  ", nl_msg, file=sys.stderr)
+
     def _encode_enum(self, attr_spec, value):
         enum = self.consts[attr_spec['enum']]
         if enum.type == 'flags' or attr_spec.get('enum-as-flags', False):
@@ -799,11 +827,12 @@ def handle_ntf(self, decoded):
     def check_ntf(self):
         while True:
             try:
-                reply = self.sock.recv(128 * 1024, socket.MSG_DONTWAIT)
+                reply = self.sock.recv(self._recv_size, socket.MSG_DONTWAIT)
             except BlockingIOError:
                 return
 
             nms = NlMsgs(reply)
+            self._recv_dbg_print(reply, nms)
             for nl_msg in nms:
                 if nl_msg.error:
                     print("Netlink error in ntf!?", os.strerror(-nl_msg.error))
@@ -854,8 +883,9 @@ def _op(self, method, vals, flags=None, dump=False):
         done = False
         rsp = []
         while not done:
-            reply = self.sock.recv(128 * 1024)
+            reply = self.sock.recv(self._recv_size)
             nms = NlMsgs(reply, attr_space=op.attr_set)
+            self._recv_dbg_print(reply, nms)
             for nl_msg in nms:
                 if nl_msg.extack:
                     self._decode_extack(msg, op, nl_msg.extack)