Fix BZ #17916 - fopen unbounded stack usage for ccs= modes

ChangeLog

@@ -1,3 +1,9 @@
+2015-02-24  Paul Pluzhnikov  <ppluzhnikov@google.com>
+
+	[BZ #17916]
+	* libio/fileops.c (_IO_new_file_fopen): Limit stack use
+	* libio/tst-fopenloc.c (do_test, do_bz17916): Add a large ccs= test
+
 2015-02-24  Eric Rannaud  <e@nanocritical.com>
 
 	[BZ #17523]

NEWS

@@ -10,8 +10,8 @@ Version 2.22
 * The following bugs are resolved with this release:
 
   4719, 14841, 13064, 14094, 15319, 15467, 15790, 16560, 17269, 17523,
-  17569, 17588, 17792, 17836, 17912, 17932, 17944, 17949, 17964, 17965,
-  17967, 17969, 17978, 17987, 17991, 17996, 17998, 17999.
+  17569, 17588, 17792, 17836, 17912, 17916, 17932, 17944, 17949, 17964,
+  17965, 17967, 17969, 17978, 17987, 17991, 17996, 17998, 17999.
 
 * Character encoding and ctype tables were updated to Unicode 7.0.0, using
   new generator scripts contributed by Pravin Satpute and Mike FABIAN (Red

libio/fileops.c

@@ -353,7 +353,15 @@ _IO_new_file_fopen (_IO_FILE *fp, const char *filename, const char *mode,
 	  struct gconv_fcts fcts;
 	  struct _IO_codecvt *cc;
 	  char *endp = __strchrnul (cs + 5, ',');
-	  char ccs[endp - (cs + 5) + 3];
+	  char *ccs = malloc (endp - (cs + 5) + 3);
+
+	  if (ccs == NULL)
+	    {
+	      int malloc_err = errno;  /* Whatever malloc failed with.  */
+	      (void) _IO_file_close_it (fp);
+	      __set_errno (malloc_err);
+	      return NULL;
+	    }
 
 	  *((char *) __mempcpy (ccs, cs + 5, endp - (cs + 5))) = '\0';
 	  strip (ccs, ccs);
@@ -365,10 +373,13 @@ _IO_new_file_fopen (_IO_FILE *fp, const char *filename, const char *mode,
 		 This means we cannot proceed since the user explicitly asked
 		 for these.  */
 	      (void) _IO_file_close_it (fp);
+	      free (ccs);
 	      __set_errno (EINVAL);
 	      return NULL;
 	    }
 
+	  free (ccs);
+
 	  assert (fcts.towc_nsteps == 1);
 	  assert (fcts.tomb_nsteps == 1);
 

libio/tst-fopenloc.c

@@ -24,10 +24,36 @@
 #include <stdlib.h>
 #include <string.h>
 #include <wchar.h>
+#include <sys/resource.h>
 
 
 static const char inputfile[] = "../iconvdata/testdata/ISO-8859-1";
 
+static
+int do_bz17916 (void)
+{
+  /* BZ #17916 -- check invalid large ccs= case.  */
+  struct rlimit rl;
+  getrlimit (RLIMIT_STACK, &rl);
+  rl.rlim_cur = 1024 * 1024;
+  setrlimit (RLIMIT_STACK, &rl);
+
+  const size_t sz = 2 * 1024 * 1024;
+  char *ccs = malloc (sz);
+  strcpy (ccs, "r,ccs=");
+  memset (ccs + 6, 'A', sz - 6 - 1);
+  ccs[sz - 1] = '\0';
+
+  FILE *fp = fopen (inputfile, ccs);
+  if (fp != NULL)
+    {
+      printf ("unxpected success\n");
+      return 1;
+    }
+  free (ccs);
+
+  return 0;
+}
 
 static int
 do_test (void)
@@ -57,7 +83,7 @@ do_test (void)
 
   fclose (fp);
 
-  return 0;
+  return do_bz17916 ();
 }
 
 #define TEST_FUNCTION do_test ()