Fix BZ #18757.

ChangeLog

@@ -1,3 +1,10 @@
+2015-09-01  Paul Pluzhnikov  <ppluzhnikov@google.com>
+
+	[BZ #18757]
+	* libio/iofopncook.c (_IO_fopencookie): Set errno on failure.
+	* libio/test-fmemopen.c (do_bz18820): Extend the test to cover
+	BZ #18757.
+
 2015-09-01  Paul Pluzhnikov  <ppluzhnikov@google.com>
 
 	* malloc/mtrace.pl: Filter out NULL entries.

NEWS

@@ -11,8 +11,9 @@ Version 2.23
 
   2898, 14341, 15786, 16141, 16517, 16519, 16520, 16734, 16973, 17787,
   17905, 18084, 18086, 18240, 18265, 18370, 18421, 18480, 18525, 18610,
-  18618, 18647, 18661, 18674, 18681, 18778, 18781, 18787, 18789, 18790,
-  18795, 18796, 18820, 18823, 18824, 18863, 18870, 18873, 18887.
+  18618, 18647, 18661, 18674, 18681, 18757, 18778, 18781, 18787, 18789,
+  18790, 18795, 18796, 18820, 18823, 18824, 18863, 18870, 18873, 18887.
+
 
 * The obsolete header <regexp.h> has been removed.  Programs that require
   this header must be updated to use <regex.h> instead.

libio/iofopncook.c

@@ -189,6 +189,7 @@ _IO_fopencookie (cookie, mode, io_functions)
       read_write = _IO_NO_READS|_IO_IS_APPENDING;
       break;
     default:
+      __set_errno (EINVAL);
       return NULL;
   }
   if (mode[0] == '+' || (mode[0] == 'b' && mode[1] == '+'))

libio/test-fmemopen.c

@@ -19,6 +19,7 @@
 
 static char buffer[] = "foobar";
 
+#include <errno.h>
 #include <stdio.h>
 #include <string.h>
 #include <errno.h>
@@ -30,13 +31,19 @@ do_bz18820 (void)
   char ch;
   FILE *stream;
 
+  errno = 0;
   stream = fmemopen (&ch, 1, "?");
   if (stream)
     {
       printf ("fmemopen: expected NULL, got %p\n", stream);
       fclose (stream);
       return 1;
     }
+  if (errno != EINVAL)
+    {
+      printf ("fmemopen: got %i, expected EINVAL (%i)\n", errno, EINVAL);
+      return 10;
+    }
 
   stream = fmemopen (NULL, 42, "?");
   if (stream)
@@ -46,6 +53,20 @@ do_bz18820 (void)
       return 2;
     }
 
+  errno = 0;
+  stream = fmemopen (NULL, ~0, "w");
+  if (stream)
+    {
+      printf ("fmemopen: expected NULL, got %p\n", stream);
+      fclose (stream);
+      return 3;
+    }
+  if (errno != ENOMEM)
+    {
+      printf ("fmemopen: got %i, expected ENOMEM (%i)\n", errno, ENOMEM);
+      return 20;
+    }
+
   return 0;
 }