Avoid too much stack use in fnmatch.

ChangeLog

@@ -1,3 +1,9 @@
+2010-08-09  Ulrich Drepper  <drepper@redhat.com>
+
+	[BZ #11883]
+	* posix/fnmatch.c: Keep track of alloca use and fall back on malloc.
+	* posix/fnmatch_loop.c: Likewise.
+
 2010-07-17  Andi Kleen  <ak@linux.intel.com>
 
 	* sysdeps/i386/i386-mcount.S (__fentry__): Define.

NEWS

@@ -1,4 +1,4 @@
-GNU C Library NEWS -- history of user-visible changes.  2010-7-29
+GNU C Library NEWS -- history of user-visible changes.  2010-8-9
 Copyright (C) 1992-2009, 2010 Free Software Foundation, Inc.
 See the end for copying conditions.
 
@@ -9,7 +9,7 @@ Version 2.13
 
 * The following bugs are resolved with this release:
 
-  11640, 11701, 11840, 11856
+  11640, 11701, 11840, 11856, 11883
 
 * POWER7 optimizations: memset, memcmp, strncmp
 

posix/fnmatch.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 1991,1992,1993,1996,1997,1998,1999,2000,2001,2002,2003,2007
+/* Copyright (C) 1991,1992,1993,1996,1997,1998,1999,2000,2001,2002,2003,2007,2010
 	Free Software Foundation, Inc.
    This file is part of the GNU C Library.
 
@@ -41,6 +41,12 @@
 # include <stdlib.h>
 #endif
 
+#ifdef _LIBC
+# include <alloca.h>
+#else
+# define alloca_account(size., var) alloca (size)
+#endif
+
 /* For platform which support the ISO C amendement 1 functionality we
    support user defined character classes.  */
 #if defined _LIBC || (defined HAVE_WCTYPE_H && defined HAVE_WCHAR_H)
@@ -330,8 +336,11 @@ fnmatch (pattern, string, flags)
       mbstate_t ps;
       size_t n;
       const char *p;
+      wchar_t *wpattern_malloc = NULL;
       wchar_t *wpattern;
+      wchar_t *wstring_malloc = NULL;
       wchar_t *wstring;
+      size_t alloca_used = 0;
 
       /* Convert the strings into wide characters.  */
       memset (&ps, '\0', sizeof (ps));
@@ -343,7 +352,8 @@ fnmatch (pattern, string, flags)
 #endif
       if (__builtin_expect (n < 1024, 1))
 	{
-	  wpattern = (wchar_t *) alloca ((n + 1) * sizeof (wchar_t));
+	  wpattern = (wchar_t *) alloca_account ((n + 1) * sizeof (wchar_t),
+						 alloca_used);
 	  n = mbsrtowcs (wpattern, &p, n + 1, &ps);
 	  if (__builtin_expect (n == (size_t) -1, 0))
 	    /* Something wrong.
@@ -365,8 +375,11 @@ fnmatch (pattern, string, flags)
 	       XXX Do we have to set `errno' to something which mbsrtows hasn't
 	       already done?  */
 	    return -1;
-	  wpattern = (wchar_t *) alloca ((n + 1) * sizeof (wchar_t));
+	  wpattern_malloc = wpattern
+	    = (wchar_t *) malloc ((n + 1) * sizeof (wchar_t));
 	  assert (mbsinit (&ps));
+	  if (wpattern == NULL)
+	    return -2;
 	  (void) mbsrtowcs (wpattern, &pattern, n + 1, &ps);
 	}
 
@@ -379,13 +392,18 @@ fnmatch (pattern, string, flags)
       p = string;
       if (__builtin_expect (n < 1024, 1))
 	{
-	  wstring = (wchar_t *) alloca ((n + 1) * sizeof (wchar_t));
+	  wstring = (wchar_t *) alloca_account ((n + 1) * sizeof (wchar_t),
+						alloca_used);
 	  n = mbsrtowcs (wstring, &p, n + 1, &ps);
 	  if (__builtin_expect (n == (size_t) -1, 0))
-	    /* Something wrong.
-	       XXX Do we have to set `errno' to something which mbsrtows hasn't
-	       already done?  */
-	    return -1;
+	    {
+	      /* Something wrong.
+		 XXX Do we have to set `errno' to something which
+		 mbsrtows hasn't already done?  */
+	    free_return:
+	      free (wpattern_malloc);
+	      return -1;
+	    }
 	  if (p)
 	    {
 	      memset (&ps, '\0', sizeof (ps));
@@ -400,19 +418,32 @@ fnmatch (pattern, string, flags)
 	    /* Something wrong.
 	       XXX Do we have to set `errno' to something which mbsrtows hasn't
 	       already done?  */
-	    return -1;
-	  wstring = (wchar_t *) alloca ((n + 1) * sizeof (wchar_t));
+	    goto free_return;
+
+	  wstring_malloc = wstring
+	    = (wchar_t *) malloc ((n + 1) * sizeof (wchar_t));
+	  if (wstring == NULL)
+	    {
+	      free (wpattern_malloc);
+	      return -2;
+	    }
 	  assert (mbsinit (&ps));
 	  (void) mbsrtowcs (wstring, &string, n + 1, &ps);
 	}
 
-      return internal_fnwmatch (wpattern, wstring, wstring + n,
-				flags & FNM_PERIOD, flags, NULL);
+      int res = internal_fnwmatch (wpattern, wstring, wstring + n,
+				   flags & FNM_PERIOD, flags, NULL,
+				   alloca_used);
+
+      free (wstring_malloc);
+      free (wpattern_malloc);
+
+      return res;
     }
 # endif  /* mbstate_t and mbsrtowcs or _LIBC.  */
 
   return internal_fnmatch (pattern, string, string + strlen (string),
-			   flags & FNM_PERIOD, flags, NULL);
+			   flags & FNM_PERIOD, flags, NULL, 0);
 }
 
 # ifdef _LIBC