Fix change to prevent cancel signal in unsafe places.

The bits tested to decide when to delay the return when switching
off async cancel mode were wrong.  Fix that.  Also close a race
condition in pthread_cancel where the bit indicating the cancellation
is unconditionally set even if the cancel type might have changed.

nptl/ChangeLog

@@ -1,5 +1,11 @@
 2009-05-15  Ulrich Drepper  <drepper@redhat.com>
 
+	* cancellation.c (__pthread_disable_asynccancel): Correct the bits
+	to test when deciding on the delay.
+	* libc-cancellation.c (__libc_disable_asynccancel): Likewise.
+	* pthread_cancel.c: Close race between deciding on sending a signal
+	and setting the CANCELING_BIT bit.
+
 	* cancellation.c (__pthread_disable_asynccancel): Don't return if
 	thread is canceled.
 	* libc-cancellation.c (__libc_disable_asynccancel): Likewise.

nptl/cancellation.c

@@ -72,10 +72,6 @@ __pthread_disable_asynccancel (int oldtype)
   struct pthread *self = THREAD_SELF;
   int newval;
 
-#ifdef THREAD_ATOMIC_AND
-  THREAD_ATOMIC_AND (self, cancelhandling, ~CANCELTYPE_BITMASK);
-  newval = THREAD_GETMEM (self, cancelhandling);
-#else
   int oldval = THREAD_GETMEM (self, cancelhandling);
 
   while (1)
@@ -93,13 +89,13 @@ __pthread_disable_asynccancel (int oldtype)
       /* Prepare the next round.  */
       oldval = curval;
     }
-#endif
 
   /* We cannot return when we are being canceled.  Upon return the
      thread might be things which would have to be undone.  The
      following loop should loop until the cancellation signal is
      delivered.  */
-  while (__builtin_expect (newval & CANCELED_BITMASK, 0))
+  while (__builtin_expect ((newval & (CANCELING_BITMASK | CANCELED_BITMASK))
+			   == CANCELING_BITMASK, 0))
     {
       lll_futex_wait (&self->cancelhandling, newval, LLL_PRIVATE);
       newval = THREAD_GETMEM (self, cancelhandling);

nptl/libc-cancellation.c

@@ -88,10 +88,6 @@ __libc_disable_asynccancel (int oldtype)
   struct pthread *self = THREAD_SELF;
   int newval;
 
-#ifdef THREAD_ATOMIC_AND
-  THREAD_ATOMIC_AND (self, cancelhandling, ~CANCELTYPE_BITMASK);
-  newval = THREAD_GETMEM (self, cancelhandling);
-#else
   int oldval = THREAD_GETMEM (self, cancelhandling);
 
   while (1)
@@ -109,13 +105,13 @@ __libc_disable_asynccancel (int oldtype)
       /* Prepare the next round.  */
       oldval = curval;
     }
-#endif
 
   /* We cannot return when we are being canceled.  Upon return the
      thread might be things which would have to be undone.  The
      following loop should loop until the cancellation signal is
      delivered.  */
-  while (__builtin_expect (newval & CANCELED_BITMASK, 0))
+  while (__builtin_expect ((newval & (CANCELING_BITMASK | CANCELED_BITMASK))
+			   == CANCELING_BITMASK, 0))
     {
       lll_futex_wait (&self->cancelhandling, newval, LLL_PRIVATE);
       newval = THREAD_GETMEM (self, cancelhandling);

nptl/pthread_cancel.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2004, 2009 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -44,6 +44,7 @@ pthread_cancel (th)
   int newval;
   do
     {
+    again:
       oldval = pd->cancelhandling;
       newval = oldval | CANCELING_BITMASK | CANCELED_BITMASK;
 
@@ -59,7 +60,10 @@ pthread_cancel (th)
       if (CANCEL_ENABLED_AND_CANCELED_AND_ASYNCHRONOUS (newval))
 	{
 	  /* Mark the cancellation as "in progress".  */
-	  atomic_bit_set (&pd->cancelhandling, CANCELING_BIT);
+	  if (atomic_compare_and_exchange_bool_acq (&pd->cancelhandling,
+						    oldval | CANCELING_BITMASK,
+						    oldval))
+	    goto again;
 
 	  /* The cancellation handler will take care of marking the
 	     thread as canceled.  */