Jakub Jelinek <jakub@redhat.com>

	Implement reference counting of scope records.
	* elf/dl-close.c (_dl_close): Remove all scopes from removed objects
	from the list in objects which remain.  Always allocate new scope
	record.
	* elf/dl-open.c (dl_open_worker): When growing array for scopes,
	don't resize, allocate a new one.
	* elf/dl-runtime.c: Update reference counters before using a scope
	array.
	* elf/dl-sym.c: Likewise.
	* elf/dl-libc.c: Adjust for l_scope name change.
	* elf/dl-load.c: Likewise.
	* elf/dl-object.c: Likewise.
	* elf/rtld.c: Likewise.
	* include/link.h: Inlcude <rtld-lowlevel.h>.  Define struct
	r_scoperec.  Replace r_scope with pointer to r_scoperec structure.
	Add l_scoperec_lock.
	* sysdeps/generic/ldsodefs.h: Include <rtld-lowlevel.h>.
	* sysdeps/generic/rtld-lowlevel.h: New file.

	* include/atomic.h: Rename atomic_and to atomic_and_val and
	atomic_or to atomic_or_val.  Define new macros atomic_and and
	atomic_or which do not return values.
	* sysdeps/x86_64/bits/atomic.h: Define atomic_and and atomic_or.
	Various cleanups.
	* sysdeps/i386/i486/bits/atomic.h: Likewise.

ChangeLog

@@ -1,4 +1,31 @@
 2006-10-09  Ulrich Drepper  <drepper@redhat.com>
+	    Jakub Jelinek  <jakub@redhat.com>
+
+	Implement reference counting of scope records.
+	* elf/dl-close.c (_dl_close): Remove all scopes from removed objects
+	from the list in objects which remain.  Always allocate new scope
+	record.
+	* elf/dl-open.c (dl_open_worker): When growing array for scopes,
+	don't resize, allocate a new one.
+	* elf/dl-runtime.c: Update reference counters before using a scope
+	array.
+	* elf/dl-sym.c: Likewise.
+	* elf/dl-libc.c: Adjust for l_scope name change.
+	* elf/dl-load.c: Likewise.
+	* elf/dl-object.c: Likewise.
+	* elf/rtld.c: Likewise.
+	* include/link.h: Inlcude <rtld-lowlevel.h>.  Define struct
+	r_scoperec.  Replace r_scope with pointer to r_scoperec structure.
+	Add l_scoperec_lock.
+	* sysdeps/generic/ldsodefs.h: Include <rtld-lowlevel.h>.
+	* sysdeps/generic/rtld-lowlevel.h: New file.
+
+	* include/atomic.h: Rename atomic_and to atomic_and_val and
+	atomic_or to atomic_or_val.  Define new macros atomic_and and
+	atomic_or which do not return values.
+	* sysdeps/x86_64/bits/atomic.h: Define atomic_and and atomic_or.
+	Various cleanups.
+	* sysdeps/i386/i486/bits/atomic.h: Likewise.
 
 	* po/sv.po: Update from translation team.
 

elf/dl-close.c

@@ -19,6 +19,7 @@
 
 #include <assert.h>
 #include <dlfcn.h>
+#include <errno.h>
 #include <libintl.h>
 #include <stddef.h>
 #include <stdio.h>
@@ -35,6 +36,10 @@
 typedef void (*fini_t) (void);
 
 
+/* Special l_idx value used to indicate which objects remain loaded.  */
+#define IDX_STILL_USED -1
+
+
 #ifdef USE_TLS
 /* Returns true we an non-empty was found.  */
 static bool
@@ -188,15 +193,15 @@ _dl_close (void *_map)
       done[done_index] = 1;
       used[done_index] = 1;
       /* Signal the object is still needed.  */
-      l->l_idx = -1;
+      l->l_idx = IDX_STILL_USED;
 
       /* Mark all dependencies as used.  */
       if (l->l_initfini != NULL)
 	{
 	  struct link_map **lp = &l->l_initfini[1];
 	  while (*lp != NULL)
 	    {
-	      if ((*lp)->l_idx != -1)
+	      if ((*lp)->l_idx != IDX_STILL_USED)
 		{
 		  assert ((*lp)->l_idx >= 0 && (*lp)->l_idx < nloaded);
 
@@ -217,7 +222,7 @@ _dl_close (void *_map)
 	  {
 	    struct link_map *jmap = l->l_reldeps[j];
 
-	    if (jmap->l_idx != -1)
+	    if (jmap->l_idx != IDX_STILL_USED)
 	      {
 		assert (jmap->l_idx >= 0 && jmap->l_idx < nloaded);
 
@@ -310,8 +315,9 @@ _dl_close (void *_map)
       /* Else used[i].  */
       else if (imap->l_type == lt_loaded)
 	{
-	  if (imap->l_searchlist.r_list == NULL
-	      && imap->l_initfini != NULL)
+	  struct r_scope_elem *new_list = NULL;
+
+	  if (imap->l_searchlist.r_list == NULL && imap->l_initfini != NULL)
 	    {
 	      /* The object is still used.  But one of the objects we are
 		 unloading right now is responsible for loading it.  If
@@ -328,44 +334,114 @@ _dl_close (void *_map)
 	      imap->l_searchlist.r_list = &imap->l_initfini[cnt + 1];
 	      imap->l_searchlist.r_nlist = cnt;
 
-	      for (cnt = 0; imap->l_scope[cnt] != NULL; ++cnt)
-		/* This relies on l_scope[] entries being always set either
-		   to its own l_symbolic_searchlist address, or some map's
-		   l_searchlist address.  */
-		if (imap->l_scope[cnt] != &imap->l_symbolic_searchlist)
-		  {
-		    struct link_map *tmap;
-
-		    tmap = (struct link_map *) ((char *) imap->l_scope[cnt]
-						- offsetof (struct link_map,
-							    l_searchlist));
-		    assert (tmap->l_ns == ns);
-		    if (tmap->l_idx != -1)
-		      {
-			imap->l_scope[cnt] = &imap->l_searchlist;
-			break;
-		      }
-		  }
+	      new_list = &imap->l_searchlist;
 	    }
-	  else
+
+	  /* Count the number of scopes which remain after the unload.
+	     When we add the local search list count it.  Always add
+	     one for the terminating NULL pointer.  */
+	  size_t remain = (new_list != NULL) + 1;
+	  bool removed_any = false;
+	  for (size_t cnt = 0; imap->l_scoperec->scope[cnt] != NULL; ++cnt)
+	    /* This relies on l_scope[] entries being always set either
+	       to its own l_symbolic_searchlist address, or some map's
+	       l_searchlist address.  */
+	    if (imap->l_scoperec->scope[cnt] != &imap->l_symbolic_searchlist)
+	      {
+		struct link_map *tmap = (struct link_map *)
+		  ((char *) imap->l_scoperec->scope[cnt]
+		   - offsetof (struct link_map, l_searchlist));
+		assert (tmap->l_ns == ns);
+		if (tmap->l_idx == IDX_STILL_USED)
+		  ++remain;
+		else
+		  removed_any = true;
+	      }
+	    else
+	      ++remain;
+
+	  if (removed_any)
 	    {
-	      unsigned int cnt = 0;
-	      while (imap->l_scope[cnt] != NULL)
+	      /* Always allocate a new array for the scope.  This is
+		 necessary since we must be able to determine the last
+		 user of the current array.  If possible use the link map's
+		 memory.  */
+	      size_t new_size;
+	      struct r_scoperec *newp;
+	      if (imap->l_scoperec != &imap->l_scoperec_mem
+		  && remain < NINIT_SCOPE_ELEMS (imap)
+		  && imap->l_scoperec_mem.nusers == 0)
+		{
+		  new_size = NINIT_SCOPE_ELEMS (imap);
+		  newp = &imap->l_scoperec_mem;
+		}
+	      else
+		{
+		  new_size = imap->l_scope_max;
+		  newp = (struct r_scoperec *)
+		    malloc (sizeof (struct r_scoperec)
+			    + new_size * sizeof (struct r_scope_elem *));
+		  if (newp == NULL)
+		    _dl_signal_error (ENOMEM, "dlclose", NULL,
+				      N_("cannot create scope list"));
+		}
+
+	      newp->nusers = 0;
+	      newp->remove_after_use = false;
+	      newp->notify = false;
+
+	      /* Copy over the remaining scope elements.  */
+	      remain = 0;
+	      for (size_t cnt = 0; imap->l_scoperec->scope[cnt] != NULL; ++cnt)
 		{
-		  if (imap->l_scope[cnt] == &map->l_searchlist)
+		  if (imap->l_scoperec->scope[cnt]
+		      != &imap->l_symbolic_searchlist)
 		    {
-		      while ((imap->l_scope[cnt] = imap->l_scope[cnt + 1])
-			     != NULL)
-			++cnt;
-		      break;
+		      struct link_map *tmap = (struct link_map *)
+			((char *) imap->l_scoperec->scope[cnt]
+			 - offsetof (struct link_map, l_searchlist));
+		      if (tmap->l_idx != IDX_STILL_USED)
+			{
+			  /* Remove the scope.  Or replace with own map's
+			     scope.  */
+			  if (new_list != NULL)
+			    {
+			      newp->scope[remain++] = new_list;
+			      new_list = NULL;
+			    }
+			  continue;
+			}
 		    }
-		  ++cnt;
+
+		  newp->scope[remain++] = imap->l_scoperec->scope[cnt];
 		}
+	      newp->scope[remain] = NULL;
+
+	      struct r_scoperec *old = imap->l_scoperec;
+
+	      __rtld_mrlock_change (imap->l_scoperec_lock);
+	      imap->l_scoperec = newp;
+	      __rtld_mrlock_done (imap->l_scoperec_lock);
+
+	      if (atomic_increment_val (&old->nusers) != 1)
+		{
+		  old->remove_after_use = true;
+		  old->notify = true;
+		  if (atomic_decrement_val (&old->nusers) != 0)
+		    __rtld_waitzero (old->nusers);
+		}
+
+	      /* No user anymore, we can free it now.  */
+	      if (old != &imap->l_scoperec_mem)
+		free (old);
+
+	      imap->l_scope_max = new_size;
 	    }
 
 	  /* The loader is gone, so mark the object as not having one.
-	     Note: l_idx != -1 -> object will be removed.  */
-	  if (imap->l_loader != NULL && imap->l_loader->l_idx != -1)
+	     Note: l_idx != IDX_STILL_USED -> object will be removed.  */
+	  if (imap->l_loader != NULL
+	      && imap->l_loader->l_idx != IDX_STILL_USED)
 	    imap->l_loader = NULL;
 
 	  /* Remember where the first dynamically loaded object is.  */
@@ -570,8 +646,8 @@ _dl_close (void *_map)
 	  free (imap->l_initfini);
 
 	  /* Remove the scope array if we allocated it.  */
-	  if (imap->l_scope != imap->l_scope_mem)
-	    free (imap->l_scope);
+	  if (imap->l_scoperec != &imap->l_scoperec_mem)
+	    free (imap->l_scoperec);
 
 	  if (imap->l_phdr_allocated)
 	    free ((void *) imap->l_phdr);

elf/dl-libc.c

@@ -1,5 +1,5 @@
 /* Handle loading and unloading shared objects for internal libc purposes.
-   Copyright (C) 1999,2000,2001,2002,2004,2005 Free Software Foundation, Inc.
+   Copyright (C) 1999-2002,2004,2005,2006 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Zack Weinberg <zack@rabi.columbia.edu>, 1999.
 
@@ -133,7 +133,8 @@ do_dlsym_private (void *ptr)
   struct do_dlsym_args *args = (struct do_dlsym_args *) ptr;
   args->ref = NULL;
   l = GLRO(dl_lookup_symbol_x) (args->name, args->map, &args->ref,
-				args->map->l_scope, &vers, 0, 0, NULL);
+				args->map->l_scoperec->scope, &vers, 0, 0,
+				NULL);
   args->loadbase = l;
 }
 

elf/dl-load.c

@@ -1473,7 +1473,7 @@ cannot enable executable stack as shared object requires");
      have to do this for the main map.  */
   if ((mode & RTLD_DEEPBIND) == 0
       && __builtin_expect (l->l_info[DT_SYMBOLIC] != NULL, 0)
-      && &l->l_searchlist != l->l_scope[0])
+      && &l->l_searchlist != l->l_scoperec->scope[0])
     {
       /* Create an appropriate searchlist.  It contains only this map.
 	 This is the definition of DT_SYMBOLIC in SysVr4.  */
@@ -1490,11 +1490,11 @@ cannot enable executable stack as shared object requires");
       l->l_symbolic_searchlist.r_nlist = 1;
 
       /* Now move the existing entries one back.  */
-      memmove (&l->l_scope[1], &l->l_scope[0],
-	       (l->l_scope_max - 1) * sizeof (l->l_scope[0]));
+      memmove (&l->l_scoperec->scope[1], &l->l_scoperec->scope[0],
+	       (l->l_scope_max - 1) * sizeof (l->l_scoperec->scope[0]));
 
       /* Now add the new entry.  */
-      l->l_scope[0] = &l->l_symbolic_searchlist;
+      l->l_scoperec->scope[0] = &l->l_symbolic_searchlist;
     }
 
   /* Remember whether this object must be initialized first.  */

elf/dl-object.c

@@ -1,5 +1,5 @@
 /* Storage management for the chain of loaded shared objects.
-   Copyright (C) 1995-2002, 2004 Free Software Foundation, Inc.
+   Copyright (C) 1995-2002, 2004, 2006 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
 
    The GNU C Library is free software; you can redistribute it and/or
@@ -82,8 +82,14 @@ _dl_new_object (char *realname, const char *libname, int type,
   /* Use the 'l_scope_mem' array by default for the the 'l_scope'
      information.  If we need more entries we will allocate a large
      array dynamically.  */
-  new->l_scope = new->l_scope_mem;
-  new->l_scope_max = sizeof (new->l_scope_mem) / sizeof (new->l_scope_mem[0]);
+  new->l_scoperec = &new->l_scoperec_mem;
+  new->l_scope_max = (sizeof (new->l_scope_realmem.scope_elems)
+		      / sizeof (new->l_scope_realmem.scope_elems[0]));
+
+  /* No need to initialize the scope lock if the initializer is zero.  */
+#if _RTLD_MRLOCK_INITIALIZER != 0
+  __rtld_mrlock_initialize (new->l_scoperec_mem.lock);
+#endif
 
   /* Counter for the scopes we have to handle.  */
   idx = 0;
@@ -98,7 +104,8 @@ _dl_new_object (char *realname, const char *libname, int type,
       l->l_next = new;
 
       /* Add the global scope.  */
-      new->l_scope[idx++] = &GL(dl_ns)[nsid]._ns_loaded->l_searchlist;
+      new->l_scoperec->scope[idx++]
+	= &GL(dl_ns)[nsid]._ns_loaded->l_searchlist;
     }
   else
     GL(dl_ns)[nsid]._ns_loaded = new;
@@ -114,15 +121,15 @@ _dl_new_object (char *realname, const char *libname, int type,
       loader = loader->l_loader;
 
   /* Insert the scope if it isn't the global scope we already added.  */
-  if (idx == 0 || &loader->l_searchlist != new->l_scope[0])
+  if (idx == 0 || &loader->l_searchlist != new->l_scoperec->scope[0])
     {
       if ((mode & RTLD_DEEPBIND) != 0 && idx != 0)
 	{
-	  new->l_scope[1] = new->l_scope[0];
+	  new->l_scoperec->scope[1] = new->l_scoperec->scope[0];
 	  idx = 0;
 	}
 
-      new->l_scope[idx] = &loader->l_searchlist;
+      new->l_scoperec->scope[idx] = &loader->l_searchlist;
     }
 
   new->l_local_scope[0] = &new->l_searchlist;