Skip to content

Commit

Permalink
[IA64-SGI] fixes for XPC disengage and open/close protocol
Browse files Browse the repository at this point in the history 
This patch addresses a few issues with the open/close protocol that
were revealed by the newly added disengage functionality combined
with more extensive testing.

Signed-off-by: Dean Nelson <dcn@sgi.com>
Signed-off-by: Tony Luck <tony.luck@intel.com>
  • Loading branch information
Dean Nelson authored and Tony Luck committed Oct 25, 2005
1 parent a607c38 commit e54af72
Show file tree
Hide file tree
Showing 5 changed files with 208 additions and 90 deletions.
21 changes: 12 additions & 9 deletions arch/ia64/sn/kernel/xpc.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -417,6 +417,9 @@ struct xpc_channel {
atomic_t n_on_msg_allocate_wq; /* #on msg allocation wait queue */
wait_queue_head_t msg_allocate_wq; /* msg allocation wait queue */

u8 delayed_IPI_flags; /* IPI flags received, but delayed */
/* action until channel disconnected */

/* queue of msg senders who want to be notified when msg received */

atomic_t n_to_notify; /* #of msg senders to notify */
Expand Down Expand Up @@ -478,7 +481,8 @@ struct xpc_channel {

#define XPC_C_DISCONNECTED 0x00002000 /* channel is disconnected */
#define XPC_C_DISCONNECTING 0x00004000 /* channel is being disconnected */
#define XPC_C_WDISCONNECT 0x00008000 /* waiting for channel disconnect */
#define XPC_C_DISCONNECTCALLOUT 0x00008000 /* chan disconnected callout made */
#define XPC_C_WDISCONNECT 0x00010000 /* waiting for channel disconnect */



Expand Down Expand Up @@ -508,13 +512,13 @@ struct xpc_partition {
int reason_line; /* line# deactivation initiated from */
int reactivate_nasid; /* nasid in partition to reactivate */

unsigned long disengage_request_timeout; /* timeout in XPC_TICKS */
unsigned long disengage_request_timeout; /* timeout in jiffies */
struct timer_list disengage_request_timer;


/* XPC infrastructure referencing and teardown control */

volatile u8 setup_state; /* infrastructure setup state */
volatile u8 setup_state; /* infrastructure setup state */
wait_queue_head_t teardown_wq; /* kthread waiting to teardown infra */
atomic_t references; /* #of references to infrastructure */

Expand Down Expand Up @@ -604,7 +608,7 @@ struct xpc_partition {


/* number of seconds to wait for other partitions to disengage */
#define XPC_DISENGAGE_REQUEST_TIMELIMIT 90
#define XPC_DISENGAGE_REQUEST_DEFAULT_TIMELIMIT 90

/* interval in seconds to print 'waiting disengagement' messages */
#define XPC_DISENGAGE_PRINTMSG_INTERVAL 10
Expand All @@ -618,20 +622,18 @@ struct xpc_partition {
extern struct xpc_registration xpc_registrations[];


/* >>> found in xpc_main.c only */
/* found in xpc_main.c */
extern struct device *xpc_part;
extern struct device *xpc_chan;
extern int xpc_disengage_request_timelimit;
extern irqreturn_t xpc_notify_IRQ_handler(int, void *, struct pt_regs *);
extern void xpc_dropped_IPI_check(struct xpc_partition *);
extern void xpc_activate_partition(struct xpc_partition *);
extern void xpc_activate_kthreads(struct xpc_channel *, int);
extern void xpc_create_kthreads(struct xpc_channel *, int);
extern void xpc_disconnect_wait(int);


/* found in xpc_main.c and efi-xpc.c */
extern void xpc_activate_partition(struct xpc_partition *);


/* found in xpc_partition.c */
extern int xpc_exiting;
extern struct xpc_vars *xpc_vars;
Expand Down Expand Up @@ -1077,6 +1079,7 @@ xpc_notify_IRQ_send_local(struct xpc_channel *ch, u8 ipi_flag,

/* given an AMO variable and a channel#, get its associated IPI flags */
#define XPC_GET_IPI_FLAGS(_amo, _c) ((u8) (((_amo) >> ((_c) * 8)) & 0xff))
#define XPC_SET_IPI_FLAGS(_amo, _c, _f) (_amo) |= ((u64) (_f) << ((_c) * 8))

#define XPC_ANY_OPENCLOSE_IPI_FLAGS_SET(_amo) ((_amo) & 0x0f0f0f0f0f0f0f0f)
#define XPC_ANY_MSG_IPI_FLAGS_SET(_amo) ((_amo) & 0x1010101010101010)
Expand Down
117 changes: 82 additions & 35 deletions arch/ia64/sn/kernel/xpc_channel.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -792,11 +792,20 @@ xpc_process_disconnect(struct xpc_channel *ch, unsigned long *irq_flags)
"reason=%d\n", ch->number, ch->partid, ch->reason);
}

/* wake the thread that is waiting for this channel to disconnect */
if (ch->flags & XPC_C_WDISCONNECT) {
spin_unlock_irqrestore(&ch->lock, *irq_flags);
up(&ch->wdisconnect_sema);
spin_lock_irqsave(&ch->lock, *irq_flags);

} else if (ch->delayed_IPI_flags) {
if (part->act_state != XPC_P_DEACTIVATING) {
/* time to take action on any delayed IPI flags */
spin_lock(&part->IPI_lock);
XPC_SET_IPI_FLAGS(part->local_IPI_amo, ch->number,
ch->delayed_IPI_flags);
spin_unlock(&part->IPI_lock);
}
ch->delayed_IPI_flags = 0;
}
}

Expand All @@ -818,6 +827,19 @@ xpc_process_openclose_IPI(struct xpc_partition *part, int ch_number,

spin_lock_irqsave(&ch->lock, irq_flags);

again:

if ((ch->flags & XPC_C_DISCONNECTED) &&
(ch->flags & XPC_C_WDISCONNECT)) {
/*
* Delay processing IPI flags until thread waiting disconnect
* has had a chance to see that the channel is disconnected.
*/
ch->delayed_IPI_flags |= IPI_flags;
spin_unlock_irqrestore(&ch->lock, irq_flags);
return;
}


if (IPI_flags & XPC_IPI_CLOSEREQUEST) {

Expand All @@ -843,14 +865,22 @@ xpc_process_openclose_IPI(struct xpc_partition *part, int ch_number,

/* both sides have finished disconnecting */
xpc_process_disconnect(ch, &irq_flags);
DBUG_ON(!(ch->flags & XPC_C_DISCONNECTED));
goto again;
}

if (ch->flags & XPC_C_DISCONNECTED) {
// >>> explain this section

if (!(IPI_flags & XPC_IPI_OPENREQUEST)) {
DBUG_ON(part->act_state !=
XPC_P_DEACTIVATING);
if ((XPC_GET_IPI_FLAGS(part->local_IPI_amo,
ch_number) & XPC_IPI_OPENREQUEST)) {

DBUG_ON(ch->delayed_IPI_flags != 0);
spin_lock(&part->IPI_lock);
XPC_SET_IPI_FLAGS(part->local_IPI_amo,
ch_number,
XPC_IPI_CLOSEREQUEST);
spin_unlock(&part->IPI_lock);
}
spin_unlock_irqrestore(&ch->lock, irq_flags);
return;
}
Expand Down Expand Up @@ -880,9 +910,13 @@ xpc_process_openclose_IPI(struct xpc_partition *part, int ch_number,
}

XPC_DISCONNECT_CHANNEL(ch, reason, &irq_flags);
} else {
xpc_process_disconnect(ch, &irq_flags);

DBUG_ON(IPI_flags & XPC_IPI_CLOSEREPLY);
spin_unlock_irqrestore(&ch->lock, irq_flags);
return;
}

xpc_process_disconnect(ch, &irq_flags);
}


Expand All @@ -898,7 +932,20 @@ xpc_process_openclose_IPI(struct xpc_partition *part, int ch_number,
}

DBUG_ON(!(ch->flags & XPC_C_CLOSEREQUEST));
DBUG_ON(!(ch->flags & XPC_C_RCLOSEREQUEST));

if (!(ch->flags & XPC_C_RCLOSEREQUEST)) {
if ((XPC_GET_IPI_FLAGS(part->local_IPI_amo, ch_number)
& XPC_IPI_CLOSEREQUEST)) {

DBUG_ON(ch->delayed_IPI_flags != 0);
spin_lock(&part->IPI_lock);
XPC_SET_IPI_FLAGS(part->local_IPI_amo,
ch_number, XPC_IPI_CLOSEREPLY);
spin_unlock(&part->IPI_lock);
}
spin_unlock_irqrestore(&ch->lock, irq_flags);
return;
}

ch->flags |= XPC_C_RCLOSEREPLY;

Expand All @@ -916,8 +963,14 @@ xpc_process_openclose_IPI(struct xpc_partition *part, int ch_number,
"channel=%d\n", args->msg_size, args->local_nentries,
ch->partid, ch->number);

if ((ch->flags & (XPC_C_DISCONNECTING | XPC_C_WDISCONNECT)) ||
part->act_state == XPC_P_DEACTIVATING) {
if (part->act_state == XPC_P_DEACTIVATING ||
(ch->flags & XPC_C_ROPENREQUEST)) {
spin_unlock_irqrestore(&ch->lock, irq_flags);
return;
}

if (ch->flags & (XPC_C_DISCONNECTING | XPC_C_WDISCONNECT)) {
ch->delayed_IPI_flags |= XPC_IPI_OPENREQUEST;
spin_unlock_irqrestore(&ch->lock, irq_flags);
return;
}
Expand All @@ -931,8 +984,11 @@ xpc_process_openclose_IPI(struct xpc_partition *part, int ch_number,
* msg_size = size of channel's messages in bytes
* local_nentries = remote partition's local_nentries
*/
DBUG_ON(args->msg_size == 0);
DBUG_ON(args->local_nentries == 0);
if (args->msg_size == 0 || args->local_nentries == 0) {
/* assume OPENREQUEST was delayed by mistake */
spin_unlock_irqrestore(&ch->lock, irq_flags);
return;
}

ch->flags |= (XPC_C_ROPENREQUEST | XPC_C_CONNECTING);
ch->remote_nentries = args->local_nentries;
Expand Down Expand Up @@ -970,7 +1026,13 @@ xpc_process_openclose_IPI(struct xpc_partition *part, int ch_number,
spin_unlock_irqrestore(&ch->lock, irq_flags);
return;
}
DBUG_ON(!(ch->flags & XPC_C_OPENREQUEST));
if (!(ch->flags & XPC_C_OPENREQUEST)) {
XPC_DISCONNECT_CHANNEL(ch, xpcOpenCloseError,
&irq_flags);
spin_unlock_irqrestore(&ch->lock, irq_flags);
return;
}

DBUG_ON(!(ch->flags & XPC_C_ROPENREQUEST));
DBUG_ON(ch->flags & XPC_C_CONNECTED);

Expand Down Expand Up @@ -1024,8 +1086,8 @@ xpc_connect_channel(struct xpc_channel *ch)
struct xpc_registration *registration = &xpc_registrations[ch->number];


if (down_interruptible(&registration->sema) != 0) {
return xpcInterrupted;
if (down_trylock(&registration->sema) != 0) {
return xpcRetry;
}

if (!XPC_CHANNEL_REGISTERED(ch->number)) {
Expand Down Expand Up @@ -1445,19 +1507,11 @@ xpc_initiate_connect(int ch_number)
if (xpc_part_ref(part)) {
ch = &part->channels[ch_number];

if (!(ch->flags & XPC_C_DISCONNECTING)) {
DBUG_ON(ch->flags & XPC_C_OPENREQUEST);
DBUG_ON(ch->flags & XPC_C_CONNECTED);
DBUG_ON(ch->flags & XPC_C_SETUP);

/*
* Initiate the establishment of a connection
* on the newly registered channel to the
* remote partition.
*/
xpc_wakeup_channel_mgr(part);
}

/*
* Initiate the establishment of a connection on the
* newly registered channel to the remote partition.
*/
xpc_wakeup_channel_mgr(part);
xpc_part_deref(part);
}
}
Expand All @@ -1467,9 +1521,6 @@ xpc_initiate_connect(int ch_number)
void
xpc_connected_callout(struct xpc_channel *ch)
{
unsigned long irq_flags;


/* let the registerer know that a connection has been established */

if (ch->func != NULL) {
Expand All @@ -1482,10 +1533,6 @@ xpc_connected_callout(struct xpc_channel *ch)
dev_dbg(xpc_chan, "ch->func() returned, reason=xpcConnected, "
"partid=%d, channel=%d\n", ch->partid, ch->number);
}

spin_lock_irqsave(&ch->lock, irq_flags);
ch->flags |= XPC_C_CONNECTCALLOUT;
spin_unlock_irqrestore(&ch->lock, irq_flags);
}


Expand Down
Loading

0 comments on commit e54af72

Please sign in to comment.