blob: 2e975762c32b42224f80234ccf4f8caddb34cfb4 [file] [log] [blame]
Dean Nelson94bd2702008-07-29 22:34:05 -07001/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved.
7 */
8
9/*
10 * Cross Partition Communication (XPC) sn2-based functions.
11 *
12 * Architecture specific implementation of common functions.
13 *
14 */
15
Dean Nelsone17d4162008-07-29 22:34:06 -070016#include <linux/delay.h>
Dean Nelson94bd2702008-07-29 22:34:05 -070017#include <asm/uncached.h>
Dean Nelson261f3b42008-07-29 22:34:16 -070018#include <asm/sn/mspec.h>
Dean Nelson94bd2702008-07-29 22:34:05 -070019#include <asm/sn/sn_sal.h>
20#include "xpc.h"
21
Dean Nelsonee6665e2008-07-29 22:34:13 -070022/*
23 * Define the number of u64s required to represent all the C-brick nasids
24 * as a bitmap. The cross-partition kernel modules deal only with
25 * C-brick nasids, thus the need for bitmaps which don't account for
26 * odd-numbered (non C-brick) nasids.
27 */
28#define XPC_MAX_PHYSNODES_SN2 (MAX_NUMALINK_NODES / 2)
29#define XP_NASID_MASK_BYTES_SN2 ((XPC_MAX_PHYSNODES_SN2 + 7) / 8)
30#define XP_NASID_MASK_WORDS_SN2 ((XPC_MAX_PHYSNODES_SN2 + 63) / 64)
31
32/*
33 * Memory for XPC's amo variables is allocated by the MSPEC driver. These
34 * pages are located in the lowest granule. The lowest granule uses 4k pages
35 * for cached references and an alternate TLB handler to never provide a
36 * cacheable mapping for the entire region. This will prevent speculative
37 * reading of cached copies of our lines from being issued which will cause
38 * a PI FSB Protocol error to be generated by the SHUB. For XPC, we need 64
39 * amo variables (based on XP_MAX_NPARTITIONS_SN2) to identify the senders of
40 * NOTIFY IRQs, 128 amo variables (based on XP_NASID_MASK_WORDS_SN2) to identify
41 * the senders of ACTIVATE IRQs, 1 amo variable to identify which remote
42 * partitions (i.e., XPCs) consider themselves currently engaged with the
43 * local XPC and 1 amo variable to request partition deactivation.
44 */
45#define XPC_NOTIFY_IRQ_AMOS_SN2 0
46#define XPC_ACTIVATE_IRQ_AMOS_SN2 (XPC_NOTIFY_IRQ_AMOS_SN2 + \
47 XP_MAX_NPARTITIONS_SN2)
48#define XPC_ENGAGED_PARTITIONS_AMO_SN2 (XPC_ACTIVATE_IRQ_AMOS_SN2 + \
49 XP_NASID_MASK_WORDS_SN2)
50#define XPC_DEACTIVATE_REQUEST_AMO_SN2 (XPC_ENGAGED_PARTITIONS_AMO_SN2 + 1)
51
52/*
53 * Buffer used to store a local copy of portions of a remote partition's
54 * reserved page (either its header and part_nasids mask, or its vars).
55 */
Dean Nelsonee6665e2008-07-29 22:34:13 -070056static void *xpc_remote_copy_buffer_base_sn2;
Dean Nelson5b8669d2008-07-29 22:34:18 -070057static char *xpc_remote_copy_buffer_sn2;
Dean Nelsonee6665e2008-07-29 22:34:13 -070058
Dean Nelson8e85c232008-07-29 22:34:13 -070059static struct xpc_vars_sn2 *xpc_vars_sn2;
60static struct xpc_vars_part_sn2 *xpc_vars_part_sn2;
Dean Nelson94bd2702008-07-29 22:34:05 -070061
Dean Nelson5b8669d2008-07-29 22:34:18 -070062static int
63xpc_setup_partitions_sn_sn2(void)
64{
65 /* nothing needs to be done */
66 return 0;
67}
68
Dean Nelson6e41017a2008-07-29 22:34:09 -070069/* SH_IPI_ACCESS shub register value on startup */
Dean Nelson8e85c232008-07-29 22:34:13 -070070static u64 xpc_sh1_IPI_access_sn2;
71static u64 xpc_sh2_IPI_access0_sn2;
72static u64 xpc_sh2_IPI_access1_sn2;
73static u64 xpc_sh2_IPI_access2_sn2;
74static u64 xpc_sh2_IPI_access3_sn2;
Dean Nelson6e41017a2008-07-29 22:34:09 -070075
76/*
77 * Change protections to allow IPI operations.
78 */
79static void
80xpc_allow_IPI_ops_sn2(void)
81{
82 int node;
83 int nasid;
84
Dean Nelsonea57f802008-07-29 22:34:14 -070085 /* !!! The following should get moved into SAL. */
Dean Nelson6e41017a2008-07-29 22:34:09 -070086 if (is_shub2()) {
Dean Nelson8e85c232008-07-29 22:34:13 -070087 xpc_sh2_IPI_access0_sn2 =
Dean Nelson6e41017a2008-07-29 22:34:09 -070088 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS0));
Dean Nelson8e85c232008-07-29 22:34:13 -070089 xpc_sh2_IPI_access1_sn2 =
Dean Nelson6e41017a2008-07-29 22:34:09 -070090 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS1));
Dean Nelson8e85c232008-07-29 22:34:13 -070091 xpc_sh2_IPI_access2_sn2 =
Dean Nelson6e41017a2008-07-29 22:34:09 -070092 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS2));
Dean Nelson8e85c232008-07-29 22:34:13 -070093 xpc_sh2_IPI_access3_sn2 =
Dean Nelson6e41017a2008-07-29 22:34:09 -070094 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS3));
95
96 for_each_online_node(node) {
97 nasid = cnodeid_to_nasid(node);
98 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0),
99 -1UL);
100 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1),
101 -1UL);
102 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2),
103 -1UL);
104 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3),
105 -1UL);
106 }
107 } else {
Dean Nelson8e85c232008-07-29 22:34:13 -0700108 xpc_sh1_IPI_access_sn2 =
Dean Nelson6e41017a2008-07-29 22:34:09 -0700109 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH1_IPI_ACCESS));
110
111 for_each_online_node(node) {
112 nasid = cnodeid_to_nasid(node);
113 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS),
114 -1UL);
115 }
116 }
117}
118
119/*
120 * Restrict protections to disallow IPI operations.
121 */
122static void
123xpc_disallow_IPI_ops_sn2(void)
124{
125 int node;
126 int nasid;
127
Dean Nelsonea57f802008-07-29 22:34:14 -0700128 /* !!! The following should get moved into SAL. */
Dean Nelson6e41017a2008-07-29 22:34:09 -0700129 if (is_shub2()) {
130 for_each_online_node(node) {
131 nasid = cnodeid_to_nasid(node);
132 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0),
Dean Nelson8e85c232008-07-29 22:34:13 -0700133 xpc_sh2_IPI_access0_sn2);
Dean Nelson6e41017a2008-07-29 22:34:09 -0700134 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1),
Dean Nelson8e85c232008-07-29 22:34:13 -0700135 xpc_sh2_IPI_access1_sn2);
Dean Nelson6e41017a2008-07-29 22:34:09 -0700136 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2),
Dean Nelson8e85c232008-07-29 22:34:13 -0700137 xpc_sh2_IPI_access2_sn2);
Dean Nelson6e41017a2008-07-29 22:34:09 -0700138 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3),
Dean Nelson8e85c232008-07-29 22:34:13 -0700139 xpc_sh2_IPI_access3_sn2);
Dean Nelson6e41017a2008-07-29 22:34:09 -0700140 }
141 } else {
142 for_each_online_node(node) {
143 nasid = cnodeid_to_nasid(node);
144 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS),
Dean Nelson8e85c232008-07-29 22:34:13 -0700145 xpc_sh1_IPI_access_sn2);
Dean Nelson6e41017a2008-07-29 22:34:09 -0700146 }
147 }
148}
149
Dean Nelson33ba3c72008-07-29 22:34:07 -0700150/*
Dean Nelson7fb5e592008-07-29 22:34:10 -0700151 * The following set of functions are used for the sending and receiving of
152 * IRQs (also known as IPIs). There are two flavors of IRQs, one that is
153 * associated with partition activity (SGI_XPC_ACTIVATE) and the other that
154 * is associated with channel activity (SGI_XPC_NOTIFY).
Dean Nelson33ba3c72008-07-29 22:34:07 -0700155 */
156
157static u64
Dean Nelsonc39838c2008-07-29 22:34:11 -0700158xpc_receive_IRQ_amo_sn2(struct amo *amo)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700159{
160 return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_CLEAR);
161}
162
163static enum xp_retval
Dean Nelsonc39838c2008-07-29 22:34:11 -0700164xpc_send_IRQ_sn2(struct amo *amo, u64 flag, int nasid, int phys_cpuid,
165 int vector)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700166{
167 int ret = 0;
168 unsigned long irq_flags;
169
170 local_irq_save(irq_flags);
171
172 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, flag);
173 sn_send_IPI_phys(nasid, phys_cpuid, vector, 0);
174
175 /*
176 * We must always use the nofault function regardless of whether we
177 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
178 * didn't, we'd never know that the other partition is down and would
Dean Nelsonc39838c2008-07-29 22:34:11 -0700179 * keep sending IRQs and amos to it until the heartbeat times out.
Dean Nelson33ba3c72008-07-29 22:34:07 -0700180 */
181 ret = xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->variable),
182 xp_nofault_PIOR_target));
183
184 local_irq_restore(irq_flags);
185
Dean Nelson261f3b42008-07-29 22:34:16 -0700186 return (ret == 0) ? xpSuccess : xpPioReadError;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700187}
188
Dean Nelsonc39838c2008-07-29 22:34:11 -0700189static struct amo *
Dean Nelson7fb5e592008-07-29 22:34:10 -0700190xpc_init_IRQ_amo_sn2(int index)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700191{
Dean Nelson8e85c232008-07-29 22:34:13 -0700192 struct amo *amo = xpc_vars_sn2->amos_page + index;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700193
Dean Nelsonc39838c2008-07-29 22:34:11 -0700194 (void)xpc_receive_IRQ_amo_sn2(amo); /* clear amo variable */
Dean Nelson33ba3c72008-07-29 22:34:07 -0700195 return amo;
196}
197
198/*
Dean Nelson7fb5e592008-07-29 22:34:10 -0700199 * Functions associated with SGI_XPC_ACTIVATE IRQ.
Dean Nelson33ba3c72008-07-29 22:34:07 -0700200 */
201
202/*
Dean Nelson6e41017a2008-07-29 22:34:09 -0700203 * Notify the heartbeat check thread that an activate IRQ has been received.
204 */
205static irqreturn_t
206xpc_handle_activate_IRQ_sn2(int irq, void *dev_id)
207{
Dean Nelson5b8669d2008-07-29 22:34:18 -0700208 unsigned long irq_flags;
209
210 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
211 xpc_activate_IRQ_rcvd++;
212 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
213
Dean Nelson6e41017a2008-07-29 22:34:09 -0700214 wake_up_interruptible(&xpc_activate_IRQ_wq);
215 return IRQ_HANDLED;
216}
217
218/*
Dean Nelsonc39838c2008-07-29 22:34:11 -0700219 * Flag the appropriate amo variable and send an IRQ to the specified node.
Dean Nelson33ba3c72008-07-29 22:34:07 -0700220 */
221static void
Dean Nelsona812dcc2008-07-29 22:34:16 -0700222xpc_send_activate_IRQ_sn2(unsigned long amos_page_pa, int from_nasid,
223 int to_nasid, int to_phys_cpuid)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700224{
Dean Nelsonc39838c2008-07-29 22:34:11 -0700225 struct amo *amos = (struct amo *)__va(amos_page_pa +
Dean Nelsonee6665e2008-07-29 22:34:13 -0700226 (XPC_ACTIVATE_IRQ_AMOS_SN2 *
Dean Nelsonc39838c2008-07-29 22:34:11 -0700227 sizeof(struct amo)));
Dean Nelson33ba3c72008-07-29 22:34:07 -0700228
Dean Nelson04de7412008-07-29 22:34:14 -0700229 (void)xpc_send_IRQ_sn2(&amos[BIT_WORD(from_nasid / 2)],
230 BIT_MASK(from_nasid / 2), to_nasid,
Dean Nelson33ba3c72008-07-29 22:34:07 -0700231 to_phys_cpuid, SGI_XPC_ACTIVATE);
232}
233
234static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700235xpc_send_local_activate_IRQ_sn2(int from_nasid)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700236{
Dean Nelson5b8669d2008-07-29 22:34:18 -0700237 unsigned long irq_flags;
Dean Nelson8e85c232008-07-29 22:34:13 -0700238 struct amo *amos = (struct amo *)__va(xpc_vars_sn2->amos_page_pa +
Dean Nelsonee6665e2008-07-29 22:34:13 -0700239 (XPC_ACTIVATE_IRQ_AMOS_SN2 *
Dean Nelsonc39838c2008-07-29 22:34:11 -0700240 sizeof(struct amo)));
Dean Nelson33ba3c72008-07-29 22:34:07 -0700241
242 /* fake the sending and receipt of an activate IRQ from remote nasid */
Dean Nelson04de7412008-07-29 22:34:14 -0700243 FETCHOP_STORE_OP(TO_AMO((u64)&amos[BIT_WORD(from_nasid / 2)].variable),
244 FETCHOP_OR, BIT_MASK(from_nasid / 2));
245
Dean Nelson5b8669d2008-07-29 22:34:18 -0700246 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
247 xpc_activate_IRQ_rcvd++;
248 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
249
Dean Nelson6e41017a2008-07-29 22:34:09 -0700250 wake_up_interruptible(&xpc_activate_IRQ_wq);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700251}
252
Dean Nelson33ba3c72008-07-29 22:34:07 -0700253/*
Dean Nelson7fb5e592008-07-29 22:34:10 -0700254 * Functions associated with SGI_XPC_NOTIFY IRQ.
Dean Nelson33ba3c72008-07-29 22:34:07 -0700255 */
256
257/*
Dean Nelson7fb5e592008-07-29 22:34:10 -0700258 * Check to see if any chctl flags were sent from the specified partition.
Dean Nelsona47d5da2008-07-29 22:34:09 -0700259 */
260static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700261xpc_check_for_sent_chctl_flags_sn2(struct xpc_partition *part)
Dean Nelsona47d5da2008-07-29 22:34:09 -0700262{
Dean Nelson7fb5e592008-07-29 22:34:10 -0700263 union xpc_channel_ctl_flags chctl;
Dean Nelsona47d5da2008-07-29 22:34:09 -0700264 unsigned long irq_flags;
265
Dean Nelson7fb5e592008-07-29 22:34:10 -0700266 chctl.all_flags = xpc_receive_IRQ_amo_sn2(part->sn.sn2.
267 local_chctl_amo_va);
268 if (chctl.all_flags == 0)
Dean Nelsona47d5da2008-07-29 22:34:09 -0700269 return;
270
Dean Nelson7fb5e592008-07-29 22:34:10 -0700271 spin_lock_irqsave(&part->chctl_lock, irq_flags);
272 part->chctl.all_flags |= chctl.all_flags;
273 spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
Dean Nelsona47d5da2008-07-29 22:34:09 -0700274
Dean Nelson7fb5e592008-07-29 22:34:10 -0700275 dev_dbg(xpc_chan, "received notify IRQ from partid=%d, chctl.all_flags="
276 "0x%lx\n", XPC_PARTID(part), chctl.all_flags);
Dean Nelsona47d5da2008-07-29 22:34:09 -0700277
278 xpc_wakeup_channel_mgr(part);
279}
280
281/*
282 * Handle the receipt of a SGI_XPC_NOTIFY IRQ by seeing whether the specified
283 * partition actually sent it. Since SGI_XPC_NOTIFY IRQs may be shared by more
Dean Nelsonc39838c2008-07-29 22:34:11 -0700284 * than one partition, we use an amo structure per partition to indicate
Dean Nelson7fb5e592008-07-29 22:34:10 -0700285 * whether a partition has sent an IRQ or not. If it has, then wake up the
Dean Nelsona47d5da2008-07-29 22:34:09 -0700286 * associated kthread to handle it.
287 *
Dean Nelson7fb5e592008-07-29 22:34:10 -0700288 * All SGI_XPC_NOTIFY IRQs received by XPC are the result of IRQs sent by XPC
Dean Nelsona47d5da2008-07-29 22:34:09 -0700289 * running on other partitions.
290 *
291 * Noteworthy Arguments:
292 *
293 * irq - Interrupt ReQuest number. NOT USED.
294 *
Dean Nelson7fb5e592008-07-29 22:34:10 -0700295 * dev_id - partid of IRQ's potential sender.
Dean Nelsona47d5da2008-07-29 22:34:09 -0700296 */
297static irqreturn_t
298xpc_handle_notify_IRQ_sn2(int irq, void *dev_id)
299{
300 short partid = (short)(u64)dev_id;
301 struct xpc_partition *part = &xpc_partitions[partid];
302
Dean Nelson261f3b42008-07-29 22:34:16 -0700303 DBUG_ON(partid < 0 || partid >= XP_MAX_NPARTITIONS_SN2);
Dean Nelsona47d5da2008-07-29 22:34:09 -0700304
305 if (xpc_part_ref(part)) {
Dean Nelson7fb5e592008-07-29 22:34:10 -0700306 xpc_check_for_sent_chctl_flags_sn2(part);
Dean Nelsona47d5da2008-07-29 22:34:09 -0700307
308 xpc_part_deref(part);
309 }
310 return IRQ_HANDLED;
311}
312
313/*
Dean Nelson7fb5e592008-07-29 22:34:10 -0700314 * Check to see if xpc_handle_notify_IRQ_sn2() dropped any IRQs on the floor
315 * because the write to their associated amo variable completed after the IRQ
Dean Nelsona47d5da2008-07-29 22:34:09 -0700316 * was received.
317 */
318static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700319xpc_check_for_dropped_notify_IRQ_sn2(struct xpc_partition *part)
Dean Nelsona47d5da2008-07-29 22:34:09 -0700320{
321 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
322
323 if (xpc_part_ref(part)) {
Dean Nelson7fb5e592008-07-29 22:34:10 -0700324 xpc_check_for_sent_chctl_flags_sn2(part);
Dean Nelsona47d5da2008-07-29 22:34:09 -0700325
326 part_sn2->dropped_notify_IRQ_timer.expires = jiffies +
Dean Nelson7fb5e592008-07-29 22:34:10 -0700327 XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL;
Dean Nelsona47d5da2008-07-29 22:34:09 -0700328 add_timer(&part_sn2->dropped_notify_IRQ_timer);
329 xpc_part_deref(part);
330 }
331}
332
333/*
Dean Nelson7fb5e592008-07-29 22:34:10 -0700334 * Send a notify IRQ to the remote partition that is associated with the
Dean Nelson33ba3c72008-07-29 22:34:07 -0700335 * specified channel.
336 */
337static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700338xpc_send_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag,
339 char *chctl_flag_string, unsigned long *irq_flags)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700340{
341 struct xpc_partition *part = &xpc_partitions[ch->partid];
Dean Nelsona47d5da2008-07-29 22:34:09 -0700342 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
Dean Nelson7fb5e592008-07-29 22:34:10 -0700343 union xpc_channel_ctl_flags chctl = { 0 };
Dean Nelson33ba3c72008-07-29 22:34:07 -0700344 enum xp_retval ret;
345
Dean Nelson83469b52008-07-29 22:34:18 -0700346 if (likely(part->act_state != XPC_P_AS_DEACTIVATING)) {
Dean Nelson7fb5e592008-07-29 22:34:10 -0700347 chctl.flags[ch->number] = chctl_flag;
348 ret = xpc_send_IRQ_sn2(part_sn2->remote_chctl_amo_va,
349 chctl.all_flags,
350 part_sn2->notify_IRQ_nasid,
351 part_sn2->notify_IRQ_phys_cpuid,
Dean Nelson33ba3c72008-07-29 22:34:07 -0700352 SGI_XPC_NOTIFY);
353 dev_dbg(xpc_chan, "%s sent to partid=%d, channel=%d, ret=%d\n",
Dean Nelson7fb5e592008-07-29 22:34:10 -0700354 chctl_flag_string, ch->partid, ch->number, ret);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700355 if (unlikely(ret != xpSuccess)) {
356 if (irq_flags != NULL)
357 spin_unlock_irqrestore(&ch->lock, *irq_flags);
358 XPC_DEACTIVATE_PARTITION(part, ret);
359 if (irq_flags != NULL)
360 spin_lock_irqsave(&ch->lock, *irq_flags);
361 }
362 }
363}
364
Dean Nelson7fb5e592008-07-29 22:34:10 -0700365#define XPC_SEND_NOTIFY_IRQ_SN2(_ch, _ipi_f, _irq_f) \
366 xpc_send_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f, _irq_f)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700367
368/*
369 * Make it look like the remote partition, which is associated with the
Dean Nelson7fb5e592008-07-29 22:34:10 -0700370 * specified channel, sent us a notify IRQ. This faked IRQ will be handled
371 * by xpc_check_for_dropped_notify_IRQ_sn2().
Dean Nelson33ba3c72008-07-29 22:34:07 -0700372 */
373static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700374xpc_send_local_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag,
375 char *chctl_flag_string)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700376{
377 struct xpc_partition *part = &xpc_partitions[ch->partid];
Dean Nelson7fb5e592008-07-29 22:34:10 -0700378 union xpc_channel_ctl_flags chctl = { 0 };
Dean Nelson33ba3c72008-07-29 22:34:07 -0700379
Dean Nelson7fb5e592008-07-29 22:34:10 -0700380 chctl.flags[ch->number] = chctl_flag;
381 FETCHOP_STORE_OP(TO_AMO((u64)&part->sn.sn2.local_chctl_amo_va->
382 variable), FETCHOP_OR, chctl.all_flags);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700383 dev_dbg(xpc_chan, "%s sent local from partid=%d, channel=%d\n",
Dean Nelson7fb5e592008-07-29 22:34:10 -0700384 chctl_flag_string, ch->partid, ch->number);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700385}
386
Dean Nelson7fb5e592008-07-29 22:34:10 -0700387#define XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(_ch, _ipi_f) \
388 xpc_send_local_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700389
390static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700391xpc_send_chctl_closerequest_sn2(struct xpc_channel *ch,
392 unsigned long *irq_flags)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700393{
Dean Nelson5b8669d2008-07-29 22:34:18 -0700394 struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700395
396 args->reason = ch->reason;
Dean Nelson7fb5e592008-07-29 22:34:10 -0700397 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREQUEST, irq_flags);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700398}
399
400static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700401xpc_send_chctl_closereply_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700402{
Dean Nelson7fb5e592008-07-29 22:34:10 -0700403 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREPLY, irq_flags);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700404}
405
406static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700407xpc_send_chctl_openrequest_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700408{
Dean Nelson5b8669d2008-07-29 22:34:18 -0700409 struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700410
Dean Nelsonbd3e64c2008-07-29 22:34:19 -0700411 args->entry_size = ch->entry_size;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700412 args->local_nentries = ch->local_nentries;
Dean Nelson7fb5e592008-07-29 22:34:10 -0700413 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREQUEST, irq_flags);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700414}
415
416static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700417xpc_send_chctl_openreply_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700418{
Dean Nelson5b8669d2008-07-29 22:34:18 -0700419 struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700420
421 args->remote_nentries = ch->remote_nentries;
422 args->local_nentries = ch->local_nentries;
Dean Nelson5b8669d2008-07-29 22:34:18 -0700423 args->local_msgqueue_pa = xp_pa(ch->sn.sn2.local_msgqueue);
Dean Nelson7fb5e592008-07-29 22:34:10 -0700424 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREPLY, irq_flags);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700425}
426
427static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700428xpc_send_chctl_msgrequest_sn2(struct xpc_channel *ch)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700429{
Dean Nelson7fb5e592008-07-29 22:34:10 -0700430 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST, NULL);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700431}
432
433static void
Dean Nelson7fb5e592008-07-29 22:34:10 -0700434xpc_send_chctl_local_msgrequest_sn2(struct xpc_channel *ch)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700435{
Dean Nelson7fb5e592008-07-29 22:34:10 -0700436 XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700437}
438
Dean Nelson5b8669d2008-07-29 22:34:18 -0700439static void
440xpc_save_remote_msgqueue_pa_sn2(struct xpc_channel *ch,
441 unsigned long msgqueue_pa)
442{
443 ch->sn.sn2.remote_msgqueue_pa = msgqueue_pa;
444}
445
Dean Nelson33ba3c72008-07-29 22:34:07 -0700446/*
447 * This next set of functions are used to keep track of when a partition is
448 * potentially engaged in accessing memory belonging to another partition.
449 */
450
451static void
Dean Nelsona47d5da2008-07-29 22:34:09 -0700452xpc_indicate_partition_engaged_sn2(struct xpc_partition *part)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700453{
454 unsigned long irq_flags;
Dean Nelsonc39838c2008-07-29 22:34:11 -0700455 struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa +
Dean Nelsonee6665e2008-07-29 22:34:13 -0700456 (XPC_ENGAGED_PARTITIONS_AMO_SN2 *
Dean Nelsonc39838c2008-07-29 22:34:11 -0700457 sizeof(struct amo)));
Dean Nelson33ba3c72008-07-29 22:34:07 -0700458
459 local_irq_save(irq_flags);
460
Dean Nelsonc39838c2008-07-29 22:34:11 -0700461 /* set bit corresponding to our partid in remote partition's amo */
Dean Nelson33ba3c72008-07-29 22:34:07 -0700462 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR,
Dean Nelson04de7412008-07-29 22:34:14 -0700463 BIT(sn_partition_id));
464
Dean Nelson33ba3c72008-07-29 22:34:07 -0700465 /*
466 * We must always use the nofault function regardless of whether we
467 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
468 * didn't, we'd never know that the other partition is down and would
Dean Nelsonc39838c2008-07-29 22:34:11 -0700469 * keep sending IRQs and amos to it until the heartbeat times out.
Dean Nelson33ba3c72008-07-29 22:34:07 -0700470 */
471 (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
472 variable),
473 xp_nofault_PIOR_target));
474
475 local_irq_restore(irq_flags);
476}
477
478static void
Dean Nelsona47d5da2008-07-29 22:34:09 -0700479xpc_indicate_partition_disengaged_sn2(struct xpc_partition *part)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700480{
Dean Nelsona47d5da2008-07-29 22:34:09 -0700481 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700482 unsigned long irq_flags;
Dean Nelsonc39838c2008-07-29 22:34:11 -0700483 struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa +
Dean Nelsonee6665e2008-07-29 22:34:13 -0700484 (XPC_ENGAGED_PARTITIONS_AMO_SN2 *
Dean Nelsonc39838c2008-07-29 22:34:11 -0700485 sizeof(struct amo)));
Dean Nelson33ba3c72008-07-29 22:34:07 -0700486
487 local_irq_save(irq_flags);
488
Dean Nelsonc39838c2008-07-29 22:34:11 -0700489 /* clear bit corresponding to our partid in remote partition's amo */
Dean Nelson33ba3c72008-07-29 22:34:07 -0700490 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND,
Dean Nelson04de7412008-07-29 22:34:14 -0700491 ~BIT(sn_partition_id));
492
Dean Nelson33ba3c72008-07-29 22:34:07 -0700493 /*
494 * We must always use the nofault function regardless of whether we
495 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
496 * didn't, we'd never know that the other partition is down and would
Dean Nelsonc39838c2008-07-29 22:34:11 -0700497 * keep sending IRQs and amos to it until the heartbeat times out.
Dean Nelson33ba3c72008-07-29 22:34:07 -0700498 */
499 (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
500 variable),
501 xp_nofault_PIOR_target));
502
503 local_irq_restore(irq_flags);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700504
Dean Nelson33ba3c72008-07-29 22:34:07 -0700505 /*
Dean Nelsona47d5da2008-07-29 22:34:09 -0700506 * Send activate IRQ to get other side to see that we've cleared our
Dean Nelsonc39838c2008-07-29 22:34:11 -0700507 * bit in their engaged partitions amo.
Dean Nelson33ba3c72008-07-29 22:34:07 -0700508 */
Dean Nelson7fb5e592008-07-29 22:34:10 -0700509 xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa,
Dean Nelsona47d5da2008-07-29 22:34:09 -0700510 cnodeid_to_nasid(0),
511 part_sn2->activate_IRQ_nasid,
512 part_sn2->activate_IRQ_phys_cpuid);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700513}
514
Dean Nelson5b8669d2008-07-29 22:34:18 -0700515static void
516xpc_assume_partition_disengaged_sn2(short partid)
517{
518 struct amo *amo = xpc_vars_sn2->amos_page +
519 XPC_ENGAGED_PARTITIONS_AMO_SN2;
520
521 /* clear bit(s) based on partid mask in our partition's amo */
522 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND,
523 ~BIT(partid));
524}
525
Dean Nelsona47d5da2008-07-29 22:34:09 -0700526static int
527xpc_partition_engaged_sn2(short partid)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700528{
Dean Nelson8e85c232008-07-29 22:34:13 -0700529 struct amo *amo = xpc_vars_sn2->amos_page +
530 XPC_ENGAGED_PARTITIONS_AMO_SN2;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700531
Dean Nelsonc39838c2008-07-29 22:34:11 -0700532 /* our partition's amo variable ANDed with partid mask */
Dean Nelson33ba3c72008-07-29 22:34:07 -0700533 return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) &
Dean Nelson04de7412008-07-29 22:34:14 -0700534 BIT(partid)) != 0;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700535}
536
Dean Nelsona47d5da2008-07-29 22:34:09 -0700537static int
538xpc_any_partition_engaged_sn2(void)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700539{
Dean Nelson8e85c232008-07-29 22:34:13 -0700540 struct amo *amo = xpc_vars_sn2->amos_page +
541 XPC_ENGAGED_PARTITIONS_AMO_SN2;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700542
Dean Nelsonc39838c2008-07-29 22:34:11 -0700543 /* our partition's amo variable */
Dean Nelsona47d5da2008-07-29 22:34:09 -0700544 return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) != 0;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700545}
546
Dean Nelson6e41017a2008-07-29 22:34:09 -0700547/* original protection values for each node */
548static u64 xpc_prot_vec_sn2[MAX_NUMNODES];
549
550/*
Dean Nelsonc39838c2008-07-29 22:34:11 -0700551 * Change protections to allow amo operations on non-Shub 1.1 systems.
Dean Nelson6e41017a2008-07-29 22:34:09 -0700552 */
553static enum xp_retval
Dean Nelsonc39838c2008-07-29 22:34:11 -0700554xpc_allow_amo_ops_sn2(struct amo *amos_page)
Dean Nelson6e41017a2008-07-29 22:34:09 -0700555{
Dean Nelson6c1c3252008-11-05 17:27:22 -0600556 enum xp_retval ret = xpSuccess;
Dean Nelson6e41017a2008-07-29 22:34:09 -0700557
558 /*
559 * On SHUB 1.1, we cannot call sn_change_memprotect() since the BIST
560 * collides with memory operations. On those systems we call
Dean Nelsonc39838c2008-07-29 22:34:11 -0700561 * xpc_allow_amo_ops_shub_wars_1_1_sn2() instead.
Dean Nelson6e41017a2008-07-29 22:34:09 -0700562 */
Dean Nelson6c1c3252008-11-05 17:27:22 -0600563 if (!enable_shub_wars_1_1())
564 ret = xp_expand_memprotect(ia64_tpa((u64)amos_page), PAGE_SIZE);
565
566 return ret;
Dean Nelson6e41017a2008-07-29 22:34:09 -0700567}
568
569/*
Dean Nelsonc39838c2008-07-29 22:34:11 -0700570 * Change protections to allow amo operations on Shub 1.1 systems.
Dean Nelson6e41017a2008-07-29 22:34:09 -0700571 */
572static void
Dean Nelsonc39838c2008-07-29 22:34:11 -0700573xpc_allow_amo_ops_shub_wars_1_1_sn2(void)
Dean Nelson6e41017a2008-07-29 22:34:09 -0700574{
575 int node;
576 int nasid;
577
578 if (!enable_shub_wars_1_1())
579 return;
580
581 for_each_online_node(node) {
582 nasid = cnodeid_to_nasid(node);
583 /* save current protection values */
584 xpc_prot_vec_sn2[node] =
585 (u64)HUB_L((u64 *)GLOBAL_MMR_ADDR(nasid,
586 SH1_MD_DQLP_MMR_DIR_PRIVEC0));
587 /* open up everything */
588 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid,
589 SH1_MD_DQLP_MMR_DIR_PRIVEC0),
590 -1UL);
591 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid,
592 SH1_MD_DQRP_MMR_DIR_PRIVEC0),
593 -1UL);
594 }
595}
596
Dean Nelson94bd2702008-07-29 22:34:05 -0700597static enum xp_retval
Dean Nelsona812dcc2008-07-29 22:34:16 -0700598xpc_get_partition_rsvd_page_pa_sn2(void *buf, u64 *cookie, unsigned long *rp_pa,
Dean Nelson261f3b42008-07-29 22:34:16 -0700599 size_t *len)
600{
601 s64 status;
602 enum xp_retval ret;
603
Dean Nelsona812dcc2008-07-29 22:34:16 -0700604 status = sn_partition_reserved_page_pa((u64)buf, cookie, rp_pa, len);
Dean Nelson261f3b42008-07-29 22:34:16 -0700605 if (status == SALRET_OK)
606 ret = xpSuccess;
607 else if (status == SALRET_MORE_PASSES)
608 ret = xpNeedMoreInfo;
609 else
610 ret = xpSalError;
611
612 return ret;
613}
614
615
Dean Nelson5b8669d2008-07-29 22:34:18 -0700616static int
617xpc_setup_rsvd_page_sn_sn2(struct xpc_rsvd_page *rp)
Dean Nelson94bd2702008-07-29 22:34:05 -0700618{
Dean Nelsonc39838c2008-07-29 22:34:11 -0700619 struct amo *amos_page;
Dean Nelson94bd2702008-07-29 22:34:05 -0700620 int i;
621 int ret;
622
Dean Nelson8e85c232008-07-29 22:34:13 -0700623 xpc_vars_sn2 = XPC_RP_VARS(rp);
Dean Nelson94bd2702008-07-29 22:34:05 -0700624
Dean Nelsona812dcc2008-07-29 22:34:16 -0700625 rp->sn.vars_pa = xp_pa(xpc_vars_sn2);
Dean Nelson94bd2702008-07-29 22:34:05 -0700626
Dean Nelsone17d4162008-07-29 22:34:06 -0700627 /* vars_part array follows immediately after vars */
Dean Nelson8e85c232008-07-29 22:34:13 -0700628 xpc_vars_part_sn2 = (struct xpc_vars_part_sn2 *)((u8 *)XPC_RP_VARS(rp) +
629 XPC_RP_VARS_SIZE);
Dean Nelsone17d4162008-07-29 22:34:06 -0700630
Dean Nelson94bd2702008-07-29 22:34:05 -0700631 /*
Dean Nelson8e85c232008-07-29 22:34:13 -0700632 * Before clearing xpc_vars_sn2, see if a page of amos had been
633 * previously allocated. If not we'll need to allocate one and set
634 * permissions so that cross-partition amos are allowed.
Dean Nelson94bd2702008-07-29 22:34:05 -0700635 *
Dean Nelsonc39838c2008-07-29 22:34:11 -0700636 * The allocated amo page needs MCA reporting to remain disabled after
Dean Nelson94bd2702008-07-29 22:34:05 -0700637 * XPC has unloaded. To make this work, we keep a copy of the pointer
Dean Nelson8e85c232008-07-29 22:34:13 -0700638 * to this page (i.e., amos_page) in the struct xpc_vars_sn2 structure,
Dean Nelson94bd2702008-07-29 22:34:05 -0700639 * which is pointed to by the reserved page, and re-use that saved copy
Dean Nelsonc39838c2008-07-29 22:34:11 -0700640 * on subsequent loads of XPC. This amo page is never freed, and its
Dean Nelson94bd2702008-07-29 22:34:05 -0700641 * memory protections are never restricted.
642 */
Dean Nelson8e85c232008-07-29 22:34:13 -0700643 amos_page = xpc_vars_sn2->amos_page;
Dean Nelson94bd2702008-07-29 22:34:05 -0700644 if (amos_page == NULL) {
Dean Nelsonc39838c2008-07-29 22:34:11 -0700645 amos_page = (struct amo *)TO_AMO(uncached_alloc_page(0, 1));
Dean Nelson94bd2702008-07-29 22:34:05 -0700646 if (amos_page == NULL) {
Dean Nelsonc39838c2008-07-29 22:34:11 -0700647 dev_err(xpc_part, "can't allocate page of amos\n");
Dean Nelson5b8669d2008-07-29 22:34:18 -0700648 return -ENOMEM;
Dean Nelson94bd2702008-07-29 22:34:05 -0700649 }
650
651 /*
Dean Nelsonc39838c2008-07-29 22:34:11 -0700652 * Open up amo-R/W to cpu. This is done on Shub 1.1 systems
653 * when xpc_allow_amo_ops_shub_wars_1_1_sn2() is called.
Dean Nelson94bd2702008-07-29 22:34:05 -0700654 */
Dean Nelsonc39838c2008-07-29 22:34:11 -0700655 ret = xpc_allow_amo_ops_sn2(amos_page);
Dean Nelson6e41017a2008-07-29 22:34:09 -0700656 if (ret != xpSuccess) {
Dean Nelsonc39838c2008-07-29 22:34:11 -0700657 dev_err(xpc_part, "can't allow amo operations\n");
Dean Nelson6e41017a2008-07-29 22:34:09 -0700658 uncached_free_page(__IA64_UNCACHED_OFFSET |
659 TO_PHYS((u64)amos_page), 1);
Dean Nelson5b8669d2008-07-29 22:34:18 -0700660 return -EPERM;
Dean Nelson94bd2702008-07-29 22:34:05 -0700661 }
662 }
663
Dean Nelson8e85c232008-07-29 22:34:13 -0700664 /* clear xpc_vars_sn2 */
665 memset(xpc_vars_sn2, 0, sizeof(struct xpc_vars_sn2));
Dean Nelson94bd2702008-07-29 22:34:05 -0700666
Dean Nelson8e85c232008-07-29 22:34:13 -0700667 xpc_vars_sn2->version = XPC_V_VERSION;
668 xpc_vars_sn2->activate_IRQ_nasid = cpuid_to_nasid(0);
669 xpc_vars_sn2->activate_IRQ_phys_cpuid = cpu_physical_id(0);
Dean Nelsona812dcc2008-07-29 22:34:16 -0700670 xpc_vars_sn2->vars_part_pa = xp_pa(xpc_vars_part_sn2);
Dean Nelson8e85c232008-07-29 22:34:13 -0700671 xpc_vars_sn2->amos_page_pa = ia64_tpa((u64)amos_page);
672 xpc_vars_sn2->amos_page = amos_page; /* save for next load of XPC */
Dean Nelson94bd2702008-07-29 22:34:05 -0700673
Dean Nelson8e85c232008-07-29 22:34:13 -0700674 /* clear xpc_vars_part_sn2 */
675 memset((u64 *)xpc_vars_part_sn2, 0, sizeof(struct xpc_vars_part_sn2) *
Dean Nelson261f3b42008-07-29 22:34:16 -0700676 XP_MAX_NPARTITIONS_SN2);
Dean Nelson94bd2702008-07-29 22:34:05 -0700677
Dean Nelsonc39838c2008-07-29 22:34:11 -0700678 /* initialize the activate IRQ related amo variables */
Dean Nelson04de7412008-07-29 22:34:14 -0700679 for (i = 0; i < xpc_nasid_mask_nlongs; i++)
Dean Nelsonee6665e2008-07-29 22:34:13 -0700680 (void)xpc_init_IRQ_amo_sn2(XPC_ACTIVATE_IRQ_AMOS_SN2 + i);
Dean Nelson94bd2702008-07-29 22:34:05 -0700681
Dean Nelsonc39838c2008-07-29 22:34:11 -0700682 /* initialize the engaged remote partitions related amo variables */
Dean Nelsonee6665e2008-07-29 22:34:13 -0700683 (void)xpc_init_IRQ_amo_sn2(XPC_ENGAGED_PARTITIONS_AMO_SN2);
684 (void)xpc_init_IRQ_amo_sn2(XPC_DEACTIVATE_REQUEST_AMO_SN2);
Dean Nelson94bd2702008-07-29 22:34:05 -0700685
Dean Nelson5b8669d2008-07-29 22:34:18 -0700686 return 0;
Dean Nelson94bd2702008-07-29 22:34:05 -0700687}
688
Dean Nelson33ba3c72008-07-29 22:34:07 -0700689static void
690xpc_increment_heartbeat_sn2(void)
691{
Dean Nelson8e85c232008-07-29 22:34:13 -0700692 xpc_vars_sn2->heartbeat++;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700693}
694
695static void
696xpc_offline_heartbeat_sn2(void)
697{
698 xpc_increment_heartbeat_sn2();
Dean Nelson8e85c232008-07-29 22:34:13 -0700699 xpc_vars_sn2->heartbeat_offline = 1;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700700}
701
702static void
703xpc_online_heartbeat_sn2(void)
704{
705 xpc_increment_heartbeat_sn2();
Dean Nelson8e85c232008-07-29 22:34:13 -0700706 xpc_vars_sn2->heartbeat_offline = 0;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700707}
708
709static void
710xpc_heartbeat_init_sn2(void)
711{
Dean Nelson8e85c232008-07-29 22:34:13 -0700712 DBUG_ON(xpc_vars_sn2 == NULL);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700713
Dean Nelson8e85c232008-07-29 22:34:13 -0700714 bitmap_zero(xpc_vars_sn2->heartbeating_to_mask, XP_MAX_NPARTITIONS_SN2);
715 xpc_heartbeating_to_mask = &xpc_vars_sn2->heartbeating_to_mask[0];
Dean Nelson33ba3c72008-07-29 22:34:07 -0700716 xpc_online_heartbeat_sn2();
717}
718
719static void
720xpc_heartbeat_exit_sn2(void)
721{
722 xpc_offline_heartbeat_sn2();
723}
724
Dean Nelson61deb862008-07-29 22:34:17 -0700725static enum xp_retval
726xpc_get_remote_heartbeat_sn2(struct xpc_partition *part)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700727{
728 struct xpc_vars_sn2 *remote_vars;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700729 enum xp_retval ret;
730
Dean Nelsonee6665e2008-07-29 22:34:13 -0700731 remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700732
Dean Nelson61deb862008-07-29 22:34:17 -0700733 /* pull the remote vars structure that contains the heartbeat */
734 ret = xp_remote_memcpy(xp_pa(remote_vars),
735 part->sn.sn2.remote_vars_pa,
736 XPC_RP_VARS_SIZE);
737 if (ret != xpSuccess)
738 return ret;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700739
Dean Nelson61deb862008-07-29 22:34:17 -0700740 dev_dbg(xpc_part, "partid=%d, heartbeat=%ld, last_heartbeat=%ld, "
741 "heartbeat_offline=%ld, HB_mask[0]=0x%lx\n", XPC_PARTID(part),
742 remote_vars->heartbeat, part->last_heartbeat,
743 remote_vars->heartbeat_offline,
744 remote_vars->heartbeating_to_mask[0]);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700745
Dean Nelson61deb862008-07-29 22:34:17 -0700746 if ((remote_vars->heartbeat == part->last_heartbeat &&
747 remote_vars->heartbeat_offline == 0) ||
748 !xpc_hb_allowed(sn_partition_id,
749 &remote_vars->heartbeating_to_mask)) {
750 ret = xpNoHeartbeat;
751 } else {
Dean Nelson33ba3c72008-07-29 22:34:07 -0700752 part->last_heartbeat = remote_vars->heartbeat;
753 }
Dean Nelson61deb862008-07-29 22:34:17 -0700754
755 return ret;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700756}
757
758/*
759 * Get a copy of the remote partition's XPC variables from the reserved page.
760 *
761 * remote_vars points to a buffer that is cacheline aligned for BTE copies and
762 * assumed to be of size XPC_RP_VARS_SIZE.
763 */
764static enum xp_retval
Dean Nelsona812dcc2008-07-29 22:34:16 -0700765xpc_get_remote_vars_sn2(unsigned long remote_vars_pa,
766 struct xpc_vars_sn2 *remote_vars)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700767{
768 enum xp_retval ret;
769
770 if (remote_vars_pa == 0)
771 return xpVarsNotSet;
772
773 /* pull over the cross partition variables */
Dean Nelsona812dcc2008-07-29 22:34:16 -0700774 ret = xp_remote_memcpy(xp_pa(remote_vars), remote_vars_pa,
Dean Nelson33ba3c72008-07-29 22:34:07 -0700775 XPC_RP_VARS_SIZE);
776 if (ret != xpSuccess)
777 return ret;
778
779 if (XPC_VERSION_MAJOR(remote_vars->version) !=
780 XPC_VERSION_MAJOR(XPC_V_VERSION)) {
781 return xpBadVersion;
782 }
783
784 return xpSuccess;
785}
786
787static void
Dean Nelsona47d5da2008-07-29 22:34:09 -0700788xpc_request_partition_activation_sn2(struct xpc_rsvd_page *remote_rp,
Dean Nelsona812dcc2008-07-29 22:34:16 -0700789 unsigned long remote_rp_pa, int nasid)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700790{
Dean Nelson7fb5e592008-07-29 22:34:10 -0700791 xpc_send_local_activate_IRQ_sn2(nasid);
Dean Nelsona47d5da2008-07-29 22:34:09 -0700792}
793
794static void
795xpc_request_partition_reactivation_sn2(struct xpc_partition *part)
796{
Dean Nelson7fb5e592008-07-29 22:34:10 -0700797 xpc_send_local_activate_IRQ_sn2(part->sn.sn2.activate_IRQ_nasid);
Dean Nelsona47d5da2008-07-29 22:34:09 -0700798}
799
800static void
801xpc_request_partition_deactivation_sn2(struct xpc_partition *part)
802{
803 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
804 unsigned long irq_flags;
Dean Nelsonc39838c2008-07-29 22:34:11 -0700805 struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa +
Dean Nelsonee6665e2008-07-29 22:34:13 -0700806 (XPC_DEACTIVATE_REQUEST_AMO_SN2 *
Dean Nelsonc39838c2008-07-29 22:34:11 -0700807 sizeof(struct amo)));
Dean Nelsona47d5da2008-07-29 22:34:09 -0700808
809 local_irq_save(irq_flags);
810
Dean Nelsonc39838c2008-07-29 22:34:11 -0700811 /* set bit corresponding to our partid in remote partition's amo */
Dean Nelsona47d5da2008-07-29 22:34:09 -0700812 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR,
Dean Nelson04de7412008-07-29 22:34:14 -0700813 BIT(sn_partition_id));
814
Dean Nelsona47d5da2008-07-29 22:34:09 -0700815 /*
816 * We must always use the nofault function regardless of whether we
817 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
818 * didn't, we'd never know that the other partition is down and would
Dean Nelsonc39838c2008-07-29 22:34:11 -0700819 * keep sending IRQs and amos to it until the heartbeat times out.
Dean Nelsona47d5da2008-07-29 22:34:09 -0700820 */
821 (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
822 variable),
823 xp_nofault_PIOR_target));
824
825 local_irq_restore(irq_flags);
826
827 /*
828 * Send activate IRQ to get other side to see that we've set our
Dean Nelsonc39838c2008-07-29 22:34:11 -0700829 * bit in their deactivate request amo.
Dean Nelsona47d5da2008-07-29 22:34:09 -0700830 */
Dean Nelson7fb5e592008-07-29 22:34:10 -0700831 xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa,
Dean Nelsona47d5da2008-07-29 22:34:09 -0700832 cnodeid_to_nasid(0),
833 part_sn2->activate_IRQ_nasid,
834 part_sn2->activate_IRQ_phys_cpuid);
835}
836
837static void
838xpc_cancel_partition_deactivation_request_sn2(struct xpc_partition *part)
839{
840 unsigned long irq_flags;
Dean Nelsonc39838c2008-07-29 22:34:11 -0700841 struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa +
Dean Nelsonee6665e2008-07-29 22:34:13 -0700842 (XPC_DEACTIVATE_REQUEST_AMO_SN2 *
Dean Nelsonc39838c2008-07-29 22:34:11 -0700843 sizeof(struct amo)));
Dean Nelsona47d5da2008-07-29 22:34:09 -0700844
845 local_irq_save(irq_flags);
846
Dean Nelsonc39838c2008-07-29 22:34:11 -0700847 /* clear bit corresponding to our partid in remote partition's amo */
Dean Nelsona47d5da2008-07-29 22:34:09 -0700848 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND,
Dean Nelson04de7412008-07-29 22:34:14 -0700849 ~BIT(sn_partition_id));
850
Dean Nelsona47d5da2008-07-29 22:34:09 -0700851 /*
852 * We must always use the nofault function regardless of whether we
853 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
854 * didn't, we'd never know that the other partition is down and would
Dean Nelsonc39838c2008-07-29 22:34:11 -0700855 * keep sending IRQs and amos to it until the heartbeat times out.
Dean Nelsona47d5da2008-07-29 22:34:09 -0700856 */
857 (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
858 variable),
859 xp_nofault_PIOR_target));
860
861 local_irq_restore(irq_flags);
862}
863
864static int
865xpc_partition_deactivation_requested_sn2(short partid)
866{
Dean Nelson8e85c232008-07-29 22:34:13 -0700867 struct amo *amo = xpc_vars_sn2->amos_page +
868 XPC_DEACTIVATE_REQUEST_AMO_SN2;
Dean Nelsona47d5da2008-07-29 22:34:09 -0700869
Dean Nelsonc39838c2008-07-29 22:34:11 -0700870 /* our partition's amo variable ANDed with partid mask */
Dean Nelsona47d5da2008-07-29 22:34:09 -0700871 return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) &
Dean Nelson04de7412008-07-29 22:34:14 -0700872 BIT(partid)) != 0;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700873}
874
875/*
876 * Update the remote partition's info.
877 */
878static void
879xpc_update_partition_info_sn2(struct xpc_partition *part, u8 remote_rp_version,
Dean Nelson81fe7882008-07-29 22:34:15 -0700880 unsigned long *remote_rp_ts_jiffies,
Dean Nelsona812dcc2008-07-29 22:34:16 -0700881 unsigned long remote_rp_pa,
882 unsigned long remote_vars_pa,
Dean Nelson33ba3c72008-07-29 22:34:07 -0700883 struct xpc_vars_sn2 *remote_vars)
884{
Dean Nelsona47d5da2008-07-29 22:34:09 -0700885 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
886
Dean Nelson33ba3c72008-07-29 22:34:07 -0700887 part->remote_rp_version = remote_rp_version;
888 dev_dbg(xpc_part, " remote_rp_version = 0x%016x\n",
889 part->remote_rp_version);
890
Dean Nelson81fe7882008-07-29 22:34:15 -0700891 part->remote_rp_ts_jiffies = *remote_rp_ts_jiffies;
892 dev_dbg(xpc_part, " remote_rp_ts_jiffies = 0x%016lx\n",
893 part->remote_rp_ts_jiffies);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700894
895 part->remote_rp_pa = remote_rp_pa;
896 dev_dbg(xpc_part, " remote_rp_pa = 0x%016lx\n", part->remote_rp_pa);
897
Dean Nelsona47d5da2008-07-29 22:34:09 -0700898 part_sn2->remote_vars_pa = remote_vars_pa;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700899 dev_dbg(xpc_part, " remote_vars_pa = 0x%016lx\n",
Dean Nelsona47d5da2008-07-29 22:34:09 -0700900 part_sn2->remote_vars_pa);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700901
Dean Nelson158bc692009-01-15 13:50:57 -0800902 part->last_heartbeat = remote_vars->heartbeat - 1;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700903 dev_dbg(xpc_part, " last_heartbeat = 0x%016lx\n",
904 part->last_heartbeat);
905
Dean Nelsona47d5da2008-07-29 22:34:09 -0700906 part_sn2->remote_vars_part_pa = remote_vars->vars_part_pa;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700907 dev_dbg(xpc_part, " remote_vars_part_pa = 0x%016lx\n",
Dean Nelsona47d5da2008-07-29 22:34:09 -0700908 part_sn2->remote_vars_part_pa);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700909
Dean Nelsona47d5da2008-07-29 22:34:09 -0700910 part_sn2->activate_IRQ_nasid = remote_vars->activate_IRQ_nasid;
911 dev_dbg(xpc_part, " activate_IRQ_nasid = 0x%x\n",
912 part_sn2->activate_IRQ_nasid);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700913
Dean Nelsona47d5da2008-07-29 22:34:09 -0700914 part_sn2->activate_IRQ_phys_cpuid =
915 remote_vars->activate_IRQ_phys_cpuid;
916 dev_dbg(xpc_part, " activate_IRQ_phys_cpuid = 0x%x\n",
917 part_sn2->activate_IRQ_phys_cpuid);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700918
Dean Nelsona47d5da2008-07-29 22:34:09 -0700919 part_sn2->remote_amos_page_pa = remote_vars->amos_page_pa;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700920 dev_dbg(xpc_part, " remote_amos_page_pa = 0x%lx\n",
Dean Nelsona47d5da2008-07-29 22:34:09 -0700921 part_sn2->remote_amos_page_pa);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700922
Dean Nelsona47d5da2008-07-29 22:34:09 -0700923 part_sn2->remote_vars_version = remote_vars->version;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700924 dev_dbg(xpc_part, " remote_vars_version = 0x%x\n",
Dean Nelsona47d5da2008-07-29 22:34:09 -0700925 part_sn2->remote_vars_version);
Dean Nelson33ba3c72008-07-29 22:34:07 -0700926}
927
928/*
Dean Nelson7fb5e592008-07-29 22:34:10 -0700929 * Prior code has determined the nasid which generated a activate IRQ.
930 * Inspect that nasid to determine if its partition needs to be activated
931 * or deactivated.
Dean Nelson33ba3c72008-07-29 22:34:07 -0700932 *
Dean Nelson7fb5e592008-07-29 22:34:10 -0700933 * A partition is considered "awaiting activation" if our partition
Dean Nelson33ba3c72008-07-29 22:34:07 -0700934 * flags indicate it is not active and it has a heartbeat. A
935 * partition is considered "awaiting deactivation" if our partition
936 * flags indicate it is active but it has no heartbeat or it is not
937 * sending its heartbeat to us.
938 *
939 * To determine the heartbeat, the remote nasid must have a properly
940 * initialized reserved page.
941 */
942static void
Dean Nelson6e41017a2008-07-29 22:34:09 -0700943xpc_identify_activate_IRQ_req_sn2(int nasid)
Dean Nelson33ba3c72008-07-29 22:34:07 -0700944{
945 struct xpc_rsvd_page *remote_rp;
946 struct xpc_vars_sn2 *remote_vars;
Dean Nelsona812dcc2008-07-29 22:34:16 -0700947 unsigned long remote_rp_pa;
948 unsigned long remote_vars_pa;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700949 int remote_rp_version;
950 int reactivate = 0;
Dean Nelson81fe7882008-07-29 22:34:15 -0700951 unsigned long remote_rp_ts_jiffies = 0;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700952 short partid;
953 struct xpc_partition *part;
Dean Nelsona47d5da2008-07-29 22:34:09 -0700954 struct xpc_partition_sn2 *part_sn2;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700955 enum xp_retval ret;
956
957 /* pull over the reserved page structure */
958
Dean Nelsonee6665e2008-07-29 22:34:13 -0700959 remote_rp = (struct xpc_rsvd_page *)xpc_remote_copy_buffer_sn2;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700960
961 ret = xpc_get_remote_rp(nasid, NULL, remote_rp, &remote_rp_pa);
962 if (ret != xpSuccess) {
963 dev_warn(xpc_part, "unable to get reserved page from nasid %d, "
964 "which sent interrupt, reason=%d\n", nasid, ret);
965 return;
966 }
967
968 remote_vars_pa = remote_rp->sn.vars_pa;
969 remote_rp_version = remote_rp->version;
Dean Nelson81fe7882008-07-29 22:34:15 -0700970 remote_rp_ts_jiffies = remote_rp->ts_jiffies;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700971
972 partid = remote_rp->SAL_partid;
973 part = &xpc_partitions[partid];
Dean Nelsona47d5da2008-07-29 22:34:09 -0700974 part_sn2 = &part->sn.sn2;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700975
976 /* pull over the cross partition variables */
977
Dean Nelsonee6665e2008-07-29 22:34:13 -0700978 remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700979
980 ret = xpc_get_remote_vars_sn2(remote_vars_pa, remote_vars);
981 if (ret != xpSuccess) {
Dean Nelson33ba3c72008-07-29 22:34:07 -0700982 dev_warn(xpc_part, "unable to get XPC variables from nasid %d, "
983 "which sent interrupt, reason=%d\n", nasid, ret);
984
985 XPC_DEACTIVATE_PARTITION(part, ret);
986 return;
987 }
988
Dean Nelson6e41017a2008-07-29 22:34:09 -0700989 part->activate_IRQ_rcvd++;
Dean Nelson33ba3c72008-07-29 22:34:07 -0700990
991 dev_dbg(xpc_part, "partid for nasid %d is %d; IRQs = %d; HB = "
Dean Nelson6e41017a2008-07-29 22:34:09 -0700992 "%ld:0x%lx\n", (int)nasid, (int)partid, part->activate_IRQ_rcvd,
Dean Nelson33ba3c72008-07-29 22:34:07 -0700993 remote_vars->heartbeat, remote_vars->heartbeating_to_mask[0]);
994
995 if (xpc_partition_disengaged(part) &&
Dean Nelson83469b52008-07-29 22:34:18 -0700996 part->act_state == XPC_P_AS_INACTIVE) {
Dean Nelson33ba3c72008-07-29 22:34:07 -0700997
998 xpc_update_partition_info_sn2(part, remote_rp_version,
Dean Nelson81fe7882008-07-29 22:34:15 -0700999 &remote_rp_ts_jiffies,
1000 remote_rp_pa, remote_vars_pa,
1001 remote_vars);
Dean Nelson33ba3c72008-07-29 22:34:07 -07001002
Dean Nelsona47d5da2008-07-29 22:34:09 -07001003 if (xpc_partition_deactivation_requested_sn2(partid)) {
1004 /*
1005 * Other side is waiting on us to deactivate even though
1006 * we already have.
1007 */
1008 return;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001009 }
1010
1011 xpc_activate_partition(part);
1012 return;
1013 }
1014
1015 DBUG_ON(part->remote_rp_version == 0);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001016 DBUG_ON(part_sn2->remote_vars_version == 0);
Dean Nelson33ba3c72008-07-29 22:34:07 -07001017
Dean Nelson81fe7882008-07-29 22:34:15 -07001018 if (remote_rp_ts_jiffies != part->remote_rp_ts_jiffies) {
Dean Nelson33ba3c72008-07-29 22:34:07 -07001019
Dean Nelsona47d5da2008-07-29 22:34:09 -07001020 /* the other side rebooted */
Dean Nelson33ba3c72008-07-29 22:34:07 -07001021
Dean Nelsona47d5da2008-07-29 22:34:09 -07001022 DBUG_ON(xpc_partition_engaged_sn2(partid));
1023 DBUG_ON(xpc_partition_deactivation_requested_sn2(partid));
Dean Nelson33ba3c72008-07-29 22:34:07 -07001024
1025 xpc_update_partition_info_sn2(part, remote_rp_version,
Dean Nelson81fe7882008-07-29 22:34:15 -07001026 &remote_rp_ts_jiffies,
1027 remote_rp_pa, remote_vars_pa,
1028 remote_vars);
Dean Nelson33ba3c72008-07-29 22:34:07 -07001029 reactivate = 1;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001030 }
1031
Dean Nelsona47d5da2008-07-29 22:34:09 -07001032 if (part->disengage_timeout > 0 && !xpc_partition_disengaged(part)) {
Dean Nelson33ba3c72008-07-29 22:34:07 -07001033 /* still waiting on other side to disengage from us */
1034 return;
1035 }
1036
Dean Nelsona47d5da2008-07-29 22:34:09 -07001037 if (reactivate)
Dean Nelson33ba3c72008-07-29 22:34:07 -07001038 XPC_DEACTIVATE_PARTITION(part, xpReactivating);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001039 else if (xpc_partition_deactivation_requested_sn2(partid))
Dean Nelson33ba3c72008-07-29 22:34:07 -07001040 XPC_DEACTIVATE_PARTITION(part, xpOtherGoingDown);
Dean Nelson33ba3c72008-07-29 22:34:07 -07001041}
1042
1043/*
Dean Nelsonc39838c2008-07-29 22:34:11 -07001044 * Loop through the activation amo variables and process any bits
Dean Nelson33ba3c72008-07-29 22:34:07 -07001045 * which are set. Each bit indicates a nasid sending a partition
1046 * activation or deactivation request.
1047 *
1048 * Return #of IRQs detected.
1049 */
1050int
Dean Nelson6e41017a2008-07-29 22:34:09 -07001051xpc_identify_activate_IRQ_sender_sn2(void)
Dean Nelson33ba3c72008-07-29 22:34:07 -07001052{
Dean Nelson04de7412008-07-29 22:34:14 -07001053 int l;
1054 int b;
1055 unsigned long nasid_mask_long;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001056 u64 nasid; /* remote nasid */
1057 int n_IRQs_detected = 0;
Dean Nelsonc39838c2008-07-29 22:34:11 -07001058 struct amo *act_amos;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001059
Dean Nelson8e85c232008-07-29 22:34:13 -07001060 act_amos = xpc_vars_sn2->amos_page + XPC_ACTIVATE_IRQ_AMOS_SN2;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001061
Dean Nelson04de7412008-07-29 22:34:14 -07001062 /* scan through activate amo variables looking for non-zero entries */
1063 for (l = 0; l < xpc_nasid_mask_nlongs; l++) {
Dean Nelson33ba3c72008-07-29 22:34:07 -07001064
1065 if (xpc_exiting)
1066 break;
1067
Dean Nelson04de7412008-07-29 22:34:14 -07001068 nasid_mask_long = xpc_receive_IRQ_amo_sn2(&act_amos[l]);
1069
1070 b = find_first_bit(&nasid_mask_long, BITS_PER_LONG);
1071 if (b >= BITS_PER_LONG) {
1072 /* no IRQs from nasids in this amo variable */
Dean Nelson33ba3c72008-07-29 22:34:07 -07001073 continue;
1074 }
1075
Dean Nelson04de7412008-07-29 22:34:14 -07001076 dev_dbg(xpc_part, "amo[%d] gave back 0x%lx\n", l,
1077 nasid_mask_long);
Dean Nelson33ba3c72008-07-29 22:34:07 -07001078
1079 /*
1080 * If this nasid has been added to the machine since
1081 * our partition was reset, this will retain the
1082 * remote nasid in our reserved pages machine mask.
1083 * This is used in the event of module reload.
1084 */
Dean Nelson04de7412008-07-29 22:34:14 -07001085 xpc_mach_nasids[l] |= nasid_mask_long;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001086
1087 /* locate the nasid(s) which sent interrupts */
1088
Dean Nelson04de7412008-07-29 22:34:14 -07001089 do {
1090 n_IRQs_detected++;
1091 nasid = (l * BITS_PER_LONG + b) * 2;
1092 dev_dbg(xpc_part, "interrupt from nasid %ld\n", nasid);
1093 xpc_identify_activate_IRQ_req_sn2(nasid);
1094
1095 b = find_next_bit(&nasid_mask_long, BITS_PER_LONG,
1096 b + 1);
1097 } while (b < BITS_PER_LONG);
Dean Nelson33ba3c72008-07-29 22:34:07 -07001098 }
1099 return n_IRQs_detected;
1100}
1101
1102static void
Dean Nelson5b8669d2008-07-29 22:34:18 -07001103xpc_process_activate_IRQ_rcvd_sn2(void)
Dean Nelson33ba3c72008-07-29 22:34:07 -07001104{
Dean Nelson5b8669d2008-07-29 22:34:18 -07001105 unsigned long irq_flags;
1106 int n_IRQs_expected;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001107 int n_IRQs_detected;
1108
Dean Nelson5b8669d2008-07-29 22:34:18 -07001109 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
1110 n_IRQs_expected = xpc_activate_IRQ_rcvd;
1111 xpc_activate_IRQ_rcvd = 0;
1112 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
1113
Dean Nelson6e41017a2008-07-29 22:34:09 -07001114 n_IRQs_detected = xpc_identify_activate_IRQ_sender_sn2();
Dean Nelson33ba3c72008-07-29 22:34:07 -07001115 if (n_IRQs_detected < n_IRQs_expected) {
Dean Nelsonc39838c2008-07-29 22:34:11 -07001116 /* retry once to help avoid missing amo */
Dean Nelson6e41017a2008-07-29 22:34:09 -07001117 (void)xpc_identify_activate_IRQ_sender_sn2();
Dean Nelson33ba3c72008-07-29 22:34:07 -07001118 }
1119}
1120
Dean Nelsone17d4162008-07-29 22:34:06 -07001121/*
Dean Nelson5b8669d2008-07-29 22:34:18 -07001122 * Setup the channel structures that are sn2 specific.
Dean Nelsone17d4162008-07-29 22:34:06 -07001123 */
1124static enum xp_retval
Dean Nelson5b8669d2008-07-29 22:34:18 -07001125xpc_setup_ch_structures_sn_sn2(struct xpc_partition *part)
Dean Nelsone17d4162008-07-29 22:34:06 -07001126{
Dean Nelsona47d5da2008-07-29 22:34:09 -07001127 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
Dean Nelson5b8669d2008-07-29 22:34:18 -07001128 struct xpc_channel_sn2 *ch_sn2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001129 enum xp_retval retval;
1130 int ret;
1131 int cpuid;
1132 int ch_number;
Dean Nelsone17d4162008-07-29 22:34:06 -07001133 struct timer_list *timer;
1134 short partid = XPC_PARTID(part);
1135
Dean Nelsone17d4162008-07-29 22:34:06 -07001136 /* allocate all the required GET/PUT values */
1137
Dean Nelson185c3a12008-07-29 22:34:11 -07001138 part_sn2->local_GPs =
Dean Nelson5b8669d2008-07-29 22:34:18 -07001139 xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE, GFP_KERNEL,
1140 &part_sn2->local_GPs_base);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001141 if (part_sn2->local_GPs == NULL) {
Dean Nelsone17d4162008-07-29 22:34:06 -07001142 dev_err(xpc_chan, "can't get memory for local get/put "
1143 "values\n");
Dean Nelson5b8669d2008-07-29 22:34:18 -07001144 return xpNoMemory;
Dean Nelsone17d4162008-07-29 22:34:06 -07001145 }
1146
Dean Nelson185c3a12008-07-29 22:34:11 -07001147 part_sn2->remote_GPs =
Dean Nelson5b8669d2008-07-29 22:34:18 -07001148 xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE, GFP_KERNEL,
1149 &part_sn2->remote_GPs_base);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001150 if (part_sn2->remote_GPs == NULL) {
Dean Nelsone17d4162008-07-29 22:34:06 -07001151 dev_err(xpc_chan, "can't get memory for remote get/put "
1152 "values\n");
1153 retval = xpNoMemory;
Dean Nelson5b8669d2008-07-29 22:34:18 -07001154 goto out_1;
Dean Nelsone17d4162008-07-29 22:34:06 -07001155 }
1156
Dean Nelsona47d5da2008-07-29 22:34:09 -07001157 part_sn2->remote_GPs_pa = 0;
Dean Nelsone17d4162008-07-29 22:34:06 -07001158
1159 /* allocate all the required open and close args */
1160
Dean Nelson5b8669d2008-07-29 22:34:18 -07001161 part_sn2->local_openclose_args =
1162 xpc_kzalloc_cacheline_aligned(XPC_OPENCLOSE_ARGS_SIZE,
1163 GFP_KERNEL, &part_sn2->
1164 local_openclose_args_base);
1165 if (part_sn2->local_openclose_args == NULL) {
Dean Nelsone17d4162008-07-29 22:34:06 -07001166 dev_err(xpc_chan, "can't get memory for local connect args\n");
1167 retval = xpNoMemory;
Dean Nelson5b8669d2008-07-29 22:34:18 -07001168 goto out_2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001169 }
1170
Dean Nelsona47d5da2008-07-29 22:34:09 -07001171 part_sn2->remote_openclose_args_pa = 0;
Dean Nelsone17d4162008-07-29 22:34:06 -07001172
Dean Nelson7fb5e592008-07-29 22:34:10 -07001173 part_sn2->local_chctl_amo_va = xpc_init_IRQ_amo_sn2(partid);
Dean Nelsone17d4162008-07-29 22:34:06 -07001174
Dean Nelson7fb5e592008-07-29 22:34:10 -07001175 part_sn2->notify_IRQ_nasid = 0;
1176 part_sn2->notify_IRQ_phys_cpuid = 0;
1177 part_sn2->remote_chctl_amo_va = NULL;
Dean Nelsone17d4162008-07-29 22:34:06 -07001178
Dean Nelson7fb5e592008-07-29 22:34:10 -07001179 sprintf(part_sn2->notify_IRQ_owner, "xpc%02d", partid);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001180 ret = request_irq(SGI_XPC_NOTIFY, xpc_handle_notify_IRQ_sn2,
Dean Nelson7fb5e592008-07-29 22:34:10 -07001181 IRQF_SHARED, part_sn2->notify_IRQ_owner,
Dean Nelsona47d5da2008-07-29 22:34:09 -07001182 (void *)(u64)partid);
Dean Nelsone17d4162008-07-29 22:34:06 -07001183 if (ret != 0) {
1184 dev_err(xpc_chan, "can't register NOTIFY IRQ handler, "
1185 "errno=%d\n", -ret);
1186 retval = xpLackOfResources;
Dean Nelson5b8669d2008-07-29 22:34:18 -07001187 goto out_3;
Dean Nelsone17d4162008-07-29 22:34:06 -07001188 }
1189
Dean Nelson7fb5e592008-07-29 22:34:10 -07001190 /* Setup a timer to check for dropped notify IRQs */
Dean Nelsona47d5da2008-07-29 22:34:09 -07001191 timer = &part_sn2->dropped_notify_IRQ_timer;
Dean Nelsone17d4162008-07-29 22:34:06 -07001192 init_timer(timer);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001193 timer->function =
Dean Nelson7fb5e592008-07-29 22:34:10 -07001194 (void (*)(unsigned long))xpc_check_for_dropped_notify_IRQ_sn2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001195 timer->data = (unsigned long)part;
Dean Nelson7fb5e592008-07-29 22:34:10 -07001196 timer->expires = jiffies + XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL;
Dean Nelsone17d4162008-07-29 22:34:06 -07001197 add_timer(timer);
1198
Dean Nelsone17d4162008-07-29 22:34:06 -07001199 for (ch_number = 0; ch_number < part->nchannels; ch_number++) {
Dean Nelson5b8669d2008-07-29 22:34:18 -07001200 ch_sn2 = &part->channels[ch_number].sn.sn2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001201
Dean Nelson5b8669d2008-07-29 22:34:18 -07001202 ch_sn2->local_GP = &part_sn2->local_GPs[ch_number];
1203 ch_sn2->local_openclose_args =
1204 &part_sn2->local_openclose_args[ch_number];
Dean Nelsone17d4162008-07-29 22:34:06 -07001205
Dean Nelson5b8669d2008-07-29 22:34:18 -07001206 mutex_init(&ch_sn2->msg_to_pull_mutex);
Dean Nelsone17d4162008-07-29 22:34:06 -07001207 }
1208
1209 /*
Dean Nelsone17d4162008-07-29 22:34:06 -07001210 * Setup the per partition specific variables required by the
1211 * remote partition to establish channel connections with us.
1212 *
1213 * The setting of the magic # indicates that these per partition
1214 * specific variables are ready to be used.
1215 */
Dean Nelsona812dcc2008-07-29 22:34:16 -07001216 xpc_vars_part_sn2[partid].GPs_pa = xp_pa(part_sn2->local_GPs);
Dean Nelson8e85c232008-07-29 22:34:13 -07001217 xpc_vars_part_sn2[partid].openclose_args_pa =
Dean Nelson5b8669d2008-07-29 22:34:18 -07001218 xp_pa(part_sn2->local_openclose_args);
Dean Nelson8e85c232008-07-29 22:34:13 -07001219 xpc_vars_part_sn2[partid].chctl_amo_pa =
Dean Nelsona812dcc2008-07-29 22:34:16 -07001220 xp_pa(part_sn2->local_chctl_amo_va);
Dean Nelsone17d4162008-07-29 22:34:06 -07001221 cpuid = raw_smp_processor_id(); /* any CPU in this partition will do */
Dean Nelson8e85c232008-07-29 22:34:13 -07001222 xpc_vars_part_sn2[partid].notify_IRQ_nasid = cpuid_to_nasid(cpuid);
1223 xpc_vars_part_sn2[partid].notify_IRQ_phys_cpuid =
1224 cpu_physical_id(cpuid);
1225 xpc_vars_part_sn2[partid].nchannels = part->nchannels;
Dean Nelson5b8669d2008-07-29 22:34:18 -07001226 xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC1_SN2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001227
1228 return xpSuccess;
1229
Dean Nelson5b8669d2008-07-29 22:34:18 -07001230 /* setup of ch structures failed */
Dean Nelsone17d4162008-07-29 22:34:06 -07001231out_3:
Dean Nelson5b8669d2008-07-29 22:34:18 -07001232 kfree(part_sn2->local_openclose_args_base);
1233 part_sn2->local_openclose_args = NULL;
1234out_2:
Dean Nelsona47d5da2008-07-29 22:34:09 -07001235 kfree(part_sn2->remote_GPs_base);
1236 part_sn2->remote_GPs = NULL;
Dean Nelson5b8669d2008-07-29 22:34:18 -07001237out_1:
Dean Nelsona47d5da2008-07-29 22:34:09 -07001238 kfree(part_sn2->local_GPs_base);
1239 part_sn2->local_GPs = NULL;
Dean Nelsone17d4162008-07-29 22:34:06 -07001240 return retval;
1241}
1242
1243/*
Dean Nelson5b8669d2008-07-29 22:34:18 -07001244 * Teardown the channel structures that are sn2 specific.
Dean Nelsone17d4162008-07-29 22:34:06 -07001245 */
1246static void
Dean Nelson5b8669d2008-07-29 22:34:18 -07001247xpc_teardown_ch_structures_sn_sn2(struct xpc_partition *part)
Dean Nelsone17d4162008-07-29 22:34:06 -07001248{
Dean Nelsona47d5da2008-07-29 22:34:09 -07001249 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001250 short partid = XPC_PARTID(part);
1251
1252 /*
Dean Nelson5b8669d2008-07-29 22:34:18 -07001253 * Indicate that the variables specific to the remote partition are no
1254 * longer available for its use.
Dean Nelsone17d4162008-07-29 22:34:06 -07001255 */
Dean Nelson8e85c232008-07-29 22:34:13 -07001256 xpc_vars_part_sn2[partid].magic = 0;
Dean Nelsone17d4162008-07-29 22:34:06 -07001257
Dean Nelsone17d4162008-07-29 22:34:06 -07001258 /* in case we've still got outstanding timers registered... */
Dean Nelsona47d5da2008-07-29 22:34:09 -07001259 del_timer_sync(&part_sn2->dropped_notify_IRQ_timer);
Dean Nelson5b8669d2008-07-29 22:34:18 -07001260 free_irq(SGI_XPC_NOTIFY, (void *)(u64)partid);
Dean Nelsone17d4162008-07-29 22:34:06 -07001261
Dean Nelson5b8669d2008-07-29 22:34:18 -07001262 kfree(part_sn2->local_openclose_args_base);
1263 part_sn2->local_openclose_args = NULL;
Dean Nelsona47d5da2008-07-29 22:34:09 -07001264 kfree(part_sn2->remote_GPs_base);
1265 part_sn2->remote_GPs = NULL;
1266 kfree(part_sn2->local_GPs_base);
1267 part_sn2->local_GPs = NULL;
Dean Nelson7fb5e592008-07-29 22:34:10 -07001268 part_sn2->local_chctl_amo_va = NULL;
Dean Nelsone17d4162008-07-29 22:34:06 -07001269}
1270
1271/*
1272 * Create a wrapper that hides the underlying mechanism for pulling a cacheline
1273 * (or multiple cachelines) from a remote partition.
1274 *
Dean Nelsona812dcc2008-07-29 22:34:16 -07001275 * src_pa must be a cacheline aligned physical address on the remote partition.
Dean Nelsone17d4162008-07-29 22:34:06 -07001276 * dst must be a cacheline aligned virtual address on this partition.
1277 * cnt must be cacheline sized
1278 */
Dean Nelsonea57f802008-07-29 22:34:14 -07001279/* ??? Replace this function by call to xp_remote_memcpy() or bte_copy()? */
Dean Nelsone17d4162008-07-29 22:34:06 -07001280static enum xp_retval
1281xpc_pull_remote_cachelines_sn2(struct xpc_partition *part, void *dst,
Dean Nelsona812dcc2008-07-29 22:34:16 -07001282 const unsigned long src_pa, size_t cnt)
Dean Nelsone17d4162008-07-29 22:34:06 -07001283{
1284 enum xp_retval ret;
1285
Dean Nelsona812dcc2008-07-29 22:34:16 -07001286 DBUG_ON(src_pa != L1_CACHE_ALIGN(src_pa));
1287 DBUG_ON((unsigned long)dst != L1_CACHE_ALIGN((unsigned long)dst));
Dean Nelsone17d4162008-07-29 22:34:06 -07001288 DBUG_ON(cnt != L1_CACHE_ALIGN(cnt));
1289
Dean Nelson83469b52008-07-29 22:34:18 -07001290 if (part->act_state == XPC_P_AS_DEACTIVATING)
Dean Nelsone17d4162008-07-29 22:34:06 -07001291 return part->reason;
1292
Dean Nelsona812dcc2008-07-29 22:34:16 -07001293 ret = xp_remote_memcpy(xp_pa(dst), src_pa, cnt);
Dean Nelsone17d4162008-07-29 22:34:06 -07001294 if (ret != xpSuccess) {
1295 dev_dbg(xpc_chan, "xp_remote_memcpy() from partition %d failed,"
1296 " ret=%d\n", XPC_PARTID(part), ret);
1297 }
1298 return ret;
1299}
1300
1301/*
1302 * Pull the remote per partition specific variables from the specified
1303 * partition.
1304 */
1305static enum xp_retval
1306xpc_pull_remote_vars_part_sn2(struct xpc_partition *part)
1307{
Dean Nelsona47d5da2008-07-29 22:34:09 -07001308 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001309 u8 buffer[L1_CACHE_BYTES * 2];
1310 struct xpc_vars_part_sn2 *pulled_entry_cacheline =
1311 (struct xpc_vars_part_sn2 *)L1_CACHE_ALIGN((u64)buffer);
1312 struct xpc_vars_part_sn2 *pulled_entry;
Dean Nelsona812dcc2008-07-29 22:34:16 -07001313 unsigned long remote_entry_cacheline_pa;
1314 unsigned long remote_entry_pa;
Dean Nelsone17d4162008-07-29 22:34:06 -07001315 short partid = XPC_PARTID(part);
1316 enum xp_retval ret;
1317
1318 /* pull the cacheline that contains the variables we're interested in */
1319
Dean Nelsona47d5da2008-07-29 22:34:09 -07001320 DBUG_ON(part_sn2->remote_vars_part_pa !=
1321 L1_CACHE_ALIGN(part_sn2->remote_vars_part_pa));
Dean Nelsone17d4162008-07-29 22:34:06 -07001322 DBUG_ON(sizeof(struct xpc_vars_part_sn2) != L1_CACHE_BYTES / 2);
1323
Dean Nelsona47d5da2008-07-29 22:34:09 -07001324 remote_entry_pa = part_sn2->remote_vars_part_pa +
Dean Nelsone17d4162008-07-29 22:34:06 -07001325 sn_partition_id * sizeof(struct xpc_vars_part_sn2);
1326
1327 remote_entry_cacheline_pa = (remote_entry_pa & ~(L1_CACHE_BYTES - 1));
1328
1329 pulled_entry = (struct xpc_vars_part_sn2 *)((u64)pulled_entry_cacheline
1330 + (remote_entry_pa &
1331 (L1_CACHE_BYTES - 1)));
1332
1333 ret = xpc_pull_remote_cachelines_sn2(part, pulled_entry_cacheline,
Dean Nelsona812dcc2008-07-29 22:34:16 -07001334 remote_entry_cacheline_pa,
Dean Nelsone17d4162008-07-29 22:34:06 -07001335 L1_CACHE_BYTES);
1336 if (ret != xpSuccess) {
1337 dev_dbg(xpc_chan, "failed to pull XPC vars_part from "
1338 "partition %d, ret=%d\n", partid, ret);
1339 return ret;
1340 }
1341
1342 /* see if they've been set up yet */
1343
Dean Nelson5b8669d2008-07-29 22:34:18 -07001344 if (pulled_entry->magic != XPC_VP_MAGIC1_SN2 &&
1345 pulled_entry->magic != XPC_VP_MAGIC2_SN2) {
Dean Nelsone17d4162008-07-29 22:34:06 -07001346
1347 if (pulled_entry->magic != 0) {
1348 dev_dbg(xpc_chan, "partition %d's XPC vars_part for "
1349 "partition %d has bad magic value (=0x%lx)\n",
1350 partid, sn_partition_id, pulled_entry->magic);
1351 return xpBadMagic;
1352 }
1353
1354 /* they've not been initialized yet */
1355 return xpRetry;
1356 }
1357
Dean Nelson5b8669d2008-07-29 22:34:18 -07001358 if (xpc_vars_part_sn2[partid].magic == XPC_VP_MAGIC1_SN2) {
Dean Nelsone17d4162008-07-29 22:34:06 -07001359
1360 /* validate the variables */
1361
1362 if (pulled_entry->GPs_pa == 0 ||
1363 pulled_entry->openclose_args_pa == 0 ||
Dean Nelson7fb5e592008-07-29 22:34:10 -07001364 pulled_entry->chctl_amo_pa == 0) {
Dean Nelsone17d4162008-07-29 22:34:06 -07001365
1366 dev_err(xpc_chan, "partition %d's XPC vars_part for "
1367 "partition %d are not valid\n", partid,
1368 sn_partition_id);
1369 return xpInvalidAddress;
1370 }
1371
1372 /* the variables we imported look to be valid */
1373
Dean Nelsona47d5da2008-07-29 22:34:09 -07001374 part_sn2->remote_GPs_pa = pulled_entry->GPs_pa;
1375 part_sn2->remote_openclose_args_pa =
Dean Nelsone17d4162008-07-29 22:34:06 -07001376 pulled_entry->openclose_args_pa;
Dean Nelson7fb5e592008-07-29 22:34:10 -07001377 part_sn2->remote_chctl_amo_va =
Dean Nelsonc39838c2008-07-29 22:34:11 -07001378 (struct amo *)__va(pulled_entry->chctl_amo_pa);
Dean Nelson7fb5e592008-07-29 22:34:10 -07001379 part_sn2->notify_IRQ_nasid = pulled_entry->notify_IRQ_nasid;
1380 part_sn2->notify_IRQ_phys_cpuid =
1381 pulled_entry->notify_IRQ_phys_cpuid;
Dean Nelsone17d4162008-07-29 22:34:06 -07001382
1383 if (part->nchannels > pulled_entry->nchannels)
1384 part->nchannels = pulled_entry->nchannels;
1385
1386 /* let the other side know that we've pulled their variables */
1387
Dean Nelson5b8669d2008-07-29 22:34:18 -07001388 xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC2_SN2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001389 }
1390
Dean Nelson5b8669d2008-07-29 22:34:18 -07001391 if (pulled_entry->magic == XPC_VP_MAGIC1_SN2)
Dean Nelsone17d4162008-07-29 22:34:06 -07001392 return xpRetry;
1393
1394 return xpSuccess;
1395}
1396
1397/*
1398 * Establish first contact with the remote partititon. This involves pulling
1399 * the XPC per partition variables from the remote partition and waiting for
1400 * the remote partition to pull ours.
1401 */
1402static enum xp_retval
1403xpc_make_first_contact_sn2(struct xpc_partition *part)
1404{
Dean Nelsona47d5da2008-07-29 22:34:09 -07001405 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001406 enum xp_retval ret;
1407
Dean Nelson33ba3c72008-07-29 22:34:07 -07001408 /*
Dean Nelsonc39838c2008-07-29 22:34:11 -07001409 * Register the remote partition's amos with SAL so it can handle
Dean Nelson33ba3c72008-07-29 22:34:07 -07001410 * and cleanup errors within that address range should the remote
1411 * partition go down. We don't unregister this range because it is
1412 * difficult to tell when outstanding writes to the remote partition
1413 * are finished and thus when it is safe to unregister. This should
1414 * not result in wasted space in the SAL xp_addr_region table because
1415 * we should get the same page for remote_amos_page_pa after module
1416 * reloads and system reboots.
1417 */
Dean Nelsona47d5da2008-07-29 22:34:09 -07001418 if (sn_register_xp_addr_region(part_sn2->remote_amos_page_pa,
Dean Nelson33ba3c72008-07-29 22:34:07 -07001419 PAGE_SIZE, 1) < 0) {
1420 dev_warn(xpc_part, "xpc_activating(%d) failed to register "
1421 "xp_addr region\n", XPC_PARTID(part));
1422
1423 ret = xpPhysAddrRegFailed;
1424 XPC_DEACTIVATE_PARTITION(part, ret);
1425 return ret;
1426 }
1427
Dean Nelsona47d5da2008-07-29 22:34:09 -07001428 /*
1429 * Send activate IRQ to get other side to activate if they've not
1430 * already begun to do so.
1431 */
Dean Nelson7fb5e592008-07-29 22:34:10 -07001432 xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa,
Dean Nelsona47d5da2008-07-29 22:34:09 -07001433 cnodeid_to_nasid(0),
1434 part_sn2->activate_IRQ_nasid,
1435 part_sn2->activate_IRQ_phys_cpuid);
Dean Nelson33ba3c72008-07-29 22:34:07 -07001436
Dean Nelsone17d4162008-07-29 22:34:06 -07001437 while ((ret = xpc_pull_remote_vars_part_sn2(part)) != xpSuccess) {
1438 if (ret != xpRetry) {
1439 XPC_DEACTIVATE_PARTITION(part, ret);
1440 return ret;
1441 }
1442
1443 dev_dbg(xpc_part, "waiting to make first contact with "
1444 "partition %d\n", XPC_PARTID(part));
1445
1446 /* wait a 1/4 of a second or so */
1447 (void)msleep_interruptible(250);
1448
Dean Nelson83469b52008-07-29 22:34:18 -07001449 if (part->act_state == XPC_P_AS_DEACTIVATING)
Dean Nelsone17d4162008-07-29 22:34:06 -07001450 return part->reason;
1451 }
1452
1453 return xpSuccess;
1454}
1455
1456/*
Dean Nelson7fb5e592008-07-29 22:34:10 -07001457 * Get the chctl flags and pull the openclose args and/or remote GPs as needed.
Dean Nelsone17d4162008-07-29 22:34:06 -07001458 */
1459static u64
Dean Nelson7fb5e592008-07-29 22:34:10 -07001460xpc_get_chctl_all_flags_sn2(struct xpc_partition *part)
Dean Nelsone17d4162008-07-29 22:34:06 -07001461{
Dean Nelsona47d5da2008-07-29 22:34:09 -07001462 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
Dean Nelsone17d4162008-07-29 22:34:06 -07001463 unsigned long irq_flags;
Dean Nelson7fb5e592008-07-29 22:34:10 -07001464 union xpc_channel_ctl_flags chctl;
Dean Nelsone17d4162008-07-29 22:34:06 -07001465 enum xp_retval ret;
1466
1467 /*
Dean Nelson7fb5e592008-07-29 22:34:10 -07001468 * See if there are any chctl flags to be handled.
Dean Nelsone17d4162008-07-29 22:34:06 -07001469 */
1470
Dean Nelson7fb5e592008-07-29 22:34:10 -07001471 spin_lock_irqsave(&part->chctl_lock, irq_flags);
1472 chctl = part->chctl;
1473 if (chctl.all_flags != 0)
1474 part->chctl.all_flags = 0;
Dean Nelsone17d4162008-07-29 22:34:06 -07001475
Dean Nelson7fb5e592008-07-29 22:34:10 -07001476 spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
Dean Nelsone17d4162008-07-29 22:34:06 -07001477
Dean Nelson7fb5e592008-07-29 22:34:10 -07001478 if (xpc_any_openclose_chctl_flags_set(&chctl)) {
Dean Nelsona47d5da2008-07-29 22:34:09 -07001479 ret = xpc_pull_remote_cachelines_sn2(part, part->
1480 remote_openclose_args,
Dean Nelsona812dcc2008-07-29 22:34:16 -07001481 part_sn2->
Dean Nelsone17d4162008-07-29 22:34:06 -07001482 remote_openclose_args_pa,
1483 XPC_OPENCLOSE_ARGS_SIZE);
1484 if (ret != xpSuccess) {
1485 XPC_DEACTIVATE_PARTITION(part, ret);
1486
1487 dev_dbg(xpc_chan, "failed to pull openclose args from "
1488 "partition %d, ret=%d\n", XPC_PARTID(part),
1489 ret);
1490
Dean Nelson7fb5e592008-07-29 22:34:10 -07001491 /* don't bother processing chctl flags anymore */
1492 chctl.all_flags = 0;
Dean Nelsone17d4162008-07-29 22:34:06 -07001493 }
1494 }
1495
Dean Nelson7fb5e592008-07-29 22:34:10 -07001496 if (xpc_any_msg_chctl_flags_set(&chctl)) {
Dean Nelsona47d5da2008-07-29 22:34:09 -07001497 ret = xpc_pull_remote_cachelines_sn2(part, part_sn2->remote_GPs,
Dean Nelsona812dcc2008-07-29 22:34:16 -07001498 part_sn2->remote_GPs_pa,
Dean Nelsone17d4162008-07-29 22:34:06 -07001499 XPC_GP_SIZE);
1500 if (ret != xpSuccess) {
1501 XPC_DEACTIVATE_PARTITION(part, ret);
1502
1503 dev_dbg(xpc_chan, "failed to pull GPs from partition "
1504 "%d, ret=%d\n", XPC_PARTID(part), ret);
1505
Dean Nelson7fb5e592008-07-29 22:34:10 -07001506 /* don't bother processing chctl flags anymore */
1507 chctl.all_flags = 0;
Dean Nelsone17d4162008-07-29 22:34:06 -07001508 }
1509 }
1510
Dean Nelson7fb5e592008-07-29 22:34:10 -07001511 return chctl.all_flags;
Dean Nelsone17d4162008-07-29 22:34:06 -07001512}
1513
Dean Nelsona47d5da2008-07-29 22:34:09 -07001514/*
Dean Nelson185c3a12008-07-29 22:34:11 -07001515 * Allocate the local message queue and the notify queue.
1516 */
1517static enum xp_retval
1518xpc_allocate_local_msgqueue_sn2(struct xpc_channel *ch)
1519{
Dean Nelson5b8669d2008-07-29 22:34:18 -07001520 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelson185c3a12008-07-29 22:34:11 -07001521 unsigned long irq_flags;
1522 int nentries;
1523 size_t nbytes;
1524
1525 for (nentries = ch->local_nentries; nentries > 0; nentries--) {
1526
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001527 nbytes = nentries * ch->entry_size;
Dean Nelson5b8669d2008-07-29 22:34:18 -07001528 ch_sn2->local_msgqueue =
1529 xpc_kzalloc_cacheline_aligned(nbytes, GFP_KERNEL,
1530 &ch_sn2->local_msgqueue_base);
1531 if (ch_sn2->local_msgqueue == NULL)
Dean Nelson185c3a12008-07-29 22:34:11 -07001532 continue;
1533
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001534 nbytes = nentries * sizeof(struct xpc_notify_sn2);
Dean Nelson5b8669d2008-07-29 22:34:18 -07001535 ch_sn2->notify_queue = kzalloc(nbytes, GFP_KERNEL);
1536 if (ch_sn2->notify_queue == NULL) {
1537 kfree(ch_sn2->local_msgqueue_base);
1538 ch_sn2->local_msgqueue = NULL;
Dean Nelson185c3a12008-07-29 22:34:11 -07001539 continue;
1540 }
1541
1542 spin_lock_irqsave(&ch->lock, irq_flags);
1543 if (nentries < ch->local_nentries) {
1544 dev_dbg(xpc_chan, "nentries=%d local_nentries=%d, "
1545 "partid=%d, channel=%d\n", nentries,
1546 ch->local_nentries, ch->partid, ch->number);
1547
1548 ch->local_nentries = nentries;
1549 }
1550 spin_unlock_irqrestore(&ch->lock, irq_flags);
1551 return xpSuccess;
1552 }
1553
1554 dev_dbg(xpc_chan, "can't get memory for local message queue and notify "
1555 "queue, partid=%d, channel=%d\n", ch->partid, ch->number);
1556 return xpNoMemory;
1557}
1558
1559/*
1560 * Allocate the cached remote message queue.
1561 */
1562static enum xp_retval
1563xpc_allocate_remote_msgqueue_sn2(struct xpc_channel *ch)
1564{
Dean Nelson5b8669d2008-07-29 22:34:18 -07001565 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelson185c3a12008-07-29 22:34:11 -07001566 unsigned long irq_flags;
1567 int nentries;
1568 size_t nbytes;
1569
1570 DBUG_ON(ch->remote_nentries <= 0);
1571
1572 for (nentries = ch->remote_nentries; nentries > 0; nentries--) {
1573
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001574 nbytes = nentries * ch->entry_size;
Dean Nelson5b8669d2008-07-29 22:34:18 -07001575 ch_sn2->remote_msgqueue =
1576 xpc_kzalloc_cacheline_aligned(nbytes, GFP_KERNEL, &ch_sn2->
1577 remote_msgqueue_base);
1578 if (ch_sn2->remote_msgqueue == NULL)
Dean Nelson185c3a12008-07-29 22:34:11 -07001579 continue;
1580
1581 spin_lock_irqsave(&ch->lock, irq_flags);
1582 if (nentries < ch->remote_nentries) {
1583 dev_dbg(xpc_chan, "nentries=%d remote_nentries=%d, "
1584 "partid=%d, channel=%d\n", nentries,
1585 ch->remote_nentries, ch->partid, ch->number);
1586
1587 ch->remote_nentries = nentries;
1588 }
1589 spin_unlock_irqrestore(&ch->lock, irq_flags);
1590 return xpSuccess;
1591 }
1592
1593 dev_dbg(xpc_chan, "can't get memory for cached remote message queue, "
1594 "partid=%d, channel=%d\n", ch->partid, ch->number);
1595 return xpNoMemory;
1596}
1597
1598/*
1599 * Allocate message queues and other stuff associated with a channel.
1600 *
1601 * Note: Assumes all of the channel sizes are filled in.
1602 */
1603static enum xp_retval
Dean Nelson5b8669d2008-07-29 22:34:18 -07001604xpc_setup_msg_structures_sn2(struct xpc_channel *ch)
Dean Nelson185c3a12008-07-29 22:34:11 -07001605{
Dean Nelson5b8669d2008-07-29 22:34:18 -07001606 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelson185c3a12008-07-29 22:34:11 -07001607 enum xp_retval ret;
1608
1609 DBUG_ON(ch->flags & XPC_C_SETUP);
1610
1611 ret = xpc_allocate_local_msgqueue_sn2(ch);
1612 if (ret == xpSuccess) {
1613
1614 ret = xpc_allocate_remote_msgqueue_sn2(ch);
1615 if (ret != xpSuccess) {
Dean Nelson5b8669d2008-07-29 22:34:18 -07001616 kfree(ch_sn2->local_msgqueue_base);
1617 ch_sn2->local_msgqueue = NULL;
1618 kfree(ch_sn2->notify_queue);
1619 ch_sn2->notify_queue = NULL;
Dean Nelson185c3a12008-07-29 22:34:11 -07001620 }
1621 }
1622 return ret;
1623}
1624
1625/*
1626 * Free up message queues and other stuff that were allocated for the specified
1627 * channel.
Dean Nelson185c3a12008-07-29 22:34:11 -07001628 */
1629static void
Dean Nelson5b8669d2008-07-29 22:34:18 -07001630xpc_teardown_msg_structures_sn2(struct xpc_channel *ch)
Dean Nelson185c3a12008-07-29 22:34:11 -07001631{
1632 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1633
1634 DBUG_ON(!spin_is_locked(&ch->lock));
Dean Nelson185c3a12008-07-29 22:34:11 -07001635
Dean Nelson5b8669d2008-07-29 22:34:18 -07001636 ch_sn2->remote_msgqueue_pa = 0;
Dean Nelson185c3a12008-07-29 22:34:11 -07001637
1638 ch_sn2->local_GP->get = 0;
1639 ch_sn2->local_GP->put = 0;
1640 ch_sn2->remote_GP.get = 0;
1641 ch_sn2->remote_GP.put = 0;
1642 ch_sn2->w_local_GP.get = 0;
1643 ch_sn2->w_local_GP.put = 0;
1644 ch_sn2->w_remote_GP.get = 0;
1645 ch_sn2->w_remote_GP.put = 0;
1646 ch_sn2->next_msg_to_pull = 0;
1647
1648 if (ch->flags & XPC_C_SETUP) {
1649 dev_dbg(xpc_chan, "ch->flags=0x%x, partid=%d, channel=%d\n",
1650 ch->flags, ch->partid, ch->number);
1651
Dean Nelson5b8669d2008-07-29 22:34:18 -07001652 kfree(ch_sn2->local_msgqueue_base);
1653 ch_sn2->local_msgqueue = NULL;
1654 kfree(ch_sn2->remote_msgqueue_base);
1655 ch_sn2->remote_msgqueue = NULL;
1656 kfree(ch_sn2->notify_queue);
1657 ch_sn2->notify_queue = NULL;
Dean Nelson185c3a12008-07-29 22:34:11 -07001658 }
1659}
1660
1661/*
Dean Nelsona47d5da2008-07-29 22:34:09 -07001662 * Notify those who wanted to be notified upon delivery of their message.
1663 */
1664static void
1665xpc_notify_senders_sn2(struct xpc_channel *ch, enum xp_retval reason, s64 put)
1666{
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001667 struct xpc_notify_sn2 *notify;
Dean Nelsona47d5da2008-07-29 22:34:09 -07001668 u8 notify_type;
1669 s64 get = ch->sn.sn2.w_remote_GP.get - 1;
1670
1671 while (++get < put && atomic_read(&ch->n_to_notify) > 0) {
1672
Dean Nelson5b8669d2008-07-29 22:34:18 -07001673 notify = &ch->sn.sn2.notify_queue[get % ch->local_nentries];
Dean Nelsona47d5da2008-07-29 22:34:09 -07001674
1675 /*
1676 * See if the notify entry indicates it was associated with
1677 * a message who's sender wants to be notified. It is possible
1678 * that it is, but someone else is doing or has done the
1679 * notification.
1680 */
1681 notify_type = notify->type;
1682 if (notify_type == 0 ||
1683 cmpxchg(&notify->type, notify_type, 0) != notify_type) {
1684 continue;
1685 }
1686
1687 DBUG_ON(notify_type != XPC_N_CALL);
1688
1689 atomic_dec(&ch->n_to_notify);
1690
1691 if (notify->func != NULL) {
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001692 dev_dbg(xpc_chan, "notify->func() called, notify=0x%p "
1693 "msg_number=%ld partid=%d channel=%d\n",
Dean Nelsona47d5da2008-07-29 22:34:09 -07001694 (void *)notify, get, ch->partid, ch->number);
1695
1696 notify->func(reason, ch->partid, ch->number,
1697 notify->key);
1698
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001699 dev_dbg(xpc_chan, "notify->func() returned, notify=0x%p"
1700 " msg_number=%ld partid=%d channel=%d\n",
1701 (void *)notify, get, ch->partid, ch->number);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001702 }
1703 }
1704}
1705
1706static void
1707xpc_notify_senders_of_disconnect_sn2(struct xpc_channel *ch)
1708{
1709 xpc_notify_senders_sn2(ch, ch->reason, ch->sn.sn2.w_local_GP.put);
1710}
1711
1712/*
1713 * Clear some of the msg flags in the local message queue.
1714 */
1715static inline void
1716xpc_clear_local_msgqueue_flags_sn2(struct xpc_channel *ch)
1717{
1718 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001719 struct xpc_msg_sn2 *msg;
Dean Nelsona47d5da2008-07-29 22:34:09 -07001720 s64 get;
1721
1722 get = ch_sn2->w_remote_GP.get;
1723 do {
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001724 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->local_msgqueue +
1725 (get % ch->local_nentries) *
1726 ch->entry_size);
Robin Holt252523e2009-01-29 14:25:07 -08001727 DBUG_ON(!(msg->flags & XPC_M_SN2_READY));
Dean Nelsona47d5da2008-07-29 22:34:09 -07001728 msg->flags = 0;
1729 } while (++get < ch_sn2->remote_GP.get);
1730}
1731
1732/*
1733 * Clear some of the msg flags in the remote message queue.
1734 */
1735static inline void
1736xpc_clear_remote_msgqueue_flags_sn2(struct xpc_channel *ch)
1737{
1738 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001739 struct xpc_msg_sn2 *msg;
Dean Nelsona47d5da2008-07-29 22:34:09 -07001740 s64 put;
1741
Robin Holt252523e2009-01-29 14:25:07 -08001742 /* flags are zeroed when the buffer is allocated */
1743 if (ch_sn2->remote_GP.put < ch->remote_nentries)
1744 return;
1745
1746 put = max(ch_sn2->w_remote_GP.put, ch->remote_nentries);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001747 do {
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001748 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue +
1749 (put % ch->remote_nentries) *
1750 ch->entry_size);
Robin Holt252523e2009-01-29 14:25:07 -08001751 DBUG_ON(!(msg->flags & XPC_M_SN2_READY));
1752 DBUG_ON(!(msg->flags & XPC_M_SN2_DONE));
1753 DBUG_ON(msg->number != put - ch->remote_nentries);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001754 msg->flags = 0;
1755 } while (++put < ch_sn2->remote_GP.put);
1756}
1757
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001758static int
1759xpc_n_of_deliverable_payloads_sn2(struct xpc_channel *ch)
1760{
1761 return ch->sn.sn2.w_remote_GP.put - ch->sn.sn2.w_local_GP.get;
1762}
1763
Dean Nelsona47d5da2008-07-29 22:34:09 -07001764static void
Dean Nelson7fb5e592008-07-29 22:34:10 -07001765xpc_process_msg_chctl_flags_sn2(struct xpc_partition *part, int ch_number)
Dean Nelsona47d5da2008-07-29 22:34:09 -07001766{
1767 struct xpc_channel *ch = &part->channels[ch_number];
1768 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001769 int npayloads_sent;
Dean Nelsona47d5da2008-07-29 22:34:09 -07001770
1771 ch_sn2->remote_GP = part->sn.sn2.remote_GPs[ch_number];
1772
1773 /* See what, if anything, has changed for each connected channel */
1774
1775 xpc_msgqueue_ref(ch);
1776
1777 if (ch_sn2->w_remote_GP.get == ch_sn2->remote_GP.get &&
1778 ch_sn2->w_remote_GP.put == ch_sn2->remote_GP.put) {
1779 /* nothing changed since GPs were last pulled */
1780 xpc_msgqueue_deref(ch);
1781 return;
1782 }
1783
1784 if (!(ch->flags & XPC_C_CONNECTED)) {
1785 xpc_msgqueue_deref(ch);
1786 return;
1787 }
1788
1789 /*
1790 * First check to see if messages recently sent by us have been
1791 * received by the other side. (The remote GET value will have
1792 * changed since we last looked at it.)
1793 */
1794
1795 if (ch_sn2->w_remote_GP.get != ch_sn2->remote_GP.get) {
1796
1797 /*
1798 * We need to notify any senders that want to be notified
1799 * that their sent messages have been received by their
1800 * intended recipients. We need to do this before updating
1801 * w_remote_GP.get so that we don't allocate the same message
1802 * queue entries prematurely (see xpc_allocate_msg()).
1803 */
1804 if (atomic_read(&ch->n_to_notify) > 0) {
1805 /*
1806 * Notify senders that messages sent have been
1807 * received and delivered by the other side.
1808 */
1809 xpc_notify_senders_sn2(ch, xpMsgDelivered,
1810 ch_sn2->remote_GP.get);
1811 }
1812
1813 /*
1814 * Clear msg->flags in previously sent messages, so that
1815 * they're ready for xpc_allocate_msg().
1816 */
1817 xpc_clear_local_msgqueue_flags_sn2(ch);
1818
1819 ch_sn2->w_remote_GP.get = ch_sn2->remote_GP.get;
1820
1821 dev_dbg(xpc_chan, "w_remote_GP.get changed to %ld, partid=%d, "
1822 "channel=%d\n", ch_sn2->w_remote_GP.get, ch->partid,
1823 ch->number);
1824
1825 /*
1826 * If anyone was waiting for message queue entries to become
1827 * available, wake them up.
1828 */
1829 if (atomic_read(&ch->n_on_msg_allocate_wq) > 0)
1830 wake_up(&ch->msg_allocate_wq);
1831 }
1832
1833 /*
1834 * Now check for newly sent messages by the other side. (The remote
1835 * PUT value will have changed since we last looked at it.)
1836 */
1837
1838 if (ch_sn2->w_remote_GP.put != ch_sn2->remote_GP.put) {
1839 /*
1840 * Clear msg->flags in previously received messages, so that
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001841 * they're ready for xpc_get_deliverable_payload_sn2().
Dean Nelsona47d5da2008-07-29 22:34:09 -07001842 */
1843 xpc_clear_remote_msgqueue_flags_sn2(ch);
1844
Robin Holt69b3bb62009-01-29 14:25:06 -08001845 smp_wmb(); /* ensure flags have been cleared before bte_copy */
Dean Nelsona47d5da2008-07-29 22:34:09 -07001846 ch_sn2->w_remote_GP.put = ch_sn2->remote_GP.put;
1847
1848 dev_dbg(xpc_chan, "w_remote_GP.put changed to %ld, partid=%d, "
1849 "channel=%d\n", ch_sn2->w_remote_GP.put, ch->partid,
1850 ch->number);
1851
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001852 npayloads_sent = xpc_n_of_deliverable_payloads_sn2(ch);
1853 if (npayloads_sent > 0) {
Dean Nelsona47d5da2008-07-29 22:34:09 -07001854 dev_dbg(xpc_chan, "msgs waiting to be copied and "
1855 "delivered=%d, partid=%d, channel=%d\n",
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001856 npayloads_sent, ch->partid, ch->number);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001857
1858 if (ch->flags & XPC_C_CONNECTEDCALLOUT_MADE)
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001859 xpc_activate_kthreads(ch, npayloads_sent);
Dean Nelsona47d5da2008-07-29 22:34:09 -07001860 }
1861 }
1862
1863 xpc_msgqueue_deref(ch);
1864}
1865
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001866static struct xpc_msg_sn2 *
Dean Nelsone17d4162008-07-29 22:34:06 -07001867xpc_pull_remote_msg_sn2(struct xpc_channel *ch, s64 get)
1868{
1869 struct xpc_partition *part = &xpc_partitions[ch->partid];
Dean Nelsona47d5da2008-07-29 22:34:09 -07001870 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelsona812dcc2008-07-29 22:34:16 -07001871 unsigned long remote_msg_pa;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001872 struct xpc_msg_sn2 *msg;
Dean Nelsona812dcc2008-07-29 22:34:16 -07001873 u32 msg_index;
1874 u32 nmsgs;
Dean Nelsone17d4162008-07-29 22:34:06 -07001875 u64 msg_offset;
1876 enum xp_retval ret;
1877
Dean Nelsona47d5da2008-07-29 22:34:09 -07001878 if (mutex_lock_interruptible(&ch_sn2->msg_to_pull_mutex) != 0) {
Dean Nelsone17d4162008-07-29 22:34:06 -07001879 /* we were interrupted by a signal */
1880 return NULL;
1881 }
1882
Dean Nelsona47d5da2008-07-29 22:34:09 -07001883 while (get >= ch_sn2->next_msg_to_pull) {
Dean Nelsone17d4162008-07-29 22:34:06 -07001884
1885 /* pull as many messages as are ready and able to be pulled */
1886
Dean Nelsona47d5da2008-07-29 22:34:09 -07001887 msg_index = ch_sn2->next_msg_to_pull % ch->remote_nentries;
Dean Nelsone17d4162008-07-29 22:34:06 -07001888
Dean Nelsona47d5da2008-07-29 22:34:09 -07001889 DBUG_ON(ch_sn2->next_msg_to_pull >= ch_sn2->w_remote_GP.put);
1890 nmsgs = ch_sn2->w_remote_GP.put - ch_sn2->next_msg_to_pull;
Dean Nelsone17d4162008-07-29 22:34:06 -07001891 if (msg_index + nmsgs > ch->remote_nentries) {
1892 /* ignore the ones that wrap the msg queue for now */
1893 nmsgs = ch->remote_nentries - msg_index;
1894 }
1895
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001896 msg_offset = msg_index * ch->entry_size;
1897 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue +
Dean Nelson5b8669d2008-07-29 22:34:18 -07001898 msg_offset);
1899 remote_msg_pa = ch_sn2->remote_msgqueue_pa + msg_offset;
Dean Nelsone17d4162008-07-29 22:34:06 -07001900
Dean Nelsona812dcc2008-07-29 22:34:16 -07001901 ret = xpc_pull_remote_cachelines_sn2(part, msg, remote_msg_pa,
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001902 nmsgs * ch->entry_size);
Dean Nelsone17d4162008-07-29 22:34:06 -07001903 if (ret != xpSuccess) {
1904
1905 dev_dbg(xpc_chan, "failed to pull %d msgs starting with"
1906 " msg %ld from partition %d, channel=%d, "
Dean Nelsona47d5da2008-07-29 22:34:09 -07001907 "ret=%d\n", nmsgs, ch_sn2->next_msg_to_pull,
Dean Nelsone17d4162008-07-29 22:34:06 -07001908 ch->partid, ch->number, ret);
1909
1910 XPC_DEACTIVATE_PARTITION(part, ret);
1911
Dean Nelsona47d5da2008-07-29 22:34:09 -07001912 mutex_unlock(&ch_sn2->msg_to_pull_mutex);
Dean Nelsone17d4162008-07-29 22:34:06 -07001913 return NULL;
1914 }
1915
Dean Nelsona47d5da2008-07-29 22:34:09 -07001916 ch_sn2->next_msg_to_pull += nmsgs;
Dean Nelsone17d4162008-07-29 22:34:06 -07001917 }
1918
Dean Nelsona47d5da2008-07-29 22:34:09 -07001919 mutex_unlock(&ch_sn2->msg_to_pull_mutex);
Dean Nelsone17d4162008-07-29 22:34:06 -07001920
1921 /* return the message we were looking for */
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001922 msg_offset = (get % ch->remote_nentries) * ch->entry_size;
1923 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue + msg_offset);
Dean Nelsone17d4162008-07-29 22:34:06 -07001924
1925 return msg;
1926}
1927
1928/*
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001929 * Get the next deliverable message's payload.
Dean Nelsone17d4162008-07-29 22:34:06 -07001930 */
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001931static void *
1932xpc_get_deliverable_payload_sn2(struct xpc_channel *ch)
Dean Nelsone17d4162008-07-29 22:34:06 -07001933{
Dean Nelsona47d5da2008-07-29 22:34:09 -07001934 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001935 struct xpc_msg_sn2 *msg;
1936 void *payload = NULL;
Dean Nelsone17d4162008-07-29 22:34:06 -07001937 s64 get;
1938
1939 do {
1940 if (ch->flags & XPC_C_DISCONNECTING)
1941 break;
1942
Dean Nelsona47d5da2008-07-29 22:34:09 -07001943 get = ch_sn2->w_local_GP.get;
Robin Holt69b3bb62009-01-29 14:25:06 -08001944 smp_rmb(); /* guarantee that .get loads before .put */
Dean Nelsona47d5da2008-07-29 22:34:09 -07001945 if (get == ch_sn2->w_remote_GP.put)
Dean Nelsone17d4162008-07-29 22:34:06 -07001946 break;
1947
1948 /* There are messages waiting to be pulled and delivered.
1949 * We need to try to secure one for ourselves. We'll do this
1950 * by trying to increment w_local_GP.get and hope that no one
1951 * else beats us to it. If they do, we'll we'll simply have
1952 * to try again for the next one.
1953 */
1954
Dean Nelsona47d5da2008-07-29 22:34:09 -07001955 if (cmpxchg(&ch_sn2->w_local_GP.get, get, get + 1) == get) {
Dean Nelsone17d4162008-07-29 22:34:06 -07001956 /* we got the entry referenced by get */
1957
1958 dev_dbg(xpc_chan, "w_local_GP.get changed to %ld, "
1959 "partid=%d, channel=%d\n", get + 1,
1960 ch->partid, ch->number);
1961
1962 /* pull the message from the remote partition */
1963
1964 msg = xpc_pull_remote_msg_sn2(ch, get);
1965
Robin Holt17e21612009-01-29 14:25:07 -08001966 if (msg != NULL) {
1967 DBUG_ON(msg->number != get);
1968 DBUG_ON(msg->flags & XPC_M_SN2_DONE);
1969 DBUG_ON(!(msg->flags & XPC_M_SN2_READY));
Dean Nelsone17d4162008-07-29 22:34:06 -07001970
Robin Holt17e21612009-01-29 14:25:07 -08001971 payload = &msg->payload;
1972 }
Dean Nelsone17d4162008-07-29 22:34:06 -07001973 break;
1974 }
1975
1976 } while (1);
1977
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001978 return payload;
Dean Nelsone17d4162008-07-29 22:34:06 -07001979}
1980
Dean Nelson33ba3c72008-07-29 22:34:07 -07001981/*
1982 * Now we actually send the messages that are ready to be sent by advancing
Dean Nelson7fb5e592008-07-29 22:34:10 -07001983 * the local message queue's Put value and then send a chctl msgrequest to the
1984 * recipient partition.
Dean Nelson33ba3c72008-07-29 22:34:07 -07001985 */
1986static void
1987xpc_send_msgs_sn2(struct xpc_channel *ch, s64 initial_put)
1988{
Dean Nelsona47d5da2008-07-29 22:34:09 -07001989 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07001990 struct xpc_msg_sn2 *msg;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001991 s64 put = initial_put + 1;
Dean Nelson7fb5e592008-07-29 22:34:10 -07001992 int send_msgrequest = 0;
Dean Nelson33ba3c72008-07-29 22:34:07 -07001993
1994 while (1) {
1995
1996 while (1) {
Dean Nelsona47d5da2008-07-29 22:34:09 -07001997 if (put == ch_sn2->w_local_GP.put)
Dean Nelson33ba3c72008-07-29 22:34:07 -07001998 break;
1999
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002000 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->
2001 local_msgqueue + (put %
2002 ch->local_nentries) *
2003 ch->entry_size);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002004
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002005 if (!(msg->flags & XPC_M_SN2_READY))
Dean Nelson33ba3c72008-07-29 22:34:07 -07002006 break;
2007
2008 put++;
2009 }
2010
2011 if (put == initial_put) {
2012 /* nothing's changed */
2013 break;
2014 }
2015
Dean Nelsona47d5da2008-07-29 22:34:09 -07002016 if (cmpxchg_rel(&ch_sn2->local_GP->put, initial_put, put) !=
Dean Nelson33ba3c72008-07-29 22:34:07 -07002017 initial_put) {
2018 /* someone else beat us to it */
Dean Nelsona47d5da2008-07-29 22:34:09 -07002019 DBUG_ON(ch_sn2->local_GP->put < initial_put);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002020 break;
2021 }
2022
2023 /* we just set the new value of local_GP->put */
2024
2025 dev_dbg(xpc_chan, "local_GP->put changed to %ld, partid=%d, "
2026 "channel=%d\n", put, ch->partid, ch->number);
2027
Dean Nelson7fb5e592008-07-29 22:34:10 -07002028 send_msgrequest = 1;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002029
2030 /*
2031 * We need to ensure that the message referenced by
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002032 * local_GP->put is not XPC_M_SN2_READY or that local_GP->put
Dean Nelson33ba3c72008-07-29 22:34:07 -07002033 * equals w_local_GP.put, so we'll go have a look.
2034 */
2035 initial_put = put;
2036 }
2037
Dean Nelson7fb5e592008-07-29 22:34:10 -07002038 if (send_msgrequest)
2039 xpc_send_chctl_msgrequest_sn2(ch);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002040}
2041
2042/*
2043 * Allocate an entry for a message from the message queue associated with the
2044 * specified channel.
2045 */
2046static enum xp_retval
2047xpc_allocate_msg_sn2(struct xpc_channel *ch, u32 flags,
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002048 struct xpc_msg_sn2 **address_of_msg)
Dean Nelson33ba3c72008-07-29 22:34:07 -07002049{
Dean Nelsona47d5da2008-07-29 22:34:09 -07002050 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002051 struct xpc_msg_sn2 *msg;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002052 enum xp_retval ret;
2053 s64 put;
2054
Dean Nelson33ba3c72008-07-29 22:34:07 -07002055 /*
2056 * Get the next available message entry from the local message queue.
2057 * If none are available, we'll make sure that we grab the latest
2058 * GP values.
2059 */
2060 ret = xpTimeout;
2061
2062 while (1) {
2063
Dean Nelsona47d5da2008-07-29 22:34:09 -07002064 put = ch_sn2->w_local_GP.put;
Robin Holt69b3bb62009-01-29 14:25:06 -08002065 smp_rmb(); /* guarantee that .put loads before .get */
Dean Nelsona47d5da2008-07-29 22:34:09 -07002066 if (put - ch_sn2->w_remote_GP.get < ch->local_nentries) {
Dean Nelson33ba3c72008-07-29 22:34:07 -07002067
2068 /* There are available message entries. We need to try
2069 * to secure one for ourselves. We'll do this by trying
2070 * to increment w_local_GP.put as long as someone else
2071 * doesn't beat us to it. If they do, we'll have to
2072 * try again.
2073 */
Dean Nelsona47d5da2008-07-29 22:34:09 -07002074 if (cmpxchg(&ch_sn2->w_local_GP.put, put, put + 1) ==
2075 put) {
Dean Nelson33ba3c72008-07-29 22:34:07 -07002076 /* we got the entry referenced by put */
2077 break;
2078 }
2079 continue; /* try again */
2080 }
2081
2082 /*
2083 * There aren't any available msg entries at this time.
2084 *
2085 * In waiting for a message entry to become available,
Dean Nelson7fb5e592008-07-29 22:34:10 -07002086 * we set a timeout in case the other side is not sending
2087 * completion interrupts. This lets us fake a notify IRQ
2088 * that will cause the notify IRQ handler to fetch the latest
2089 * GP values as if an interrupt was sent by the other side.
Dean Nelson33ba3c72008-07-29 22:34:07 -07002090 */
2091 if (ret == xpTimeout)
Dean Nelson7fb5e592008-07-29 22:34:10 -07002092 xpc_send_chctl_local_msgrequest_sn2(ch);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002093
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002094 if (flags & XPC_NOWAIT)
Dean Nelson33ba3c72008-07-29 22:34:07 -07002095 return xpNoWait;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002096
2097 ret = xpc_allocate_msg_wait(ch);
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002098 if (ret != xpInterrupted && ret != xpTimeout)
Dean Nelson33ba3c72008-07-29 22:34:07 -07002099 return ret;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002100 }
2101
2102 /* get the message's address and initialize it */
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002103 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->local_msgqueue +
2104 (put % ch->local_nentries) *
2105 ch->entry_size);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002106
2107 DBUG_ON(msg->flags != 0);
2108 msg->number = put;
2109
2110 dev_dbg(xpc_chan, "w_local_GP.put changed to %ld; msg=0x%p, "
2111 "msg_number=%ld, partid=%d, channel=%d\n", put + 1,
2112 (void *)msg, msg->number, ch->partid, ch->number);
2113
2114 *address_of_msg = msg;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002115 return xpSuccess;
2116}
2117
2118/*
2119 * Common code that does the actual sending of the message by advancing the
Dean Nelson7fb5e592008-07-29 22:34:10 -07002120 * local message queue's Put value and sends a chctl msgrequest to the
2121 * partition the message is being sent to.
Dean Nelson33ba3c72008-07-29 22:34:07 -07002122 */
2123static enum xp_retval
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002124xpc_send_payload_sn2(struct xpc_channel *ch, u32 flags, void *payload,
2125 u16 payload_size, u8 notify_type, xpc_notify_func func,
2126 void *key)
Dean Nelson33ba3c72008-07-29 22:34:07 -07002127{
2128 enum xp_retval ret = xpSuccess;
Dean Nelson5b8669d2008-07-29 22:34:18 -07002129 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002130 struct xpc_msg_sn2 *msg = msg;
2131 struct xpc_notify_sn2 *notify = notify;
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002132 s64 msg_number;
2133 s64 put;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002134
2135 DBUG_ON(notify_type == XPC_N_CALL && func == NULL);
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002136
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002137 if (XPC_MSG_SIZE(payload_size) > ch->entry_size)
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002138 return xpPayloadTooBig;
2139
2140 xpc_msgqueue_ref(ch);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002141
2142 if (ch->flags & XPC_C_DISCONNECTING) {
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002143 ret = ch->reason;
2144 goto out_1;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002145 }
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002146 if (!(ch->flags & XPC_C_CONNECTED)) {
2147 ret = xpNotConnected;
2148 goto out_1;
2149 }
2150
2151 ret = xpc_allocate_msg_sn2(ch, flags, &msg);
2152 if (ret != xpSuccess)
2153 goto out_1;
2154
2155 msg_number = msg->number;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002156
2157 if (notify_type != 0) {
2158 /*
2159 * Tell the remote side to send an ACK interrupt when the
2160 * message has been delivered.
2161 */
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002162 msg->flags |= XPC_M_SN2_INTERRUPT;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002163
2164 atomic_inc(&ch->n_to_notify);
2165
Dean Nelson5b8669d2008-07-29 22:34:18 -07002166 notify = &ch_sn2->notify_queue[msg_number % ch->local_nentries];
Dean Nelson33ba3c72008-07-29 22:34:07 -07002167 notify->func = func;
2168 notify->key = key;
2169 notify->type = notify_type;
2170
Dean Nelsonea57f802008-07-29 22:34:14 -07002171 /* ??? Is a mb() needed here? */
Dean Nelson33ba3c72008-07-29 22:34:07 -07002172
2173 if (ch->flags & XPC_C_DISCONNECTING) {
2174 /*
2175 * An error occurred between our last error check and
2176 * this one. We will try to clear the type field from
2177 * the notify entry. If we succeed then
2178 * xpc_disconnect_channel() didn't already process
2179 * the notify entry.
2180 */
2181 if (cmpxchg(&notify->type, notify_type, 0) ==
2182 notify_type) {
2183 atomic_dec(&ch->n_to_notify);
2184 ret = ch->reason;
2185 }
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002186 goto out_1;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002187 }
2188 }
2189
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002190 memcpy(&msg->payload, payload, payload_size);
2191
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002192 msg->flags |= XPC_M_SN2_READY;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002193
2194 /*
2195 * The preceding store of msg->flags must occur before the following
Dean Nelsona47d5da2008-07-29 22:34:09 -07002196 * load of local_GP->put.
Dean Nelson33ba3c72008-07-29 22:34:07 -07002197 */
Robin Holt69b3bb62009-01-29 14:25:06 -08002198 smp_mb();
Dean Nelson33ba3c72008-07-29 22:34:07 -07002199
2200 /* see if the message is next in line to be sent, if so send it */
2201
Dean Nelson5b8669d2008-07-29 22:34:18 -07002202 put = ch_sn2->local_GP->put;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002203 if (put == msg_number)
2204 xpc_send_msgs_sn2(ch, put);
2205
Dean Nelson97bf1aa2008-07-29 22:34:08 -07002206out_1:
Dean Nelson33ba3c72008-07-29 22:34:07 -07002207 xpc_msgqueue_deref(ch);
2208 return ret;
2209}
2210
2211/*
2212 * Now we actually acknowledge the messages that have been delivered and ack'd
2213 * by advancing the cached remote message queue's Get value and if requested
Dean Nelson7fb5e592008-07-29 22:34:10 -07002214 * send a chctl msgrequest to the message sender's partition.
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002215 *
2216 * If a message has XPC_M_SN2_INTERRUPT set, send an interrupt to the partition
2217 * that sent the message.
Dean Nelson33ba3c72008-07-29 22:34:07 -07002218 */
2219static void
2220xpc_acknowledge_msgs_sn2(struct xpc_channel *ch, s64 initial_get, u8 msg_flags)
2221{
Dean Nelsona47d5da2008-07-29 22:34:09 -07002222 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002223 struct xpc_msg_sn2 *msg;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002224 s64 get = initial_get + 1;
Dean Nelson7fb5e592008-07-29 22:34:10 -07002225 int send_msgrequest = 0;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002226
2227 while (1) {
2228
2229 while (1) {
Dean Nelsona47d5da2008-07-29 22:34:09 -07002230 if (get == ch_sn2->w_local_GP.get)
Dean Nelson33ba3c72008-07-29 22:34:07 -07002231 break;
2232
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002233 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->
2234 remote_msgqueue + (get %
2235 ch->remote_nentries) *
2236 ch->entry_size);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002237
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002238 if (!(msg->flags & XPC_M_SN2_DONE))
Dean Nelson33ba3c72008-07-29 22:34:07 -07002239 break;
2240
2241 msg_flags |= msg->flags;
2242 get++;
2243 }
2244
2245 if (get == initial_get) {
2246 /* nothing's changed */
2247 break;
2248 }
2249
Dean Nelsona47d5da2008-07-29 22:34:09 -07002250 if (cmpxchg_rel(&ch_sn2->local_GP->get, initial_get, get) !=
Dean Nelson33ba3c72008-07-29 22:34:07 -07002251 initial_get) {
2252 /* someone else beat us to it */
Dean Nelsona47d5da2008-07-29 22:34:09 -07002253 DBUG_ON(ch_sn2->local_GP->get <= initial_get);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002254 break;
2255 }
2256
2257 /* we just set the new value of local_GP->get */
2258
2259 dev_dbg(xpc_chan, "local_GP->get changed to %ld, partid=%d, "
2260 "channel=%d\n", get, ch->partid, ch->number);
2261
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002262 send_msgrequest = (msg_flags & XPC_M_SN2_INTERRUPT);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002263
2264 /*
2265 * We need to ensure that the message referenced by
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002266 * local_GP->get is not XPC_M_SN2_DONE or that local_GP->get
Dean Nelson33ba3c72008-07-29 22:34:07 -07002267 * equals w_local_GP.get, so we'll go have a look.
2268 */
2269 initial_get = get;
2270 }
2271
Dean Nelson7fb5e592008-07-29 22:34:10 -07002272 if (send_msgrequest)
2273 xpc_send_chctl_msgrequest_sn2(ch);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002274}
2275
2276static void
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002277xpc_received_payload_sn2(struct xpc_channel *ch, void *payload)
Dean Nelson33ba3c72008-07-29 22:34:07 -07002278{
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002279 struct xpc_msg_sn2 *msg;
2280 s64 msg_number;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002281 s64 get;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002282
2283 msg = container_of(payload, struct xpc_msg_sn2, payload);
2284 msg_number = msg->number;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002285
2286 dev_dbg(xpc_chan, "msg=0x%p, msg_number=%ld, partid=%d, channel=%d\n",
2287 (void *)msg, msg_number, ch->partid, ch->number);
2288
Robin Holt252523e2009-01-29 14:25:07 -08002289 DBUG_ON((((u64)msg - (u64)ch->sn.sn2.remote_msgqueue) / ch->entry_size) !=
Dean Nelson33ba3c72008-07-29 22:34:07 -07002290 msg_number % ch->remote_nentries);
Robin Holt252523e2009-01-29 14:25:07 -08002291 DBUG_ON(!(msg->flags & XPC_M_SN2_READY));
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002292 DBUG_ON(msg->flags & XPC_M_SN2_DONE);
Dean Nelson33ba3c72008-07-29 22:34:07 -07002293
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002294 msg->flags |= XPC_M_SN2_DONE;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002295
2296 /*
2297 * The preceding store of msg->flags must occur before the following
Dean Nelsona47d5da2008-07-29 22:34:09 -07002298 * load of local_GP->get.
Dean Nelson33ba3c72008-07-29 22:34:07 -07002299 */
Robin Holt69b3bb62009-01-29 14:25:06 -08002300 smp_mb();
Dean Nelson33ba3c72008-07-29 22:34:07 -07002301
2302 /*
2303 * See if this message is next in line to be acknowledged as having
2304 * been delivered.
2305 */
Dean Nelsona47d5da2008-07-29 22:34:09 -07002306 get = ch->sn.sn2.local_GP->get;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002307 if (get == msg_number)
2308 xpc_acknowledge_msgs_sn2(ch, get, msg->flags);
2309}
2310
Dean Nelson6e41017a2008-07-29 22:34:09 -07002311int
Dean Nelson94bd2702008-07-29 22:34:05 -07002312xpc_init_sn2(void)
2313{
Dean Nelson6e41017a2008-07-29 22:34:09 -07002314 int ret;
Dean Nelsonee6665e2008-07-29 22:34:13 -07002315 size_t buf_size;
Dean Nelson6e41017a2008-07-29 22:34:09 -07002316
Dean Nelson5b8669d2008-07-29 22:34:18 -07002317 xpc_setup_partitions_sn = xpc_setup_partitions_sn_sn2;
Dean Nelson261f3b42008-07-29 22:34:16 -07002318 xpc_get_partition_rsvd_page_pa = xpc_get_partition_rsvd_page_pa_sn2;
Dean Nelson5b8669d2008-07-29 22:34:18 -07002319 xpc_setup_rsvd_page_sn = xpc_setup_rsvd_page_sn_sn2;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002320 xpc_increment_heartbeat = xpc_increment_heartbeat_sn2;
2321 xpc_offline_heartbeat = xpc_offline_heartbeat_sn2;
2322 xpc_online_heartbeat = xpc_online_heartbeat_sn2;
2323 xpc_heartbeat_init = xpc_heartbeat_init_sn2;
2324 xpc_heartbeat_exit = xpc_heartbeat_exit_sn2;
Dean Nelson61deb862008-07-29 22:34:17 -07002325 xpc_get_remote_heartbeat = xpc_get_remote_heartbeat_sn2;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002326
Dean Nelsona47d5da2008-07-29 22:34:09 -07002327 xpc_request_partition_activation = xpc_request_partition_activation_sn2;
2328 xpc_request_partition_reactivation =
2329 xpc_request_partition_reactivation_sn2;
2330 xpc_request_partition_deactivation =
2331 xpc_request_partition_deactivation_sn2;
2332 xpc_cancel_partition_deactivation_request =
2333 xpc_cancel_partition_deactivation_request_sn2;
2334
Dean Nelson6e41017a2008-07-29 22:34:09 -07002335 xpc_process_activate_IRQ_rcvd = xpc_process_activate_IRQ_rcvd_sn2;
Dean Nelson5b8669d2008-07-29 22:34:18 -07002336 xpc_setup_ch_structures_sn = xpc_setup_ch_structures_sn_sn2;
2337 xpc_teardown_ch_structures_sn = xpc_teardown_ch_structures_sn_sn2;
Dean Nelsone17d4162008-07-29 22:34:06 -07002338 xpc_make_first_contact = xpc_make_first_contact_sn2;
Dean Nelson5b8669d2008-07-29 22:34:18 -07002339
Dean Nelson7fb5e592008-07-29 22:34:10 -07002340 xpc_get_chctl_all_flags = xpc_get_chctl_all_flags_sn2;
Dean Nelson5b8669d2008-07-29 22:34:18 -07002341 xpc_send_chctl_closerequest = xpc_send_chctl_closerequest_sn2;
2342 xpc_send_chctl_closereply = xpc_send_chctl_closereply_sn2;
2343 xpc_send_chctl_openrequest = xpc_send_chctl_openrequest_sn2;
2344 xpc_send_chctl_openreply = xpc_send_chctl_openreply_sn2;
2345
2346 xpc_save_remote_msgqueue_pa = xpc_save_remote_msgqueue_pa_sn2;
2347
2348 xpc_setup_msg_structures = xpc_setup_msg_structures_sn2;
2349 xpc_teardown_msg_structures = xpc_teardown_msg_structures_sn2;
2350
Dean Nelsona47d5da2008-07-29 22:34:09 -07002351 xpc_notify_senders_of_disconnect = xpc_notify_senders_of_disconnect_sn2;
Dean Nelson7fb5e592008-07-29 22:34:10 -07002352 xpc_process_msg_chctl_flags = xpc_process_msg_chctl_flags_sn2;
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002353 xpc_n_of_deliverable_payloads = xpc_n_of_deliverable_payloads_sn2;
2354 xpc_get_deliverable_payload = xpc_get_deliverable_payload_sn2;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002355
Dean Nelsona47d5da2008-07-29 22:34:09 -07002356 xpc_indicate_partition_engaged = xpc_indicate_partition_engaged_sn2;
Dean Nelsona47d5da2008-07-29 22:34:09 -07002357 xpc_indicate_partition_disengaged =
2358 xpc_indicate_partition_disengaged_sn2;
Dean Nelson5b8669d2008-07-29 22:34:18 -07002359 xpc_partition_engaged = xpc_partition_engaged_sn2;
2360 xpc_any_partition_engaged = xpc_any_partition_engaged_sn2;
Dean Nelsona47d5da2008-07-29 22:34:09 -07002361 xpc_assume_partition_disengaged = xpc_assume_partition_disengaged_sn2;
Dean Nelson33ba3c72008-07-29 22:34:07 -07002362
Dean Nelsonbd3e64c2008-07-29 22:34:19 -07002363 xpc_send_payload = xpc_send_payload_sn2;
2364 xpc_received_payload = xpc_received_payload_sn2;
2365
2366 if (offsetof(struct xpc_msg_sn2, payload) > XPC_MSG_HDR_MAX_SIZE) {
2367 dev_err(xpc_part, "header portion of struct xpc_msg_sn2 is "
2368 "larger than %d\n", XPC_MSG_HDR_MAX_SIZE);
2369 return -E2BIG;
2370 }
Dean Nelson6e41017a2008-07-29 22:34:09 -07002371
Dean Nelsonee6665e2008-07-29 22:34:13 -07002372 buf_size = max(XPC_RP_VARS_SIZE,
2373 XPC_RP_HEADER_SIZE + XP_NASID_MASK_BYTES_SN2);
2374 xpc_remote_copy_buffer_sn2 = xpc_kmalloc_cacheline_aligned(buf_size,
2375 GFP_KERNEL,
2376 &xpc_remote_copy_buffer_base_sn2);
2377 if (xpc_remote_copy_buffer_sn2 == NULL) {
2378 dev_err(xpc_part, "can't get memory for remote copy buffer\n");
2379 return -ENOMEM;
2380 }
2381
Dean Nelsonc39838c2008-07-29 22:34:11 -07002382 /* open up protections for IPI and [potentially] amo operations */
Dean Nelson6e41017a2008-07-29 22:34:09 -07002383 xpc_allow_IPI_ops_sn2();
Dean Nelsonc39838c2008-07-29 22:34:11 -07002384 xpc_allow_amo_ops_shub_wars_1_1_sn2();
Dean Nelson6e41017a2008-07-29 22:34:09 -07002385
2386 /*
2387 * This is safe to do before the xpc_hb_checker thread has started
2388 * because the handler releases a wait queue. If an interrupt is
2389 * received before the thread is waiting, it will not go to sleep,
2390 * but rather immediately process the interrupt.
2391 */
2392 ret = request_irq(SGI_XPC_ACTIVATE, xpc_handle_activate_IRQ_sn2, 0,
2393 "xpc hb", NULL);
2394 if (ret != 0) {
2395 dev_err(xpc_part, "can't register ACTIVATE IRQ handler, "
2396 "errno=%d\n", -ret);
2397 xpc_disallow_IPI_ops_sn2();
Dean Nelsonee6665e2008-07-29 22:34:13 -07002398 kfree(xpc_remote_copy_buffer_base_sn2);
Dean Nelson6e41017a2008-07-29 22:34:09 -07002399 }
2400 return ret;
Dean Nelson94bd2702008-07-29 22:34:05 -07002401}
2402
2403void
2404xpc_exit_sn2(void)
2405{
Dean Nelson6e41017a2008-07-29 22:34:09 -07002406 free_irq(SGI_XPC_ACTIVATE, NULL);
2407 xpc_disallow_IPI_ops_sn2();
Dean Nelsonee6665e2008-07-29 22:34:13 -07002408 kfree(xpc_remote_copy_buffer_base_sn2);
Dean Nelson94bd2702008-07-29 22:34:05 -07002409}