From: Andi Kleen <ak@muc.de>
Turn per cpu TSS into per cpu data.
Signed-off-by: Andi Kleen <ak@muc.de>
Signed-off-by: Andrew Morton <akpm@osdl.org>
---
25-akpm/arch/x86_64/kernel/init_task.c | 3 +--
25-akpm/arch/x86_64/kernel/ioport.c | 2 +-
25-akpm/arch/x86_64/kernel/process.c | 4 ++--
25-akpm/arch/x86_64/kernel/setup64.c | 2 +-
25-akpm/arch/x86_64/kernel/smpboot.c | 2 +-
25-akpm/arch/x86_64/kernel/suspend.c | 2 +-
25-akpm/arch/x86_64/kernel/traps.c | 5 +++--
25-akpm/include/asm-x86_64/processor.h | 7 ++++---
8 files changed, 14 insertions(+), 13 deletions(-)
diff -puN arch/x86_64/kernel/init_task.c~x86-64-turn-tss-into-per-cpu-data arch/x86_64/kernel/init_task.c
--- 25/arch/x86_64/kernel/init_task.c~x86-64-turn-tss-into-per-cpu-data 2004-09-15 02:32:32.583014856 -0700
+++ 25-akpm/arch/x86_64/kernel/init_task.c 2004-09-15 02:32:32.601012120 -0700
@@ -44,8 +44,7 @@ EXPORT_SYMBOL(init_task);
* section. Since TSS's are completely CPU-local, we want them
* on exact cacheline boundaries, to eliminate cacheline ping-pong.
*/
-struct tss_struct init_tss[NR_CPUS] __cacheline_aligned;
-
+DEFINE_PER_CPU(struct tss_struct, init_tss) ____cacheline_maxaligned_in_smp;
#define ALIGN_TO_4K __attribute__((section(".data.init_task")))
diff -puN arch/x86_64/kernel/ioport.c~x86-64-turn-tss-into-per-cpu-data arch/x86_64/kernel/ioport.c
--- 25/arch/x86_64/kernel/ioport.c~x86-64-turn-tss-into-per-cpu-data 2004-09-15 02:32:32.585014552 -0700
+++ 25-akpm/arch/x86_64/kernel/ioport.c 2004-09-15 02:32:32.597012728 -0700
@@ -61,7 +61,7 @@ asmlinkage long sys_ioperm(unsigned long
* do it in the per-thread copy and in the TSS ...
*/
set_bitmap(t->io_bitmap_ptr, from, num, !turn_on);
- tss = init_tss + get_cpu();
+ tss = &per_cpu(init_tss, get_cpu());
if (tss->io_bitmap_base == IO_BITMAP_OFFSET) { /* already active? */
set_bitmap(tss->io_bitmap, from, num, !turn_on);
} else {
diff -puN arch/x86_64/kernel/process.c~x86-64-turn-tss-into-per-cpu-data arch/x86_64/kernel/process.c
--- 25/arch/x86_64/kernel/process.c~x86-64-turn-tss-into-per-cpu-data 2004-09-15 02:32:32.586014400 -0700
+++ 25-akpm/arch/x86_64/kernel/process.c 2004-09-15 02:32:32.598012576 -0700
@@ -251,7 +251,7 @@ void exit_thread(void)
{
struct task_struct *me = current;
if (me->thread.io_bitmap_ptr) {
- struct tss_struct *tss = init_tss + get_cpu();
+ struct tss_struct *tss = &per_cpu(init_tss,get_cpu());
kfree(me->thread.io_bitmap_ptr);
me->thread.io_bitmap_ptr = NULL;
tss->io_bitmap_base = INVALID_IO_BITMAP_OFFSET;
@@ -404,7 +404,7 @@ struct task_struct *__switch_to(struct t
struct thread_struct *prev = &prev_p->thread,
*next = &next_p->thread;
int cpu = smp_processor_id();
- struct tss_struct *tss = init_tss + cpu;
+ struct tss_struct *tss = &per_cpu(init_tss, cpu);
unlazy_fpu(prev_p);
diff -puN arch/x86_64/kernel/setup64.c~x86-64-turn-tss-into-per-cpu-data arch/x86_64/kernel/setup64.c
--- 25/arch/x86_64/kernel/setup64.c~x86-64-turn-tss-into-per-cpu-data 2004-09-15 02:32:32.588014096 -0700
+++ 25-akpm/arch/x86_64/kernel/setup64.c 2004-09-15 02:32:32.599012424 -0700
@@ -237,7 +237,7 @@ void __init cpu_init (void)
#else
int cpu = smp_processor_id();
#endif
- struct tss_struct * t = &init_tss[cpu];
+ struct tss_struct *t = &per_cpu(init_tss, cpu);
unsigned long v;
char *estacks = NULL;
struct task_struct *me;
diff -puN arch/x86_64/kernel/smpboot.c~x86-64-turn-tss-into-per-cpu-data arch/x86_64/kernel/smpboot.c
--- 25/arch/x86_64/kernel/smpboot.c~x86-64-turn-tss-into-per-cpu-data 2004-09-15 02:32:32.590013792 -0700
+++ 25-akpm/arch/x86_64/kernel/smpboot.c 2004-09-15 02:32:32.601012120 -0700
@@ -575,7 +575,7 @@ static void __init do_boot_cpu (int apic
start_rip = setup_trampoline();
init_rsp = idle->thread.rsp;
- init_tss[cpu].rsp0 = init_rsp;
+ per_cpu(init_tss,cpu).rsp0 = init_rsp;
initial_code = start_secondary;
clear_ti_thread_flag(idle->thread_info, TIF_FORK);
diff -puN arch/x86_64/kernel/suspend.c~x86-64-turn-tss-into-per-cpu-data arch/x86_64/kernel/suspend.c
--- 25/arch/x86_64/kernel/suspend.c~x86-64-turn-tss-into-per-cpu-data 2004-09-15 02:32:32.591013640 -0700
+++ 25-akpm/arch/x86_64/kernel/suspend.c 2004-09-15 02:32:32.598012576 -0700
@@ -129,7 +129,7 @@ void restore_processor_state(void)
void fix_processor_context(void)
{
int cpu = smp_processor_id();
- struct tss_struct * t = init_tss + cpu;
+ struct tss_struct *t = &per_cpu(init_tss, cpu);
set_tss_desc(cpu,t); /* This just modifies memory; should not be neccessary. But... This is neccessary, because 386 hardware has concept of busy TSS or some similar stupidity. */
diff -puN arch/x86_64/kernel/traps.c~x86-64-turn-tss-into-per-cpu-data arch/x86_64/kernel/traps.c
--- 25/arch/x86_64/kernel/traps.c~x86-64-turn-tss-into-per-cpu-data 2004-09-15 02:32:32.593013336 -0700
+++ 25-akpm/arch/x86_64/kernel/traps.c 2004-09-15 02:32:32.600012272 -0700
@@ -113,9 +113,10 @@ unsigned long *in_exception_stack(int cp
{
int k;
for (k = 0; k < N_EXCEPTION_STACKS; k++) {
- unsigned long end = init_tss[cpu].ist[k] + EXCEPTION_STKSZ;
+ struct tss_struct *tss = &per_cpu(init_tss, cpu);
+ unsigned long end = tss->ist[k] + EXCEPTION_STKSZ;
- if (stack >= init_tss[cpu].ist[k] && stack <= end)
+ if (stack >= tss->ist[k] && stack <= end)
return (unsigned long *)end;
}
return NULL;
diff -puN include/asm-x86_64/processor.h~x86-64-turn-tss-into-per-cpu-data include/asm-x86_64/processor.h
--- 25/include/asm-x86_64/processor.h~x86-64-turn-tss-into-per-cpu-data 2004-09-15 02:32:32.594013184 -0700
+++ 25-akpm/include/asm-x86_64/processor.h 2004-09-15 02:32:32.602011968 -0700
@@ -18,6 +18,7 @@
#include <asm/current.h>
#include <asm/system.h>
#include <asm/mmsegment.h>
+#include <asm/percpu.h>
#include <linux/personality.h>
#define TF_MASK 0x00000100
@@ -77,9 +78,6 @@ struct cpuinfo_x86 {
#define X86_VENDOR_NUM 8
#define X86_VENDOR_UNKNOWN 0xff
-extern struct cpuinfo_x86 boot_cpu_data;
-extern struct tss_struct init_tss[NR_CPUS];
-
#ifdef CONFIG_SMP
extern struct cpuinfo_x86 cpu_data[];
#define current_cpu_data cpu_data[smp_processor_id()]
@@ -229,6 +227,9 @@ struct tss_struct {
unsigned long io_bitmap[IO_BITMAP_LONGS + 1];
} __attribute__((packed)) ____cacheline_aligned;
+extern struct cpuinfo_x86 boot_cpu_data;
+DECLARE_PER_CPU(struct tss_struct,init_tss);
+
#define ARCH_MIN_TASKALIGN 16
struct thread_struct {
_