From 254585788fce648441c41bf2854ac0899f3e2b1a Mon Sep 17 00:00:00 2001 From: Kyle McMartin Date: Thu, 2 Dec 2010 11:57:44 -0500 Subject: [PATCH] kswapd fixes from mmotm (queuing them here since it's unclear if they'll go into .37 or wait for .38) --- kernel.spec | 10 + ...counter-threshold-when-memory-is-low.patch | 389 ++++++++++++++++++ ...back-for-adjusting-percpu-thresholds.patch | 167 ++++++++ 3 files changed, 566 insertions(+) create mode 100644 mm-page-allocator-adjust-the-per-cpu-counter-threshold-when-memory-is-low.patch create mode 100644 mm-vmstat-use-a-single-setter-function-and-callback-for-adjusting-percpu-thresholds.patch diff --git a/kernel.spec b/kernel.spec index b01ba32a2..473b6e29d 100644 --- a/kernel.spec +++ b/kernel.spec @@ -705,6 +705,9 @@ Patch12401: debug-tty-print-dev-name.patch Patch12402: tty-ldisc-fix-open-flag-handling.patch Patch12403: tty-open-hangup-race-fixup.patch +Patch12410: mm-page-allocator-adjust-the-per-cpu-counter-threshold-when-memory-is-low.patch +Patch12411: mm-vmstat-use-a-single-setter-function-and-callback-for-adjusting-percpu-thresholds.patch + %endif BuildRoot: %{_tmppath}/kernel-%{KVERREL}-root @@ -1300,6 +1303,10 @@ ApplyPatch debug-tty-print-dev-name.patch ApplyPatch tty-ldisc-fix-open-flag-handling.patch ApplyPatch tty-open-hangup-race-fixup.patch +# backport some fixes for kswapd from mmotm, rhbz#649694 +ApplyPatch mm-page-allocator-adjust-the-per-cpu-counter-threshold-when-memory-is-low.patch +ApplyPatch mm-vmstat-use-a-single-setter-function-and-callback-for-adjusting-percpu-thresholds.patch + # END OF PATCH APPLICATIONS %endif @@ -1912,6 +1919,9 @@ fi # || || %changelog +* Thu Dec 02 2010 Kyle McMartin +- Grab some of Mel's fixes from -mmotm to hopefully sort out #649694. + * Wed Dec 01 2010 Kyle McMartin 2.6.37-0.rc4.git1.1 - Linux 2.6.37-rc4-git1 - Pull in DRM fixes that are queued for -rc5 [3074adc8] diff --git a/mm-page-allocator-adjust-the-per-cpu-counter-threshold-when-memory-is-low.patch b/mm-page-allocator-adjust-the-per-cpu-counter-threshold-when-memory-is-low.patch new file mode 100644 index 000000000..561c5897e --- /dev/null +++ b/mm-page-allocator-adjust-the-per-cpu-counter-threshold-when-memory-is-low.patch @@ -0,0 +1,389 @@ +From df43fae25437d7bc7dfff72599c1e825038b67cf Mon Sep 17 00:00:00 2001 +From: Mel Gorman +Date: Wed, 24 Nov 2010 22:18:23 -0500 +Subject: [PATCH 1/2] mm: page allocator: Adjust the per-cpu counter threshold when memory is low + +Commit aa45484 ("calculate a better estimate of NR_FREE_PAGES when memory +is low") noted that watermarks were based on the vmstat NR_FREE_PAGES. To +avoid synchronization overhead, these counters are maintained on a per-cpu +basis and drained both periodically and when a threshold is above a +threshold. On large CPU systems, the difference between the estimate and +real value of NR_FREE_PAGES can be very high. The system can get into a +case where pages are allocated far below the min watermark potentially +causing livelock issues. The commit solved the problem by taking a better +reading of NR_FREE_PAGES when memory was low. + +Unfortately, as reported by Shaohua Li this accurate reading can consume a +large amount of CPU time on systems with many sockets due to cache line +bouncing. This patch takes a different approach. For large machines +where counter drift might be unsafe and while kswapd is awake, the per-cpu +thresholds for the target pgdat are reduced to limit the level of drift to +what should be a safe level. This incurs a performance penalty in heavy +memory pressure by a factor that depends on the workload and the machine +but the machine should function correctly without accidentally exhausting +all memory on a node. There is an additional cost when kswapd wakes and +sleeps but the event is not expected to be frequent - in Shaohua's test +case, there was one recorded sleep and wake event at least. + +To ensure that kswapd wakes up, a safe version of zone_watermark_ok() is +introduced that takes a more accurate reading of NR_FREE_PAGES when called +from wakeup_kswapd, when deciding whether it is really safe to go back to +sleep in sleeping_prematurely() and when deciding if a zone is really +balanced or not in balance_pgdat(). We are still using an expensive +function but limiting how often it is called. + +When the test case is reproduced, the time spent in the watermark +functions is reduced. The following report is on the percentage of time +spent cumulatively spent in the functions zone_nr_free_pages(), +zone_watermark_ok(), __zone_watermark_ok(), zone_watermark_ok_safe(), +zone_page_state_snapshot(), zone_page_state(). + +vanilla 11.6615% +disable-threshold 0.2584% + +Reported-by: Shaohua Li +Signed-off-by: Mel Gorman +Reviewed-by: Christoph Lameter +Signed-off-by: Andrew Morton +[[http://userweb.kernel.org/~akpm/mmotm/broken-out/mm-page-allocator-adjust-the-per-cpu-counter-threshold-when-memory-is-low.patch]] +--- + include/linux/mmzone.h | 10 ++----- + include/linux/vmstat.h | 5 +++ + mm/mmzone.c | 21 --------------- + mm/page_alloc.c | 35 +++++++++++++++++++----- + mm/vmscan.c | 23 +++++++++------- + mm/vmstat.c | 68 +++++++++++++++++++++++++++++++++++++++++++++++- + 6 files changed, 115 insertions(+), 47 deletions(-) + +diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h +index 3984c4e..8d789d7 100644 +--- a/include/linux/mmzone.h ++++ b/include/linux/mmzone.h +@@ -448,12 +448,6 @@ static inline int zone_is_oom_locked(const struct zone *zone) + return test_bit(ZONE_OOM_LOCKED, &zone->flags); + } + +-#ifdef CONFIG_SMP +-unsigned long zone_nr_free_pages(struct zone *zone); +-#else +-#define zone_nr_free_pages(zone) zone_page_state(zone, NR_FREE_PAGES) +-#endif /* CONFIG_SMP */ +- + /* + * The "priority" of VM scanning is how much of the queues we will scan in one + * go. A value of 12 for DEF_PRIORITY implies that we will scan 1/4096th of the +@@ -651,7 +645,9 @@ typedef struct pglist_data { + extern struct mutex zonelists_mutex; + void build_all_zonelists(void *data); + void wakeup_kswapd(struct zone *zone, int order); +-int zone_watermark_ok(struct zone *z, int order, unsigned long mark, ++bool zone_watermark_ok(struct zone *z, int order, unsigned long mark, ++ int classzone_idx, int alloc_flags); ++bool zone_watermark_ok_safe(struct zone *z, int order, unsigned long mark, + int classzone_idx, int alloc_flags); + enum memmap_context { + MEMMAP_EARLY, +diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h +index eaaea37..e4cc21c 100644 +--- a/include/linux/vmstat.h ++++ b/include/linux/vmstat.h +@@ -254,6 +254,8 @@ extern void dec_zone_state(struct zone *, enum zone_stat_item); + extern void __dec_zone_state(struct zone *, enum zone_stat_item); + + void refresh_cpu_vm_stats(int); ++void reduce_pgdat_percpu_threshold(pg_data_t *pgdat); ++void restore_pgdat_percpu_threshold(pg_data_t *pgdat); + #else /* CONFIG_SMP */ + + /* +@@ -298,6 +300,9 @@ static inline void __dec_zone_page_state(struct page *page, + #define dec_zone_page_state __dec_zone_page_state + #define mod_zone_page_state __mod_zone_page_state + ++static inline void reduce_pgdat_percpu_threshold(pg_data_t *pgdat) { } ++static inline void restore_pgdat_percpu_threshold(pg_data_t *pgdat) { } ++ + static inline void refresh_cpu_vm_stats(int cpu) { } + #endif + +diff --git a/mm/mmzone.c b/mm/mmzone.c +index e35bfb8..f5b7d17 100644 +--- a/mm/mmzone.c ++++ b/mm/mmzone.c +@@ -87,24 +87,3 @@ int memmap_valid_within(unsigned long pfn, + return 1; + } + #endif /* CONFIG_ARCH_HAS_HOLES_MEMORYMODEL */ +- +-#ifdef CONFIG_SMP +-/* Called when a more accurate view of NR_FREE_PAGES is needed */ +-unsigned long zone_nr_free_pages(struct zone *zone) +-{ +- unsigned long nr_free_pages = zone_page_state(zone, NR_FREE_PAGES); +- +- /* +- * While kswapd is awake, it is considered the zone is under some +- * memory pressure. Under pressure, there is a risk that +- * per-cpu-counter-drift will allow the min watermark to be breached +- * potentially causing a live-lock. While kswapd is awake and +- * free pages are low, get a better estimate for free pages +- */ +- if (nr_free_pages < zone->percpu_drift_mark && +- !waitqueue_active(&zone->zone_pgdat->kswapd_wait)) +- return zone_page_state_snapshot(zone, NR_FREE_PAGES); +- +- return nr_free_pages; +-} +-#endif /* CONFIG_SMP */ +diff --git a/mm/page_alloc.c b/mm/page_alloc.c +index f12ad18..0286150 100644 +--- a/mm/page_alloc.c ++++ b/mm/page_alloc.c +@@ -1454,24 +1454,24 @@ static inline int should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) + #endif /* CONFIG_FAIL_PAGE_ALLOC */ + + /* +- * Return 1 if free pages are above 'mark'. This takes into account the order ++ * Return true if free pages are above 'mark'. This takes into account the order + * of the allocation. + */ +-int zone_watermark_ok(struct zone *z, int order, unsigned long mark, +- int classzone_idx, int alloc_flags) ++static bool __zone_watermark_ok(struct zone *z, int order, unsigned long mark, ++ int classzone_idx, int alloc_flags, long free_pages) + { + /* free_pages my go negative - that's OK */ + long min = mark; +- long free_pages = zone_nr_free_pages(z) - (1 << order) + 1; + int o; + ++ free_pages -= (1 << order) + 1; + if (alloc_flags & ALLOC_HIGH) + min -= min / 2; + if (alloc_flags & ALLOC_HARDER) + min -= min / 4; + + if (free_pages <= min + z->lowmem_reserve[classzone_idx]) +- return 0; ++ return false; + for (o = 0; o < order; o++) { + /* At the next order, this order's pages become unavailable */ + free_pages -= z->free_area[o].nr_free << o; +@@ -1480,9 +1480,28 @@ int zone_watermark_ok(struct zone *z, int order, unsigned long mark, + min >>= 1; + + if (free_pages <= min) +- return 0; ++ return false; + } +- return 1; ++ return true; ++} ++ ++bool zone_watermark_ok(struct zone *z, int order, unsigned long mark, ++ int classzone_idx, int alloc_flags) ++{ ++ return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, ++ zone_page_state(z, NR_FREE_PAGES)); ++} ++ ++bool zone_watermark_ok_safe(struct zone *z, int order, unsigned long mark, ++ int classzone_idx, int alloc_flags) ++{ ++ long free_pages = zone_page_state(z, NR_FREE_PAGES); ++ ++ if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark) ++ free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES); ++ ++ return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, ++ free_pages); + } + + #ifdef CONFIG_NUMA +@@ -2436,7 +2455,7 @@ void show_free_areas(void) + " all_unreclaimable? %s" + "\n", + zone->name, +- K(zone_nr_free_pages(zone)), ++ K(zone_page_state(zone, NR_FREE_PAGES)), + K(min_wmark_pages(zone)), + K(low_wmark_pages(zone)), + K(high_wmark_pages(zone)), +diff --git a/mm/vmscan.c b/mm/vmscan.c +index c5dfabf..3e71cb1 100644 +--- a/mm/vmscan.c ++++ b/mm/vmscan.c +@@ -2082,7 +2082,7 @@ static int sleeping_prematurely(pg_data_t *pgdat, int order, long remaining) + if (zone->all_unreclaimable) + continue; + +- if (!zone_watermark_ok(zone, order, high_wmark_pages(zone), ++ if (!zone_watermark_ok_safe(zone, order, high_wmark_pages(zone), + 0, 0)) + return 1; + } +@@ -2169,7 +2169,7 @@ loop_again: + shrink_active_list(SWAP_CLUSTER_MAX, zone, + &sc, priority, 0); + +- if (!zone_watermark_ok(zone, order, ++ if (!zone_watermark_ok_safe(zone, order, + high_wmark_pages(zone), 0, 0)) { + end_zone = i; + break; +@@ -2215,7 +2215,7 @@ loop_again: + * We put equal pressure on every zone, unless one + * zone has way too many pages free already. + */ +- if (!zone_watermark_ok(zone, order, ++ if (!zone_watermark_ok_safe(zone, order, + 8*high_wmark_pages(zone), end_zone, 0)) + shrink_zone(priority, zone, &sc); + reclaim_state->reclaimed_slab = 0; +@@ -2236,7 +2236,7 @@ loop_again: + total_scanned > sc.nr_reclaimed + sc.nr_reclaimed / 2) + sc.may_writepage = 1; + +- if (!zone_watermark_ok(zone, order, ++ if (!zone_watermark_ok_safe(zone, order, + high_wmark_pages(zone), end_zone, 0)) { + all_zones_ok = 0; + /* +@@ -2244,7 +2244,7 @@ loop_again: + * means that we have a GFP_ATOMIC allocation + * failure risk. Hurry up! + */ +- if (!zone_watermark_ok(zone, order, ++ if (!zone_watermark_ok_safe(zone, order, + min_wmark_pages(zone), end_zone, 0)) + has_under_min_watermark_zone = 1; + } +@@ -2378,7 +2378,9 @@ static int kswapd(void *p) + */ + if (!sleeping_prematurely(pgdat, order, remaining)) { + trace_mm_vmscan_kswapd_sleep(pgdat->node_id); ++ restore_pgdat_percpu_threshold(pgdat); + schedule(); ++ reduce_pgdat_percpu_threshold(pgdat); + } else { + if (remaining) + count_vm_event(KSWAPD_LOW_WMARK_HIT_QUICKLY); +@@ -2417,16 +2419,17 @@ void wakeup_kswapd(struct zone *zone, int order) + if (!populated_zone(zone)) + return; + +- pgdat = zone->zone_pgdat; +- if (zone_watermark_ok(zone, order, low_wmark_pages(zone), 0, 0)) ++ if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL)) + return; ++ pgdat = zone->zone_pgdat; + if (pgdat->kswapd_max_order < order) + pgdat->kswapd_max_order = order; +- trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, zone_idx(zone), order); +- if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL)) +- return; + if (!waitqueue_active(&pgdat->kswapd_wait)) + return; ++ if (zone_watermark_ok_safe(zone, order, low_wmark_pages(zone), 0, 0)) ++ return; ++ ++ trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, zone_idx(zone), order); + wake_up_interruptible(&pgdat->kswapd_wait); + } + +diff --git a/mm/vmstat.c b/mm/vmstat.c +index 355a9e6..4d7faeb 100644 +--- a/mm/vmstat.c ++++ b/mm/vmstat.c +@@ -81,6 +81,30 @@ EXPORT_SYMBOL(vm_stat); + + #ifdef CONFIG_SMP + ++static int calculate_pressure_threshold(struct zone *zone) ++{ ++ int threshold; ++ int watermark_distance; ++ ++ /* ++ * As vmstats are not up to date, there is drift between the estimated ++ * and real values. For high thresholds and a high number of CPUs, it ++ * is possible for the min watermark to be breached while the estimated ++ * value looks fine. The pressure threshold is a reduced value such ++ * that even the maximum amount of drift will not accidentally breach ++ * the min watermark ++ */ ++ watermark_distance = low_wmark_pages(zone) - min_wmark_pages(zone); ++ threshold = max(1, (int)(watermark_distance / num_online_cpus())); ++ ++ /* ++ * Maximum threshold is 125 ++ */ ++ threshold = min(125, threshold); ++ ++ return threshold; ++} ++ + static int calculate_threshold(struct zone *zone) + { + int threshold; +@@ -159,6 +183,48 @@ static void refresh_zone_stat_thresholds(void) + } + } + ++void reduce_pgdat_percpu_threshold(pg_data_t *pgdat) ++{ ++ struct zone *zone; ++ int cpu; ++ int threshold; ++ int i; ++ ++ get_online_cpus(); ++ for (i = 0; i < pgdat->nr_zones; i++) { ++ zone = &pgdat->node_zones[i]; ++ if (!zone->percpu_drift_mark) ++ continue; ++ ++ threshold = calculate_pressure_threshold(zone); ++ for_each_online_cpu(cpu) ++ per_cpu_ptr(zone->pageset, cpu)->stat_threshold ++ = threshold; ++ } ++ put_online_cpus(); ++} ++ ++void restore_pgdat_percpu_threshold(pg_data_t *pgdat) ++{ ++ struct zone *zone; ++ int cpu; ++ int threshold; ++ int i; ++ ++ get_online_cpus(); ++ for (i = 0; i < pgdat->nr_zones; i++) { ++ zone = &pgdat->node_zones[i]; ++ if (!zone->percpu_drift_mark) ++ continue; ++ ++ threshold = calculate_threshold(zone); ++ for_each_online_cpu(cpu) ++ per_cpu_ptr(zone->pageset, cpu)->stat_threshold ++ = threshold; ++ } ++ put_online_cpus(); ++} ++ + /* + * For use when we know that interrupts are disabled. + */ +@@ -826,7 +892,7 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat, + "\n scanned %lu" + "\n spanned %lu" + "\n present %lu", +- zone_nr_free_pages(zone), ++ zone_page_state(zone, NR_FREE_PAGES), + min_wmark_pages(zone), + low_wmark_pages(zone), + high_wmark_pages(zone), +-- +1.7.3.2 + diff --git a/mm-vmstat-use-a-single-setter-function-and-callback-for-adjusting-percpu-thresholds.patch b/mm-vmstat-use-a-single-setter-function-and-callback-for-adjusting-percpu-thresholds.patch new file mode 100644 index 000000000..058b1399a --- /dev/null +++ b/mm-vmstat-use-a-single-setter-function-and-callback-for-adjusting-percpu-thresholds.patch @@ -0,0 +1,167 @@ +From 82e3d4969144377d13da97d511e849e8cf3e6dcc Mon Sep 17 00:00:00 2001 +From: Mel Gorman +Date: Wed, 24 Nov 2010 22:24:24 -0500 +Subject: [PATCH 2/2] mm: vmstat: Use a single setter function and callback for adjusting percpu thresholds + +reduce_pgdat_percpu_threshold() and restore_pgdat_percpu_threshold() exist +to adjust the per-cpu vmstat thresholds while kswapd is awake to avoid +errors due to counter drift. The functions duplicate some code so this +patch replaces them with a single set_pgdat_percpu_threshold() that takes +a callback function to calculate the desired threshold as a parameter. + +Signed-off-by: Mel Gorman +Reviewed-by: Christoph Lameter +Reviewed-by: KAMEZAWA Hiroyuki +Signed-off-by: Andrew Morton +[the various mmotm patches updating this were rolled up. --kyle] +[[http://userweb.kernel.org/~akpm/mmotm/broken-out/mm-vmstat-use-a-single-setter-function-and-callback-for-adjusting-percpu-thresholds-fix-set_pgdat_percpu_threshold-dont-use-for_each_online_cpu.patch]] +--- + include/linux/vmstat.h | 10 ++++++---- + mm/vmscan.c | 19 +++++++++++++++++-- + mm/vmstat.c | 36 +++++++----------------------------- + 3 files changed, 30 insertions(+), 35 deletions(-) + +diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h +index e4cc21c..833e676 100644 +--- a/include/linux/vmstat.h ++++ b/include/linux/vmstat.h +@@ -254,8 +254,11 @@ extern void dec_zone_state(struct zone *, enum zone_stat_item); + extern void __dec_zone_state(struct zone *, enum zone_stat_item); + + void refresh_cpu_vm_stats(int); +-void reduce_pgdat_percpu_threshold(pg_data_t *pgdat); +-void restore_pgdat_percpu_threshold(pg_data_t *pgdat); ++ ++int calculate_pressure_threshold(struct zone *zone); ++int calculate_normal_threshold(struct zone *zone); ++void set_pgdat_percpu_threshold(pg_data_t *pgdat, ++ int (*calculate_pressure)(struct zone *)); + #else /* CONFIG_SMP */ + + /* +@@ -300,8 +303,7 @@ static inline void __dec_zone_page_state(struct page *page, + #define dec_zone_page_state __dec_zone_page_state + #define mod_zone_page_state __mod_zone_page_state + +-static inline void reduce_pgdat_percpu_threshold(pg_data_t *pgdat) { } +-static inline void restore_pgdat_percpu_threshold(pg_data_t *pgdat) { } ++#define set_pgdat_percpu_threshold(pgdat, callback) { } + + static inline void refresh_cpu_vm_stats(int cpu) { } + #endif +diff --git a/mm/vmscan.c b/mm/vmscan.c +index 3e71cb1..ba39948 100644 +--- a/mm/vmscan.c ++++ b/mm/vmscan.c +@@ -2378,9 +2378,24 @@ static int kswapd(void *p) + */ + if (!sleeping_prematurely(pgdat, order, remaining)) { + trace_mm_vmscan_kswapd_sleep(pgdat->node_id); +- restore_pgdat_percpu_threshold(pgdat); ++ ++ /* ++ * vmstat counters are not perfectly ++ * accurate and the estimated value ++ * for counters such as NR_FREE_PAGES ++ * can deviate from the true value by ++ * nr_online_cpus * threshold. To ++ * avoid the zone watermarks being ++ * breached while under pressure, we ++ * reduce the per-cpu vmstat threshold ++ * while kswapd is awake and restore ++ * them before going back to sleep. ++ */ ++ set_pgdat_percpu_threshold(pgdat, ++ calculate_normal_threshold); + schedule(); +- reduce_pgdat_percpu_threshold(pgdat); ++ set_pgdat_percpu_threshold(pgdat, ++ calculate_pressure_threshold); + } else { + if (remaining) + count_vm_event(KSWAPD_LOW_WMARK_HIT_QUICKLY); +diff --git a/mm/vmstat.c b/mm/vmstat.c +index 4d7faeb..511c2c0 100644 +--- a/mm/vmstat.c ++++ b/mm/vmstat.c +@@ -81,7 +81,7 @@ EXPORT_SYMBOL(vm_stat); + + #ifdef CONFIG_SMP + +-static int calculate_pressure_threshold(struct zone *zone) ++int calculate_pressure_threshold(struct zone *zone) + { + int threshold; + int watermark_distance; +@@ -105,7 +105,7 @@ static int calculate_pressure_threshold(struct zone *zone) + return threshold; + } + +-static int calculate_threshold(struct zone *zone) ++int calculate_normal_threshold(struct zone *zone) + { + int threshold; + int mem; /* memory in 128 MB units */ +@@ -164,7 +164,7 @@ static void refresh_zone_stat_thresholds(void) + for_each_populated_zone(zone) { + unsigned long max_drift, tolerate_drift; + +- threshold = calculate_threshold(zone); ++ threshold = calculate_normal_threshold(zone); + + for_each_online_cpu(cpu) + per_cpu_ptr(zone->pageset, cpu)->stat_threshold +@@ -183,46 +183,24 @@ static void refresh_zone_stat_thresholds(void) + } + } + +-void reduce_pgdat_percpu_threshold(pg_data_t *pgdat) ++void set_pgdat_percpu_threshold(pg_data_t *pgdat, ++ int (*calculate_pressure)(struct zone *)) + { + struct zone *zone; + int cpu; + int threshold; + int i; + +- get_online_cpus(); +- for (i = 0; i < pgdat->nr_zones; i++) { +- zone = &pgdat->node_zones[i]; +- if (!zone->percpu_drift_mark) +- continue; +- +- threshold = calculate_pressure_threshold(zone); +- for_each_online_cpu(cpu) +- per_cpu_ptr(zone->pageset, cpu)->stat_threshold +- = threshold; +- } +- put_online_cpus(); +-} +- +-void restore_pgdat_percpu_threshold(pg_data_t *pgdat) +-{ +- struct zone *zone; +- int cpu; +- int threshold; +- int i; +- +- get_online_cpus(); + for (i = 0; i < pgdat->nr_zones; i++) { + zone = &pgdat->node_zones[i]; + if (!zone->percpu_drift_mark) + continue; + +- threshold = calculate_threshold(zone); +- for_each_online_cpu(cpu) ++ threshold = (*calculate_pressure)(zone); ++ for_each_possible_cpu(cpu) + per_cpu_ptr(zone->pageset, cpu)->stat_threshold + = threshold; + } +- put_online_cpus(); + } + + /* +-- +1.7.3.2 +