Contributors: 20
| Author |
Tokens |
Token Proportion |
Commits |
Commit Proportion |
| Rafael Aquini |
362 |
36.06% |
2 |
3.77% |
| David Hildenbrand (Red Hat) |
159 |
15.84% |
12 |
22.64% |
| Nadav Amit |
144 |
14.34% |
3 |
5.66% |
| David Hildenbrand |
97 |
9.66% |
11 |
20.75% |
| Konstantin Khlebnikov |
66 |
6.57% |
3 |
5.66% |
| MinChan Kim |
49 |
4.88% |
2 |
3.77% |
| Michael S. Tsirkin |
31 |
3.09% |
2 |
3.77% |
| Huacai Chen |
23 |
2.29% |
1 |
1.89% |
| Nico Pache |
14 |
1.39% |
1 |
1.89% |
| Linus Torvalds (pre-git) |
12 |
1.20% |
4 |
7.55% |
| Andrew Morton |
12 |
1.20% |
1 |
1.89% |
| Christoph Lameter |
10 |
1.00% |
2 |
3.77% |
| Jiang Liu |
6 |
0.60% |
1 |
1.89% |
| Mel Gorman |
4 |
0.40% |
2 |
3.77% |
| Rusty Russell |
4 |
0.40% |
1 |
1.89% |
| Matthew Wilcox |
3 |
0.30% |
1 |
1.89% |
| Denis V. Lunev |
3 |
0.30% |
1 |
1.89% |
| Miaohe Lin |
3 |
0.30% |
1 |
1.89% |
| Ingo Molnar |
1 |
0.10% |
1 |
1.89% |
| Thomas Gleixner |
1 |
0.10% |
1 |
1.89% |
| Total |
1004 |
|
53 |
|
// SPDX-License-Identifier: GPL-2.0-only
/*
* Common interface for implementing a memory balloon, including support
* for migration of pages inflated in a memory balloon.
*
* Copyright (C) 2012, Red Hat, Inc. Rafael Aquini <aquini@redhat.com>
*/
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/export.h>
#include <linux/balloon.h>
/*
* Lock protecting the balloon_dev_info of all devices. We don't really
* expect more than one device.
*/
static DEFINE_SPINLOCK(balloon_pages_lock);
/**
* balloon_page_insert - insert a page into the balloon's page list and make
* the page->private assignment accordingly.
* @balloon : pointer to balloon device
* @page : page to be assigned as a 'balloon page'
*
* Caller must ensure the balloon_pages_lock is held.
*/
static void balloon_page_insert(struct balloon_dev_info *balloon,
struct page *page)
{
lockdep_assert_held(&balloon_pages_lock);
__SetPageOffline(page);
if (IS_ENABLED(CONFIG_BALLOON_MIGRATION)) {
SetPageMovableOps(page);
set_page_private(page, (unsigned long)balloon);
}
list_add(&page->lru, &balloon->pages);
}
/**
* balloon_page_finalize - prepare a balloon page that was removed from the
* balloon list for release to the page allocator
* @page: page to be released to the page allocator
*
* Caller must ensure the balloon_pages_lock is held.
*/
static void balloon_page_finalize(struct page *page)
{
lockdep_assert_held(&balloon_pages_lock);
if (IS_ENABLED(CONFIG_BALLOON_MIGRATION))
set_page_private(page, 0);
/* PageOffline is sticky until the page is freed to the buddy. */
}
static void balloon_page_enqueue_one(struct balloon_dev_info *b_dev_info,
struct page *page)
{
balloon_page_insert(b_dev_info, page);
if (b_dev_info->adjust_managed_page_count)
adjust_managed_page_count(page, -1);
__count_vm_event(BALLOON_INFLATE);
inc_node_page_state(page, NR_BALLOON_PAGES);
}
/**
* balloon_page_list_enqueue() - inserts a list of pages into the balloon page
* list.
* @b_dev_info: balloon device descriptor where we will insert a new page to
* @pages: pages to enqueue - allocated using balloon_page_alloc.
*
* Driver must call this function to properly enqueue balloon pages before
* definitively removing them from the guest system.
*
* Return: number of pages that were enqueued.
*/
size_t balloon_page_list_enqueue(struct balloon_dev_info *b_dev_info,
struct list_head *pages)
{
struct page *page, *tmp;
unsigned long flags;
size_t n_pages = 0;
spin_lock_irqsave(&balloon_pages_lock, flags);
list_for_each_entry_safe(page, tmp, pages, lru) {
list_del(&page->lru);
balloon_page_enqueue_one(b_dev_info, page);
n_pages++;
}
spin_unlock_irqrestore(&balloon_pages_lock, flags);
return n_pages;
}
EXPORT_SYMBOL_GPL(balloon_page_list_enqueue);
/**
* balloon_page_list_dequeue() - removes pages from balloon's page list and
* returns a list of the pages.
* @b_dev_info: balloon device descriptor where we will grab a page from.
* @pages: pointer to the list of pages that would be returned to the caller.
* @n_req_pages: number of requested pages.
*
* Driver must call this function to properly de-allocate a previous enlisted
* balloon pages before definitively releasing it back to the guest system.
* This function tries to remove @n_req_pages from the ballooned pages and
* return them to the caller in the @pages list.
*
* Note that this function may fail to dequeue some pages even if the balloon
* isn't empty - since the page list can be temporarily empty due to compaction
* of isolated pages.
*
* Return: number of pages that were added to the @pages list.
*/
size_t balloon_page_list_dequeue(struct balloon_dev_info *b_dev_info,
struct list_head *pages, size_t n_req_pages)
{
struct page *page, *tmp;
unsigned long flags;
size_t n_pages = 0;
spin_lock_irqsave(&balloon_pages_lock, flags);
list_for_each_entry_safe(page, tmp, &b_dev_info->pages, lru) {
if (n_pages == n_req_pages)
break;
list_del(&page->lru);
if (b_dev_info->adjust_managed_page_count)
adjust_managed_page_count(page, 1);
balloon_page_finalize(page);
__count_vm_event(BALLOON_DEFLATE);
list_add(&page->lru, pages);
dec_node_page_state(page, NR_BALLOON_PAGES);
n_pages++;
}
spin_unlock_irqrestore(&balloon_pages_lock, flags);
return n_pages;
}
EXPORT_SYMBOL_GPL(balloon_page_list_dequeue);
/**
* balloon_page_alloc - allocates a new page for insertion into the balloon
* page list.
*
* Driver must call this function to properly allocate a new balloon page.
* Driver must call balloon_page_enqueue before definitively removing the page
* from the guest system.
*
* Return: struct page for the allocated page or NULL on allocation failure.
*/
struct page *balloon_page_alloc(void)
{
gfp_t gfp_flags = __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
if (IS_ENABLED(CONFIG_BALLOON_MIGRATION))
gfp_flags |= GFP_HIGHUSER_MOVABLE;
else
gfp_flags |= GFP_HIGHUSER;
return alloc_page(gfp_flags);
}
EXPORT_SYMBOL_GPL(balloon_page_alloc);
/**
* balloon_page_enqueue - inserts a new page into the balloon page list.
*
* @b_dev_info: balloon device descriptor where we will insert a new page
* @page: new page to enqueue - allocated using balloon_page_alloc.
*
* Drivers must call this function to properly enqueue a new allocated balloon
* page before definitively removing the page from the guest system.
*
* Drivers must not enqueue pages while page->lru is still in
* use, and must not use page->lru until a page was unqueued again.
*/
void balloon_page_enqueue(struct balloon_dev_info *b_dev_info,
struct page *page)
{
unsigned long flags;
spin_lock_irqsave(&balloon_pages_lock, flags);
balloon_page_enqueue_one(b_dev_info, page);
spin_unlock_irqrestore(&balloon_pages_lock, flags);
}
EXPORT_SYMBOL_GPL(balloon_page_enqueue);
/**
* balloon_page_dequeue - removes a page from balloon's page list and returns
* its address to allow the driver to release the page.
* @b_dev_info: balloon device descriptor where we will grab a page from.
*
* Driver must call this function to properly dequeue a previously enqueued page
* before definitively releasing it back to the guest system.
*
* Caller must perform its own accounting to ensure that this
* function is called only if some pages are actually enqueued.
*
* Note that this function may fail to dequeue some pages even if there are
* some enqueued pages - since the page list can be temporarily empty due to
* the compaction of isolated pages.
*
* TODO: remove the caller accounting requirements, and allow caller to wait
* until all pages can be dequeued.
*
* Return: struct page for the dequeued page, or NULL if no page was dequeued.
*/
struct page *balloon_page_dequeue(struct balloon_dev_info *b_dev_info)
{
unsigned long flags;
LIST_HEAD(pages);
int n_pages;
n_pages = balloon_page_list_dequeue(b_dev_info, &pages, 1);
if (n_pages != 1) {
/*
* If we are unable to dequeue a balloon page because the page
* list is empty and there are no isolated pages, then something
* went out of track and some balloon pages are lost.
* BUG() here, otherwise the balloon driver may get stuck in
* an infinite loop while attempting to release all its pages.
*/
spin_lock_irqsave(&balloon_pages_lock, flags);
if (unlikely(list_empty(&b_dev_info->pages) &&
!b_dev_info->isolated_pages))
BUG();
spin_unlock_irqrestore(&balloon_pages_lock, flags);
return NULL;
}
return list_first_entry(&pages, struct page, lru);
}
EXPORT_SYMBOL_GPL(balloon_page_dequeue);
#ifdef CONFIG_BALLOON_MIGRATION
static struct balloon_dev_info *balloon_page_device(struct page *page)
{
return (struct balloon_dev_info *)page_private(page);
}
static bool balloon_page_isolate(struct page *page, isolate_mode_t mode)
{
struct balloon_dev_info *b_dev_info;
unsigned long flags;
spin_lock_irqsave(&balloon_pages_lock, flags);
b_dev_info = balloon_page_device(page);
if (!b_dev_info) {
/*
* The page already got deflated and removed from the
* balloon list.
*/
spin_unlock_irqrestore(&balloon_pages_lock, flags);
return false;
}
list_del(&page->lru);
b_dev_info->isolated_pages++;
spin_unlock_irqrestore(&balloon_pages_lock, flags);
return true;
}
static void balloon_page_putback(struct page *page)
{
struct balloon_dev_info *b_dev_info = balloon_page_device(page);
unsigned long flags;
/*
* When we isolated the page, the page was still inflated in a balloon
* device. As isolated balloon pages cannot get deflated, we still have
* a balloon device here.
*/
if (WARN_ON_ONCE(!b_dev_info))
return;
spin_lock_irqsave(&balloon_pages_lock, flags);
list_add(&page->lru, &b_dev_info->pages);
b_dev_info->isolated_pages--;
spin_unlock_irqrestore(&balloon_pages_lock, flags);
}
static int balloon_page_migrate(struct page *newpage, struct page *page,
enum migrate_mode mode)
{
struct balloon_dev_info *b_dev_info = balloon_page_device(page);
unsigned long flags;
int rc;
/*
* When we isolated the page, the page was still inflated in a balloon
* device. As isolated balloon pages cannot get deflated, we still have
* a balloon device here.
*/
if (WARN_ON_ONCE(!b_dev_info))
return -EAGAIN;
rc = b_dev_info->migratepage(b_dev_info, newpage, page, mode);
if (rc < 0 && rc != -ENOENT)
return rc;
spin_lock_irqsave(&balloon_pages_lock, flags);
if (!rc) {
/* Insert the new page into the balloon list. */
get_page(newpage);
balloon_page_insert(b_dev_info, newpage);
__count_vm_event(BALLOON_MIGRATE);
if (b_dev_info->adjust_managed_page_count &&
page_zone(page) != page_zone(newpage)) {
/*
* When we migrate a page to a different zone we
* have to fixup the count of both involved zones.
*/
adjust_managed_page_count(page, 1);
adjust_managed_page_count(newpage, -1);
}
} else {
/* Old page was deflated but new page not inflated. */
__count_vm_event(BALLOON_DEFLATE);
if (b_dev_info->adjust_managed_page_count)
adjust_managed_page_count(page, 1);
}
b_dev_info->isolated_pages--;
/* Free the now-deflated page we isolated in balloon_page_isolate(). */
balloon_page_finalize(page);
spin_unlock_irqrestore(&balloon_pages_lock, flags);
put_page(page);
return 0;
}
static const struct movable_operations balloon_mops = {
.migrate_page = balloon_page_migrate,
.isolate_page = balloon_page_isolate,
.putback_page = balloon_page_putback,
};
static int __init balloon_init(void)
{
return set_movable_ops(&balloon_mops, PGTY_offline);
}
core_initcall(balloon_init);
#endif /* CONFIG_BALLOON_MIGRATION */