summaryrefslogtreecommitdiff
path: root/mm/gup.c
diff options
context:
space:
mode:
authorMatthew Wilcox (Oracle) <willy@infradead.org>2022-01-07 00:46:43 +0300
committerMatthew Wilcox (Oracle) <willy@infradead.org>2022-03-21 19:56:35 +0300
commit5232c63f46fdd779303527ec36c518cc1e9c6b4e (patch)
tree86a2005fadd461b81c8f9db76ea33e5bcf634ff6 /mm/gup.c
parent6315d8a23ce308433cf615e435ca2ee2aee7d11c (diff)
downloadlinux-5232c63f46fdd779303527ec36c518cc1e9c6b4e.tar.xz
mm: Make compound_pincount always available
Move compound_pincount from the third page to the second page, which means it's available for all compound pages. That lets us delete hpage_pincount_available(). On 32-bit systems, there isn't enough space for both compound_pincount and compound_nr in the second page (it would collide with page->private, which is in use for pages in the swap cache), so revert the optimisation of storing both compound_order and compound_nr on 32-bit systems. Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org> Reviewed-by: John Hubbard <jhubbard@nvidia.com> Reviewed-by: Christoph Hellwig <hch@lst.de> Reviewed-by: Jason Gunthorpe <jgg@nvidia.com> Reviewed-by: William Kucharski <william.kucharski@oracle.com>
Diffstat (limited to 'mm/gup.c')
-rw-r--r--mm/gup.c20
1 files changed, 9 insertions, 11 deletions
diff --git a/mm/gup.c b/mm/gup.c
index 1809dc037a8e..56b6b01a430b 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -99,12 +99,11 @@ retry:
*
* FOLL_GET: page's refcount will be incremented by @refs.
*
- * FOLL_PIN on compound pages that are > two pages long: page's refcount will
- * be incremented by @refs, and page[2].hpage_pinned_refcount will be
- * incremented by @refs * GUP_PIN_COUNTING_BIAS.
+ * FOLL_PIN on compound pages: page's refcount will be incremented by
+ * @refs, and page[1].compound_pincount will be incremented by @refs.
*
- * FOLL_PIN on normal pages, or compound pages that are two pages long:
- * page's refcount will be incremented by @refs * GUP_PIN_COUNTING_BIAS.
+ * FOLL_PIN on normal pages: page's refcount will be incremented by
+ * @refs * GUP_PIN_COUNTING_BIAS.
*
* Return: head page (with refcount appropriately incremented) for success, or
* NULL upon failure. If neither FOLL_GET nor FOLL_PIN was set, that's
@@ -135,16 +134,15 @@ __maybe_unused struct page *try_grab_compound_head(struct page *page,
return NULL;
/*
- * When pinning a compound page of order > 1 (which is
- * what hpage_pincount_available() checks for), use an
- * exact count to track it.
+ * When pinning a compound page, use an exact count to
+ * track it.
*
* However, be sure to *also* increment the normal page
* refcount field at least once, so that the page really
* is pinned. That's why the refcount from the earlier
* try_get_compound_head() is left intact.
*/
- if (hpage_pincount_available(page))
+ if (PageHead(page))
atomic_add(refs, compound_pincount_ptr(page));
else
page_ref_add(page, refs * (GUP_PIN_COUNTING_BIAS - 1));
@@ -166,7 +164,7 @@ static void put_compound_head(struct page *page, int refs, unsigned int flags)
if (flags & FOLL_PIN) {
mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_RELEASED,
refs);
- if (hpage_pincount_available(page))
+ if (PageHead(page))
atomic_sub(refs, compound_pincount_ptr(page));
else
refs *= GUP_PIN_COUNTING_BIAS;
@@ -211,7 +209,7 @@ bool __must_check try_grab_page(struct page *page, unsigned int flags)
* increment the normal page refcount field at least once,
* so that the page really is pinned.
*/
- if (hpage_pincount_available(page)) {
+ if (PageHead(page)) {
page_ref_add(page, 1);
atomic_add(1, compound_pincount_ptr(page));
} else {