From: VMware, Inc <> Date: Mon, 15 Oct 2012 04:49:10 +0000 (-0700) Subject: vmmemctl: add batching to the balloon driver. X-Git-Tag: 2012.10.14-874563~40 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=d5df2bdfc81ef8b0c4ba6c0e47042b6316069e32;p=thirdparty%2Fopen-vm-tools.git vmmemctl: add batching to the balloon driver. Currently, we call the monitor for each reclaimed page. Batch those pages and do a monitor call when enough pages have been reclaimed. Ultimately, this should improve the performance by reducing the number of context switch. Signed-off-by: Dmitry Torokhov --- diff --git a/open-vm-tools/modules/freebsd/vmmemctl/os.c b/open-vm-tools/modules/freebsd/vmmemctl/os.c index f2754c8e0..29f25a5c8 100644 --- a/open-vm-tools/modules/freebsd/vmmemctl/os.c +++ b/open-vm-tools/modules/freebsd/vmmemctl/os.c @@ -244,13 +244,118 @@ OS_ReservedPageGetLimit(void) *----------------------------------------------------------------------------- */ -unsigned long +PPN64 OS_ReservedPageGetPPN(PageHandle handle) // IN: A valid page handle { return (((vm_page_t)handle)->phys_addr) >> PAGE_SHIFT; } +/* + *----------------------------------------------------------------------------- + * + * OS_ReservedPageGetHandle -- + * + * Convert a ppn (of a physical page previously reserved with + * OS_ReservedPageAlloc()) to a page handle. + * + * Results: + * The page handle. + * + * Side effects: + * None. + * + *----------------------------------------------------------------------------- + */ + +PageHandle +OS_ReservedPageGetHandle(PPN64 ppn) // IN +{ + return (PageHandle)PHYS_TO_VM_PAGE(ppn << PAGE_SHIFT); +} + + +/* + *----------------------------------------------------------------------------- + * + * OS_MapPageHandle -- + * + * Map a page handle into kernel address space, and return the + * mapping to that page handle. + * + * Results: + * The mapping. + * + * Side effects: + * None. + * + *----------------------------------------------------------------------------- + */ + +Mapping +OS_MapPageHandle(PageHandle handle) // IN +{ + vm_offset_t res = kmem_alloc_nofault(kernel_map, PAGE_SIZE); + vm_page_t page = (vm_page_t)handle; + + if (!res) { + return MAPPING_INVALID; + } + + pmap_qenter(res, &page, 1); + + return (Mapping)res; +} + + +/* + *----------------------------------------------------------------------------- + * + * OS_Mapping2Addr -- + * + * Return the address of a previously mapped page handle (with + * OS_MapPageHandle). + * + * Results: + * The mapping address. + * + * Side effects: + * None. + * + *----------------------------------------------------------------------------- + */ + +void * +OS_Mapping2Addr(Mapping mapping) // IN +{ + return (void *)mapping; +} + + +/* + *----------------------------------------------------------------------------- + * + * OS_UnmapPage -- + * + * Unmap a previously mapped page handle. + * + * Results: + * None. + * + * Side effects: + * None. + * + *----------------------------------------------------------------------------- + */ + +void +OS_UnmapPage(Mapping mapping) // IN +{ + pmap_qremove((vm_offset_t)mapping, 1); + kmem_free(kernel_map, (vm_offset_t)mapping, PAGE_SIZE); +} + + static void os_pmap_alloc(os_pmap *p) // IN { diff --git a/open-vm-tools/modules/shared/vmmemctl/backdoor_balloon.c b/open-vm-tools/modules/shared/vmmemctl/backdoor_balloon.c index d511c7a2e..674391b64 100644 --- a/open-vm-tools/modules/shared/vmmemctl/backdoor_balloon.c +++ b/open-vm-tools/modules/shared/vmmemctl/backdoor_balloon.c @@ -104,14 +104,15 @@ void BackdoorBalloon(Backdoor_proto *myBp) // IN/OUT */ int -Backdoor_MonitorStart(Balloon *b) // IN +Backdoor_MonitorStart(Balloon *b, // IN + uint32 protoVersion) // IN { uint32 status; Backdoor_proto bp; /* prepare backdoor args */ bp.in.cx.halfs.low = BALLOON_BDOOR_CMD_START; - bp.in.size = BALLOON_PROTOCOL_VERSION; + bp.in.size = protoVersion; /* invoke backdoor */ BackdoorBalloon(&bp); @@ -266,6 +267,8 @@ Backdoor_MonitorGetTarget(Balloon *b, // IN * * Attempts to contact monitor and add PPN corresponding to * the page handle to set of "balloon locked" pages. + * If the current protocol support batching, it will balloon all + * PPNs listed in the batch page. * * Results: * Returns BALLOON_SUCCESS if successful, otherwise error code. @@ -278,28 +281,28 @@ Backdoor_MonitorGetTarget(Balloon *b, // IN int Backdoor_MonitorLockPage(Balloon *b, // IN - PPN ppn) // IN + PPN64 ppn) // IN { - uint32 ppn32; - uint32 status, target; + uint32 status; Backdoor_proto bp; - /* Ensure PPN fits in 32-bits, i.e. guest memory is limited to 16TB. */ - ppn32 = (uint32)ppn; - if (ppn32 != ppn) { - return BALLOON_ERROR_PPN_INVALID; + if (b->hypervisorProtocolVersion == BALLOON_PROTOCOL_VERSION_2) { + /* Ensure PPN fits in 32-bits, i.e. guest memory is limited to 16TB. */ + uint32 ppn32 = (uint32)ppn; + if (ppn32 != ppn) { + return BALLOON_ERROR_PPN_INVALID; + } } /* prepare backdoor args */ bp.in.cx.halfs.low = BALLOON_BDOOR_CMD_LOCK; - bp.in.size = ppn32; + bp.in.size = (size_t)ppn; /* invoke backdoor */ BackdoorBalloon(&bp); /* parse return values */ status = bp.out.ax.word; - target = bp.out.bx.word; /* set flag if reset requested */ if (status == BALLOON_ERROR_RESET) { @@ -315,7 +318,6 @@ Backdoor_MonitorLockPage(Balloon *b, // IN return status; } - /* *---------------------------------------------------------------------- * @@ -323,6 +325,8 @@ Backdoor_MonitorLockPage(Balloon *b, // IN * * Attempts to contact monitor and remove PPN corresponding to * the page handle from set of "balloon locked" pages. + * If the current protocol support batching, it will remove all + * the PPNs listed in the batch page. * * Results: * Returns BALLOON_SUCCESS if successful, otherwise error code. @@ -335,28 +339,28 @@ Backdoor_MonitorLockPage(Balloon *b, // IN int Backdoor_MonitorUnlockPage(Balloon *b, // IN - PPN ppn) // IN + PPN64 ppn) // IN { - uint32 ppn32; - uint32 status, target; + uint32 status; Backdoor_proto bp; - /* Ensure PPN fits in 32-bits, i.e. guest memory is limited to 16TB. */ - ppn32 = (uint32)ppn; - if (ppn32 != ppn) { - return BALLOON_ERROR_PPN_INVALID; + if (b->hypervisorProtocolVersion == BALLOON_PROTOCOL_VERSION_2) { + /* Ensure PPN fits in 32-bits, i.e. guest memory is limited to 16TB. */ + uint32 ppn32 = (uint32)ppn; + if (ppn32 != ppn) { + return BALLOON_ERROR_PPN_INVALID; + } } /* prepare backdoor args */ bp.in.cx.halfs.low = BALLOON_BDOOR_CMD_UNLOCK; - bp.in.size = ppn32; + bp.in.size = (size_t)ppn; /* invoke backdoor */ BackdoorBalloon(&bp); /* parse return values */ status = bp.out.ax.word; - target = bp.out.bx.word; /* set flag if reset requested */ if (status == BALLOON_ERROR_RESET) { diff --git a/open-vm-tools/modules/shared/vmmemctl/backdoor_balloon.h b/open-vm-tools/modules/shared/vmmemctl/backdoor_balloon.h index 13c800bc5..724ffd832 100644 --- a/open-vm-tools/modules/shared/vmmemctl/backdoor_balloon.h +++ b/open-vm-tools/modules/shared/vmmemctl/backdoor_balloon.h @@ -72,10 +72,10 @@ #include "backdoor.h" #include "balloon_def.h" -int Backdoor_MonitorStart(Balloon *b); +int Backdoor_MonitorStart(Balloon *b, uint32 protoVersion); int Backdoor_MonitorGuestType(Balloon *b); int Backdoor_MonitorGetTarget(Balloon *b, uint32 *target); -int Backdoor_MonitorLockPage(Balloon *b, PPN ppn); -int Backdoor_MonitorUnlockPage(Balloon *b, PPN ppn); +int Backdoor_MonitorLockPage(Balloon *b, PPN64 ppn); +int Backdoor_MonitorUnlockPage(Balloon *b, PPN64 ppn); #endif /* _BACKDOOR_BALLOON_H_ */ diff --git a/open-vm-tools/modules/shared/vmmemctl/balloonInt.h b/open-vm-tools/modules/shared/vmmemctl/balloonInt.h index 5f79f95a5..9ffc54b09 100644 --- a/open-vm-tools/modules/shared/vmmemctl/balloonInt.h +++ b/open-vm-tools/modules/shared/vmmemctl/balloonInt.h @@ -67,7 +67,11 @@ #define BALLOON_NAME "vmmemctl" #define BALLOON_NAME_VERBOSE "VMware memory control driver" +#if defined __linux__ || defined __FreeBSD__ || defined _WIN32 #define BALLOON_PROTOCOL_VERSION BALLOON_PROTOCOL_VERSION_3 +#else +#define BALLOON_PROTOCOL_VERSION BALLOON_PROTOCOL_VERSION_2 +#endif #define BALLOON_RATE_ADAPT 1 diff --git a/open-vm-tools/modules/shared/vmmemctl/balloon_def.h b/open-vm-tools/modules/shared/vmmemctl/balloon_def.h index 34de14a3d..da607bee4 100644 --- a/open-vm-tools/modules/shared/vmmemctl/balloon_def.h +++ b/open-vm-tools/modules/shared/vmmemctl/balloon_def.h @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2000 VMware, Inc. All rights reserved. + * Copyright (C) 2000-2012 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published @@ -75,6 +75,8 @@ #include "includeCheck.h" #include "vm_basic_types.h" +#include "vm_basic_defs.h" +#include "vm_assert.h" /* * constants @@ -88,6 +90,30 @@ #define BALLOON_BDOOR_PORT (0x5670) #define BALLOON_BDOOR_MAGIC (0x456c6d6f) +/* + * Backdoor commands availability: + * + * +====================+===============+ + * | CMD | Protocol | + * +--------------------+---------------+ + * | START | 2 (*) | + * | TARGET | 2 | + * | LOCK | 2 (*) | + * | UNLOCK | 2 (*) | + * | GUEST_ID | 2 | + * +====================+===============+ + * + * (*) The START, LOCK and UNLOCK has been slightly modified when + * protocol v3 is used: + * - START now returns BALLOON_SUCCESS_WITH_VERSION, with the protocol + * version stored in %ecx, only if guest also support protocol v3 or + * up. + * - LOCK and UNLOCK are now taking a PPN of a page that contains a + * BalloonBatchPage, instead of directly taking the PPN to + * lock/unlock. + * + */ + /* backdoor command numbers */ #define BALLOON_BDOOR_CMD_START (0) #define BALLOON_BDOOR_CMD_TARGET (1) @@ -131,4 +157,196 @@ typedef enum { */ #define BALLOON_SUCCESS_WITH_VERSION (0x03000000) +/* + * BatchPage. + */ +#define BALLOON_BATCH_MAX_PAGES \ + ((PAGE_SIZE - sizeof(uint64)) / sizeof(PPN64)) + +#define BALLOON_BATCH_STATUS_SHIFT 56 + +typedef struct BalloonBatchPage { + uint64 nPages; + PPN64 pages[BALLOON_BATCH_MAX_PAGES]; +} BalloonBatchPage; + + +/* + *----------------------------------------------------------------------------- + * + * Balloon_BatchInit -- + * + * Results: + * Return the number of pages that can be embedded inside the batch + * page. If the protocol does not support batching, return 0. + * + * Side effects: + * None. + * + *----------------------------------------------------------------------------- + */ +static INLINE uint16 +Balloon_BatchInit(uint32 protoVersion) // IN +{ + switch (protoVersion) { + case BALLOON_PROTOCOL_VERSION_3: + return BALLOON_BATCH_MAX_PAGES; + case BALLOON_PROTOCOL_VERSION_2: + default: + return 0; + } +} + + +/* + *----------------------------------------------------------------------------- + * + * Balloon_BatchGetNumPages -- + * + * Return the number of pages contained in the batch page. + * + * Results: + * See above. + * + * Side effects: + * None. + * + *----------------------------------------------------------------------------- + */ +static INLINE uint64 +Balloon_BatchGetNumPages(BalloonBatchPage *batchPage) // IN +{ + return batchPage->nPages; +} + + +/* + *----------------------------------------------------------------------------- + * + * Balloon_BatchSetNumPages -- + * + * Set the number of pages contained in the batch page. + * + * Results: + * None. + * + * Side effects: + * None. + * + *----------------------------------------------------------------------------- + */ +static INLINE void +Balloon_BatchSetNumPages(BalloonBatchPage *batchPage, // IN + uint64 nPages) // IN +{ + ASSERT(nPages <= BALLOON_BATCH_MAX_PAGES); + batchPage->nPages = nPages; +} + + +/* + *----------------------------------------------------------------------------- + * + * Balloon_BatchGetPPN -- + * + * Get the page stored in the batch page at idx. + * + * Results: + * See above. + * + * Side effects: + * None. + * + *----------------------------------------------------------------------------- + */ +static INLINE PPN64 +Balloon_BatchGetPPN(BalloonBatchPage *batchPage, // IN + uint16 idx) // IN +{ + ASSERT(idx < batchPage->nPages); + return batchPage->pages[idx] & MASK64(BALLOON_BATCH_STATUS_SHIFT); +} + + +/* + *----------------------------------------------------------------------------- + * + * Balloon_BatchGetStatus -- + * + * Get the error code associated with a page. + * + * Results: + * See above. + * + * Side effects: + * None. + * + *----------------------------------------------------------------------------- + */ +static INLINE uint8 +Balloon_BatchGetStatus(BalloonBatchPage *batchPage, // IN + uint16 idx) // IN +{ + ASSERT(idx < batchPage->nPages); + return (uint8)(batchPage->pages[idx] >> BALLOON_BATCH_STATUS_SHIFT); +} + + +/* + *----------------------------------------------------------------------------- + * + * Balloon_BatchSetPPN -- + * + * Store the page in the batch page at idx. + * + * Results: + * None. + * + * Side effects: + * None. + * + *----------------------------------------------------------------------------- + */ +static INLINE void +Balloon_BatchSetPPN(BalloonBatchPage *batchPage, // IN + uint16 idx, // IN + PPN64 ppn) // IN +{ + ASSERT(idx < BALLOON_BATCH_MAX_PAGES); + batchPage->pages[idx] = ppn; +} + + +/* + *----------------------------------------------------------------------------- + * + * Balloon_BatchSetStatus -- + * + * Set the error code associated with a page. + * + * Results: + * None. + * + * Side effects: + * None. + * + *----------------------------------------------------------------------------- + */ +static INLINE void +Balloon_BatchSetStatus(BalloonBatchPage *batchPage, // IN + uint16 idx, // IN + int error) // IN +{ + PPN64 ppn; + + ASSERT(idx < BALLOON_BATCH_MAX_PAGES); + ppn = Balloon_BatchGetPPN(batchPage, idx); + ppn |= ((PPN64)error << BALLOON_BATCH_STATUS_SHIFT); + batchPage->pages[idx] = ppn; +} + +MY_ASSERTS(BALLOON_BATCH_SIZE, + ASSERT_ON_COMPILE(sizeof(BalloonBatchPage) == PAGE_SIZE); +) + #endif /* _BALLOON_DEF_H */ diff --git a/open-vm-tools/modules/shared/vmmemctl/os.h b/open-vm-tools/modules/shared/vmmemctl/os.h index 8171e0b93..2f3fe29db 100644 --- a/open-vm-tools/modules/shared/vmmemctl/os.h +++ b/open-vm-tools/modules/shared/vmmemctl/os.h @@ -75,8 +75,10 @@ */ typedef uintptr_t PageHandle; +typedef uintptr_t Mapping; -#define PAGE_HANDLE_INVALID 0 +#define PAGE_HANDLE_INVALID 0 +#define MAPPING_INVALID 0 /* * Operations @@ -91,8 +93,13 @@ extern void OS_Free(void *ptr, size_t size); extern void OS_Yield(void); extern unsigned long OS_ReservedPageGetLimit(void); -extern unsigned long OS_ReservedPageGetPPN(PageHandle handle); +extern PPN64 OS_ReservedPageGetPPN(PageHandle handle); +extern PageHandle OS_ReservedPageGetHandle(PPN64 ppn); extern PageHandle OS_ReservedPageAlloc(int canSleep); extern void OS_ReservedPageFree(PageHandle handle); +extern Mapping OS_MapPageHandle(PageHandle handle); +extern void *OS_Mapping2Addr(Mapping mapping); +extern void OS_UnmapPage(Mapping mapping); + #endif /* OS_H */ diff --git a/open-vm-tools/modules/shared/vmmemctl/vmballoon.c b/open-vm-tools/modules/shared/vmmemctl/vmballoon.c index ed689a3df..2674bbb29 100644 --- a/open-vm-tools/modules/shared/vmmemctl/vmballoon.c +++ b/open-vm-tools/modules/shared/vmmemctl/vmballoon.c @@ -93,8 +93,6 @@ extern "C" { #define NULL 0 #endif -#define BALLOON_CHUNK_PAGES 1000 - /* * When guest is under memory pressure, use a reduced page allocation * rate for next several cycles. @@ -105,14 +103,18 @@ extern "C" { #define BALLOON_ALLOC_YIELD_THRESHOLD 1024 /* - * Types + * Balloon operations */ +static int BalloonPageFree(Balloon *b); +static int BalloonAdjustSize(Balloon *b, uint32 target); +static void BalloonReset(Balloon *b); -typedef struct BalloonChunk { - PageHandle page[BALLOON_CHUNK_PAGES]; - uint32 pageCount; - DblLnkLst_Links node; -} BalloonChunk; +static void BalloonAddPagev2(Balloon *b, uint16 idx, PageHandle page); +static void BalloonAddPagev3(Balloon *b, uint16 idx, PageHandle page); +static int BalloonLockv2(Balloon *b, uint16 nPages); +static int BalloonLockv3(Balloon *b, uint16 nPages); +static int BalloonUnlockv2(Balloon *b, uint16 nPages); +static int BalloonUnlockv3(Balloon *b, uint16 nPages); /* * Globals @@ -120,14 +122,17 @@ typedef struct BalloonChunk { static Balloon globalBalloon; -/* - * Balloon operations - */ -static int BalloonPageAlloc(Balloon *b, BalloonPageAllocType allocType); -static int BalloonPageFree(Balloon *b, int monitorUnlock); -static int BalloonAdjustSize(Balloon *b, uint32 target); -static void BalloonReset(Balloon *b); +static const BalloonOps balloonOpsv2 = { + .addPage = BalloonAddPagev2, + .lock = BalloonLockv2, + .unlock = BalloonUnlockv2 +}; +static const struct BalloonOps balloonOpsv3 = { + .addPage = BalloonAddPagev3, + .lock = BalloonLockv3, + .unlock = BalloonUnlockv3 +}; /* *---------------------------------------------------------------------- @@ -255,12 +260,64 @@ Balloon_Deallocate(Balloon *b) // IN /* free all pages, skipping monitor unlock */ while (b->nChunks > 0) { - (void) BalloonPageFree(b, FALSE); + (void) BalloonPageFree(b); if (++cnt >= b->rateFree) { cnt = 0; OS_Yield(); } } + + /* Release the batch page */ + if (b->hypervisorProtocolVersion > BALLOON_PROTOCOL_VERSION_2) { + if (b->batchPageMapping != MAPPING_INVALID) { + OS_UnmapPage(b->batchPageMapping); + b->batchPageMapping = MAPPING_INVALID; + b->batchPage = NULL; + } + + if (b->pageHandle != PAGE_HANDLE_INVALID) { + OS_ReservedPageFree(b->pageHandle); + b->pageHandle = PAGE_HANDLE_INVALID; + } + } +} + +/* + *---------------------------------------------------------------------- + * + * BalloonInitBatching -- + * + * Allocate and map the batch page. + * + * Results: + * BALLOON_SUCCESS or an error code in case of failure. + * + * Side effects: + * None. + * + *---------------------------------------------------------------------- + */ + +static int +BalloonInitBatching(Balloon *b) // IN +{ + b->batchMaxPages = Balloon_BatchInit(b->hypervisorProtocolVersion); + ASSERT(b->batchMaxPages != 0); + + b->pageHandle = OS_ReservedPageAlloc(BALLOON_PAGE_ALLOC_NOSLEEP); + if (b->pageHandle == PAGE_HANDLE_INVALID) { + return BALLOON_PAGE_ALLOC_FAILURE; + } + + b->batchPageMapping = OS_MapPageHandle(b->pageHandle); + if (b->batchPageMapping == MAPPING_INVALID) { + OS_ReservedPageFree(b->pageHandle); + b->pageHandle = PAGE_HANDLE_INVALID; + return BALLOON_PAGE_ALLOC_FAILURE; + } + b->batchPage = OS_Mapping2Addr(b->batchPageMapping); + + return BALLOON_SUCCESS; } @@ -284,20 +341,42 @@ Balloon_Deallocate(Balloon *b) // IN static void BalloonReset(Balloon *b) // IN { - uint32 status; + int status; /* free all pages, skipping monitor unlock */ Balloon_Deallocate(b); - /* send start command */ - status = Backdoor_MonitorStart(b); - if (status == BALLOON_SUCCESS) { - /* clear flag */ - b->resetFlag = 0; + status = Backdoor_MonitorStart(b, BALLOON_PROTOCOL_VERSION); + if (status != BALLOON_SUCCESS) { + return; + } - /* report guest type */ - (void) Backdoor_MonitorGuestType(b); + if (b->hypervisorProtocolVersion > BALLOON_PROTOCOL_VERSION_2) { + status = BalloonInitBatching(b); + if (status != BALLOON_SUCCESS) { + status = Backdoor_MonitorStart(b, BALLOON_PROTOCOL_VERSION_2); + if (status != BALLOON_SUCCESS) { + return; + } + } + } + + switch (b->hypervisorProtocolVersion) { + case BALLOON_PROTOCOL_VERSION_3: + b->balloonOps = &balloonOpsv3; + break; + case BALLOON_PROTOCOL_VERSION_2: + default: + b->balloonOps = &balloonOpsv2; + b->batchMaxPages = 1; + break; } + + /* clear flag */ + b->resetFlag = FALSE; + + /* report guest type */ + (void) Backdoor_MonitorGuestType(b); } @@ -461,12 +540,14 @@ BalloonGetChunk(Balloon *b) // IN /* *---------------------------------------------------------------------- * - * BalloonPageStore -- + * BalloonGetChunkOrFallback -- * - * Add "page" to the given "chunk". + * Attempt to find a "chunk" with a free slot to store locked page. + * If the allocation fails, use the previously allocated + * fallbackChunk. * * Results: - * None. + * A valid chunk. * * Side effects: * None. @@ -474,23 +555,30 @@ BalloonGetChunk(Balloon *b) // IN *---------------------------------------------------------------------- */ -static void -BalloonPageStore(BalloonChunk *chunk, PageHandle page) +static BalloonChunk * +BalloonGetChunkOrFallback(Balloon *b) // IN { - chunk->page[chunk->pageCount++] = page; -} + BalloonChunk *chunk = BalloonGetChunk(b); + if (chunk == NULL) { + ASSERT(b->fallbackChunk != NULL); + chunk = b->fallbackChunk; + b->fallbackChunk = NULL; + DblLnkLst_LinkFirst(&b->chunks, &chunk->node); + b->nChunks++; + } + return chunk; +} /* *---------------------------------------------------------------------- * - * BalloonPageAlloc -- + * BalloonPageStore -- * - * Attempts to allocate a physical page, inflating balloon "b". - * Informs monitor of PPN for allocated page via backdoor. + * Add "page" to the given "chunk". * * Results: - * Returns BALLOON_SUCCESS if successful, otherwise error code. + * None. * * Side effects: * None. @@ -498,72 +586,10 @@ BalloonPageStore(BalloonChunk *chunk, PageHandle page) *---------------------------------------------------------------------- */ -static int -BalloonPageAlloc(Balloon *b, // IN - BalloonPageAllocType allocType) // IN +static void +BalloonPageStore(BalloonChunk *chunk, PageHandle page) { - PageHandle page; - PPN pagePPN; - BalloonChunk *chunk = NULL; - Bool locked; - int status; - - /* allocate page, fail if unable */ - do { - STATS_INC(b->stats.primAlloc[allocType]); - - /* - * Attempts to allocate and reserve a physical page. - * - * If canSleep == 1, i.e., BALLOON_PAGE_ALLOC_CANSLEEP: - * The allocation can wait (sleep) for page writeout (swap) by the guest. - * otherwise canSleep == 0, i.e., BALLOON_PAGE_ALLOC_NOSLEEP: - * If allocation of a page requires disk writeout, then just fail. DON'T sleep. - * - * Returns the physical address of the allocated page, or 0 if error. - */ - page = OS_ReservedPageAlloc(allocType); - if (page == PAGE_HANDLE_INVALID) { - STATS_INC(b->stats.primAllocFail[allocType]); - return BALLOON_PAGE_ALLOC_FAILURE; - } - - /* Get the chunk to store allocated page. */ - if (!chunk) { - chunk = BalloonGetChunk(b); - if (!chunk) { - OS_ReservedPageFree(page); - return BALLOON_PAGE_ALLOC_FAILURE; - } - } - - /* inform monitor via backdoor */ - pagePPN = OS_ReservedPageGetPPN(page); - status = Backdoor_MonitorLockPage(b, pagePPN); - locked = status == BALLOON_SUCCESS; - if (!locked) { - if (status == BALLOON_ERROR_RESET || - status == BALLOON_ERROR_PPN_NOTNEEDED) { - OS_ReservedPageFree(page); - return status; - } - - /* place on list of non-balloonable pages, retry allocation */ - status = BalloonErrorPageStore(b, page); - if (status != BALLOON_SUCCESS) { - OS_ReservedPageFree(page); - return status; - } - } - } while (!locked); - - /* track allocated page */ - BalloonPageStore(chunk, page); - - /* update balloon size */ - b->nPages++; - - return status; + chunk->page[chunk->pageCount++] = page; } @@ -573,8 +599,7 @@ BalloonPageAlloc(Balloon *b, // IN * BalloonPageFree -- * * Attempts to deallocate a physical page, deflating balloon "b". - * Informs monitor of PPN for deallocated page via backdoor if - * "monitorUnlock" is specified. + * Never informs monitor. * * Results: * Returns BALLOON_SUCCESS if successful, otherwise error code. @@ -586,48 +611,21 @@ BalloonPageAlloc(Balloon *b, // IN */ static int -BalloonPageFree(Balloon *b, // IN - int monitorUnlock) // IN +BalloonPageFree(Balloon *b) // IN { - DblLnkLst_Links *node, *next; - BalloonChunk *chunk = NULL; + BalloonChunk *chunk; PageHandle page; - int status; - DblLnkLst_ForEachSafe(node, next, &b->chunks) { - chunk = DblLnkLst_Container(node, BalloonChunk, node); - if (chunk->pageCount > 0) { - break; - } - - /* destroy empty chunk */ - DblLnkLst_Unlink1(node); - BalloonChunk_Destroy(chunk); - chunk = NULL; - - /* update stats */ - b->nChunks--; - } - - if (!chunk) { - /* We could not find a single non-empty chunk. */ + if (!DblLnkLst_IsLinked(&b->chunks)) { + /* The chunk list is empty, the balloon cannot be deflated */ return BALLOON_FAILURE; } + chunk = DblLnkLst_Container(b->chunks.next, BalloonChunk, node); + /* deallocate last page */ page = chunk->page[--chunk->pageCount]; - /* inform monitor via backdoor */ - if (monitorUnlock) { - PPN pagePPN = OS_ReservedPageGetPPN(page); - status = Backdoor_MonitorUnlockPage(b, pagePPN); - if (status != BALLOON_SUCCESS) { - /* reset next pointer, fail */ - chunk->pageCount++; - return status; - } - } - /* deallocate page */ OS_ReservedPageFree(page); STATS_INC(b->stats.primFree); @@ -648,11 +646,10 @@ BalloonPageFree(Balloon *b, // IN return BALLOON_SUCCESS; } - /* *---------------------------------------------------------------------- * - * BalloonInflate-- + * BalloonInflate -- * * Attempts to allocate physical pages to inflate balloon. * @@ -666,10 +663,10 @@ BalloonPageFree(Balloon *b, // IN */ static int -BalloonInflate(Balloon *b, // IN - uint32 target) // IN +BalloonInflate(Balloon *b, // IN + uint32 target) // IN { - unsigned int goal; + uint32 goal, nPages; unsigned int rate; unsigned int i; unsigned int allocations = 0; @@ -699,17 +696,14 @@ BalloonInflate(Balloon *b, // IN rate = b->slowPageAllocationCycles ? b->rateAlloc : BALLOON_NOSLEEP_ALLOC_MAX; + nPages = 0; for (i = 0; i < goal; i++) { + PageHandle handle; - status = BalloonPageAlloc(b, allocType); - if (status != BALLOON_SUCCESS) { - if (status != BALLOON_PAGE_ALLOC_FAILURE) { - /* - * Not a page allocation failure, stop this cycle. - * Maybe we'll get new target from the host soon. - */ - break; - } + STATS_INC(b->stats.primAlloc[allocType]); + handle = OS_ReservedPageAlloc(allocType); + if (handle == PAGE_HANDLE_INVALID) { + STATS_INC(b->stats.primAllocFail[allocType]); if (allocType == BALLOON_PAGE_ALLOC_CANSLEEP) { /* @@ -729,25 +723,41 @@ BalloonInflate(Balloon *b, // IN */ b->slowPageAllocationCycles = SLOW_PAGE_ALLOCATION_CYCLES; - if (i >= b->rateAlloc) + if (allocations >= b->rateAlloc) { break; + } allocType = BALLOON_PAGE_ALLOC_CANSLEEP; /* Lower rate for sleeping allocations. */ rate = b->rateAlloc; + continue; } - if (++allocations > BALLOON_ALLOC_YIELD_THRESHOLD) { + allocations++; + b->balloonOps->addPage(b, nPages++, handle); + if (nPages == b->batchMaxPages) { + status = b->balloonOps->lock(b, nPages); + nPages = 0; + + if (status != BALLOON_SUCCESS) { + break; + } + } + + if (allocations % BALLOON_ALLOC_YIELD_THRESHOLD == 0) { OS_Yield(); - allocations = 0; } - if (i >= rate) { + if (allocations >= rate) { /* We allocated enough pages, let's take a break. */ break; } } + if (nPages > 0) { + status = b->balloonOps->lock(b, nPages); + } + /* * We reached our goal without failures so try increasing * allocation rate. @@ -765,6 +775,336 @@ BalloonInflate(Balloon *b, // IN } +/* + *---------------------------------------------------------------------- + * + * BalloonLockv3 -- + * + * Lock all the batched page, previously stored by + * BalloonAddPagev3. + * + * Results: + * BALLOON_SUCCESS or an error code. + * + * Side effects: + * None. + * + *---------------------------------------------------------------------- + */ +static int +BalloonLockv3(Balloon *b, // IN + uint16 nPages) // IN +{ + int status; + uint32 i; + uint32 nLockedPages; + PageHandle handle; + PPN64 batchPagePPN; + BalloonChunk *chunk = NULL; + + Balloon_BatchSetNumPages(b->batchPage, nPages); + batchPagePPN = OS_ReservedPageGetPPN(b->pageHandle); + + /* + * Make sure that we will always have an available chunk before doing + * the LOCK_BATCHED call. + */ + ASSERT(b->batchMaxPages < BALLOON_CHUNK_PAGES); + b->fallbackChunk = BalloonChunk_Create(); + if (b->fallbackChunk == NULL) { + status = BALLOON_PAGE_ALLOC_FAILURE; + } else { + status = Backdoor_MonitorLockPage(b, batchPagePPN); + } + + if (status != BALLOON_SUCCESS) { + for (i = 0; i < nPages; i++) { + PPN64 ppn = Balloon_BatchGetPPN(b->batchPage, i); + handle = OS_ReservedPageGetHandle(ppn); + + OS_ReservedPageFree(handle); + } + + goto out; + } + + nLockedPages = 0; + for (i = 0; i < nPages; i++) { + PPN64 ppn; + int error; + + ppn = Balloon_BatchGetPPN(b->batchPage, i); + handle = OS_ReservedPageGetHandle(ppn); + error = Balloon_BatchGetStatus(b->batchPage, i); + if (error != BALLOON_SUCCESS) { + switch (error) { + case BALLOON_ERROR_PPN_PINNED: + case BALLOON_ERROR_PPN_INVALID: + if (BalloonErrorPageStore(b, handle) == BALLOON_SUCCESS) { + break; + } + // Fallthrough. + case BALLOON_ERROR_RESET: + case BALLOON_ERROR_PPN_NOTNEEDED: + OS_ReservedPageFree(handle); + break; + default: + /* + * If we fall here, there is definitively a bug in the + * driver that needs to be fixed, I'm not sure if + * PINNED and INVALID PPN can be seen as a bug in the + * driver. + */ + ASSERT(FALSE); + } + continue; + } + + if (chunk == NULL) { + chunk = BalloonGetChunkOrFallback(b); + } + + BalloonPageStore(chunk, handle); + if (chunk->pageCount == BALLOON_CHUNK_PAGES) { + chunk = NULL; + } + nLockedPages++; + } + + b->nPages += nLockedPages; + +out: + if (b->fallbackChunk != NULL) { + BalloonChunk_Destroy(b->fallbackChunk); + b->fallbackChunk = NULL; + } + + return status; +} + +/* + *---------------------------------------------------------------------- + * + * BalloonUnlockv3 -- + * + * Unlock all the batched page, previously stored by + * BalloonAddPagev3. + * + * Results: + * BALLOON_SUCCESS or an error code. + * + * Side effects: + * None. + * + *---------------------------------------------------------------------- + */ +static int +BalloonUnlockv3(Balloon *b, // IN + uint16 nPages) // IN +{ + uint32 i; + int status = BALLOON_SUCCESS; + uint32 nUnlockedPages; + PPN64 batchPagePPN; + BalloonChunk *chunk = NULL; + + Balloon_BatchSetNumPages(b->batchPage, nPages); + batchPagePPN = OS_ReservedPageGetPPN(b->pageHandle); + status = Backdoor_MonitorUnlockPage(b, batchPagePPN); + + if (status != BALLOON_SUCCESS) { + for (i = 0; i < nPages; i++) { + PPN64 ppn = Balloon_BatchGetPPN(b->batchPage, i); + PageHandle handle = OS_ReservedPageGetHandle(ppn); + + chunk = BalloonGetChunkOrFallback(b); + BalloonPageStore(chunk, handle); + } + goto out; + } + + nUnlockedPages = 0; + for (i = 0; i < nPages; i++) { + int status = Balloon_BatchGetStatus(b->batchPage, i); + PPN64 ppn = Balloon_BatchGetPPN(b->batchPage, i); + PageHandle handle = OS_ReservedPageGetHandle(ppn); + + if (status != BALLOON_SUCCESS) { + chunk = BalloonGetChunkOrFallback(b); + BalloonPageStore(chunk, handle); + continue; + } + + OS_ReservedPageFree(handle); + STATS_INC(b->stats.primFree); + + nUnlockedPages++; + } + + b->nPages -= nUnlockedPages; + +out: + if (b->fallbackChunk != NULL) { + BalloonChunk_Destroy(b->fallbackChunk); + b->fallbackChunk = NULL; + } + return status; +} + +/* + *---------------------------------------------------------------------- + * + * BalloonAddPagev3 -- + * + * Add a page to the batch page, that will be ballooned later. + * + * Results: + * Nothing. + * + * Side effects: + * None. + * + *---------------------------------------------------------------------- + */ + +static void +BalloonAddPagev3(Balloon *b, // IN + uint16 idx, // IN + PageHandle page) // IN +{ + PPN64 ppn = OS_ReservedPageGetPPN(page); + Balloon_BatchSetPPN(b->batchPage, idx, ppn); +} + +/* + *---------------------------------------------------------------------- + * + * Balloonlockv2 -- + * + * Lock a page, previously stored with a call to BalloonAddPagev2, + * by notifying the monitor. + * + * Results: + * BALLOON_SUCCESS or an error code. + * + * Side effects: + * None. + * + *---------------------------------------------------------------------- + */ + +static int +BalloonLockv2(Balloon *b, // IN + uint16 nPages) // IN +{ + PPN pagePPN; + BalloonChunk *chunk; + int status; + + /* Get the chunk to store allocated page. */ + chunk = BalloonGetChunk(b); + if (chunk == NULL) { + OS_ReservedPageFree(b->pageHandle); + return BALLOON_PAGE_ALLOC_FAILURE; + } + + /* inform monitor via backdoor */ + pagePPN = OS_ReservedPageGetPPN(b->pageHandle); + status = Backdoor_MonitorLockPage(b, pagePPN); + if (status != BALLOON_SUCCESS) { + int old_status = status; + + if (status == BALLOON_ERROR_RESET || + status == BALLOON_ERROR_PPN_NOTNEEDED) { + OS_ReservedPageFree(b->pageHandle); + return status; + } + + /* place on list of non-balloonable pages, retry allocation */ + status = BalloonErrorPageStore(b, b->pageHandle); + if (status != BALLOON_SUCCESS) { + OS_ReservedPageFree(b->pageHandle); + return status; + } + + return old_status; + } + + /* track allocated page */ + BalloonPageStore(chunk, b->pageHandle); + + /* update balloon size */ + b->nPages++; + + return status; +} + +/* + *---------------------------------------------------------------------- + * + * BalloonUnlockv2 -- + * + * Unlock a page, previously stored with a call to + * BalloonAddPagev2, by notifying the monitor. + * + * Results: + * BALLOON_SUCCESS or an error code. + * + * Side effects: + * None. + * + *---------------------------------------------------------------------- + */ + +static int +BalloonUnlockv2(Balloon *b, // IN + uint16 nPages) // IN +{ + PPN pagePPN = OS_ReservedPageGetPPN(b->pageHandle); + int status = Backdoor_MonitorUnlockPage(b, pagePPN); + + if (status != BALLOON_SUCCESS) { + BalloonChunk *chunk = BalloonGetChunkOrFallback(b); + BalloonPageStore(chunk, b->pageHandle); + goto out; + } + + OS_ReservedPageFree(b->pageHandle); + STATS_INC(b->stats.primFree); + +out: + if (b->fallbackChunk != NULL) { + BalloonChunk_Destroy(b->fallbackChunk); + b->fallbackChunk = NULL; + } + return status; +} + +/* + *---------------------------------------------------------------------- + * + * BalloonAddPagev2 -- + * + * Add a page to be ballooned later. + * + * Results: + * Nothing. + * + * Side effects: + * None. + * + *---------------------------------------------------------------------- + */ + +static void +BalloonAddPagev2(Balloon *b, // IN + uint16 idx, // IN + PageHandle page) // IN +{ + b->pageHandle = page; +} + + /* *---------------------------------------------------------------------- * @@ -782,34 +1122,72 @@ BalloonInflate(Balloon *b, // IN */ static int -BalloonDeflate(Balloon *b, // IN - uint32 target) // IN +BalloonDeflate(Balloon *b, // IN + uint32 target) // IN { - int status, i; - uint32 nFree = b->nPages - target; + int status = BALLOON_SUCCESS; + uint32 goal, nPages; + BalloonChunk *chunk = NULL; + + goal = b->nPages - target; /* limit deallocation rate */ - nFree = MIN(nFree, b->rateFree); + goal = MIN(goal, b->rateFree); - /* free pages to reach target */ - for (i = 0; i < nFree; i++) { - status = BalloonPageFree(b, TRUE); - if (status != BALLOON_SUCCESS) { - if (BALLOON_RATE_ADAPT) { - /* quickly decrease rate if error */ - b->rateFree = MAX(b->rateFree / 2, BALLOON_RATE_FREE_MIN); + nPages = 0; + for ( ; goal > 0; goal--) { + PageHandle lockedHandle; + + if (chunk == NULL) { + if (!DblLnkLst_IsLinked(&b->chunks)) { + /* The chunk list is empty, the balloon cannot be deflated */ + status = BALLOON_FAILURE; + goto out; } - return status; + + chunk = DblLnkLst_Container(b->chunks.next, BalloonChunk, node); + } + + lockedHandle = chunk->page[--chunk->pageCount]; + if (!chunk->pageCount) { + DblLnkLst_Unlink1(&chunk->node); + /* + * Do not free the chunk, we may need it if the UNLOCK cmd fails + */ + b->fallbackChunk = chunk; + + b->nChunks--; + chunk = NULL; + } + + b->balloonOps->addPage(b, nPages++, lockedHandle); + if (nPages == b->batchMaxPages) { + status = b->balloonOps->unlock(b, nPages); + nPages = 0; + + if (status != BALLOON_SUCCESS) { + if (BALLOON_RATE_ADAPT) { + /* quickly decrease rate if error */ + b->rateFree = MAX(b->rateFree / 2, BALLOON_RATE_FREE_MIN); + } + goto out; + } + } } - if (BALLOON_RATE_ADAPT) { + if (nPages) { + status = b->balloonOps->unlock(b, nPages); + } + + if (status == BALLOON_SUCCESS && BALLOON_RATE_ADAPT) { /* slowly increase rate if no errors */ b->rateFree = MIN(b->rateFree + BALLOON_RATE_FREE_INC, BALLOON_RATE_FREE_MAX); } - return BALLOON_SUCCESS; +out: + return status; } @@ -849,7 +1227,7 @@ BalloonAdjustSize(Balloon *b, // IN * * Balloon_Init -- * - * Initializes state oif the balloon. + * Initializes state of the balloon. * * Results: * Returns TRUE on success (always). @@ -876,6 +1254,10 @@ Balloon_Init(BalloonGuest guestType) /* initialize reset flag */ b->resetFlag = TRUE; + b->pageHandle = PAGE_HANDLE_INVALID; + b->batchPageMapping = MAPPING_INVALID; + b->batchPage = NULL; + return TRUE; } @@ -906,7 +1288,7 @@ Balloon_Cleanup(void) * Reset connection before deallocating memory to avoid potential for * additional spurious resets from guest touching deallocated pages. */ - Backdoor_MonitorStart(b); + Backdoor_MonitorStart(b, BALLOON_PROTOCOL_VERSION); Balloon_Deallocate(b); } diff --git a/open-vm-tools/modules/shared/vmmemctl/vmballoon.h b/open-vm-tools/modules/shared/vmmemctl/vmballoon.h index 016da687c..c859f4595 100644 --- a/open-vm-tools/modules/shared/vmmemctl/vmballoon.h +++ b/open-vm-tools/modules/shared/vmmemctl/vmballoon.h @@ -58,7 +58,7 @@ * *********************************************************/ -/* +/* * vmballoon.h: Definitions and macros for vmballoon driver. */ @@ -123,6 +123,16 @@ typedef struct { uint32 pageCount; } BalloonErrorPages; +#define BALLOON_CHUNK_PAGES 1000 + +typedef struct BalloonChunk { + PageHandle page[BALLOON_CHUNK_PAGES]; + uint32 pageCount; + DblLnkLst_Links node; +} BalloonChunk; + +struct BalloonOps; + typedef struct { /* sets of reserved physical pages */ DblLnkLst_Links chunks; @@ -152,8 +162,25 @@ typedef struct { /* balloon protocol to use */ uint32 hypervisorProtocolVersion; + + /* balloon operations, tied to the protocol version */ + const struct BalloonOps *balloonOps; + + /* Either the batch page handle, or the page to lock on v2 */ + PageHandle pageHandle; + Mapping batchPageMapping; + BalloonBatchPage *batchPage; + uint16 batchMaxPages; + + BalloonChunk *fallbackChunk; } Balloon; +typedef struct BalloonOps { + void (*addPage)(Balloon *b, uint16 idx, PageHandle page); + int (*lock)(Balloon *b, uint16 nPages); + int (*unlock)(Balloon *b, uint16 nPages); +} BalloonOps; + /* * Operations */ diff --git a/open-vm-tools/modules/solaris/vmmemctl/os.c b/open-vm-tools/modules/solaris/vmmemctl/os.c index 491374f6e..daf8a617d 100644 --- a/open-vm-tools/modules/solaris/vmmemctl/os.c +++ b/open-vm-tools/modules/solaris/vmmemctl/os.c @@ -40,6 +40,7 @@ #include "os.h" #include "vmballoon.h" #include "balloon_def.h" +#include "vm_assert.h" #include "vmballoon_kstats.h" #include "buildNumber.h" @@ -234,13 +235,123 @@ OS_ReservedPageGetLimit(void) *----------------------------------------------------------------------------- */ -unsigned long +PPN64 OS_ReservedPageGetPPN(PageHandle handle) // IN: A valid page handle { return page_pptonum(((os_page *)handle)->pp); } +/* + *----------------------------------------------------------------------------- + * + * OS_ReservedPageGetHandle -- + * + * Convert a ppn (of a physical page previously reserved with + * OS_ReservedPageAlloc()) to a page handle. + * + * Results: + * The page handle. + * + * Side effects: + * None. + * + * Note: + * Currently not implemented on Solaris. + * + *----------------------------------------------------------------------------- + */ + +PageHandle +OS_ReservedPageGetHandle(PPN64 ppn) // IN +{ + // Solaris does not use protocol v3. + NOT_IMPLEMENTED(); +} + + +/* + *----------------------------------------------------------------------------- + * + * OS_MapPageHandle -- + * + * Map a page handle into kernel address space, and return the + * mapping to that page handle. + * + * Results: + * The mapping. + * + * Side effects: + * None. + * + * Note: + * Currently not implemented on Solaris. + * + *----------------------------------------------------------------------------- + */ + +Mapping +OS_MapPageHandle(PageHandle handle) // IN +{ + // Solaris does not use protocol v3. + NOT_IMPLEMENTED(); +} + + +/* + *----------------------------------------------------------------------------- + * + * OS_Mapping2Addr -- + * + * Return the address of a previously mapped page handle (with + * OS_MapPageHandle). + * + * Results: + * The mapping address. + * + * Side effects: + * None. + * + * Note: + * Currently not implemented on Solaris. + * + *----------------------------------------------------------------------------- + */ + +void * +OS_Mapping2Addr(Mapping mapping) // IN +{ + // Solaris does not use protocol v3. + NOT_IMPLEMENTED(); +} + + +/* + *----------------------------------------------------------------------------- + * + * OS_UnmapPage -- + * + * Unmap a previously mapped page handle. + * + * Results: + * None. + * + * Side effects: + * None. + * + * Note: + * Currently not implemented on Solaris. + * + *----------------------------------------------------------------------------- + */ + +void +OS_UnmapPage(Mapping mapping) // IN +{ + // Solaris does not use protocol v3. + NOT_IMPLEMENTED(); +} + /* * NOTE: cast id before shifting to avoid overflow (id_t is 32 bits, * u_offset_t is 64 bits). Also, can't use ptob because it will