VirtualAllocExNuma
|
|
Home |
 |
Back To Tips Page |
|
|
VirtualAllocExNuma |
|
The example given in the MSDN in the article titled "Allocating Memory from a NUMA Node", which is linked to from the VirtualAllocExNuma description, represents several extremely bad coding practices. It looks like a beginner wrote it. This project cleans up the code and makes it look like it was written by a programmer with more than six weeks' experience. For critique of the original code, see this article.
The output from the program is shown below. This represents one test on one snapshot of one Vista machine.
I changed the specs a bit; the original wanted a command line argument of the number of bytes to allocate, expressed in hexadecimal. This certainly made no sense as a user interface. I changed it to take a quantity of pages expressed as a decimal number.
The subroutines shown were adapted to work in my NUMA Explorer.
Note that although there was a desire to allocate 4096 pages for CPU1 on its preferred node (node 0) in fact only 2959 pages could be allocated on that node; the remainder were allocated on node 1. I have seen traces where the blocks were often under 10 pages and there was interleaving for hundreds of blocks that were piecewise on different nodes. These are far too long and boring to paste into a Web page in their entirety. But here's part of one
Compare the two side-by-side. Note that to reduce the column width, a few comments and long lines of both sources have been re-wrapped.
| Original Code | Rewritten Code |
#include <windows.h> #include <stdlib.h> #include <stdio.h> #include <psapi.h> SIZE_T AllocationSize; DWORD PageSize; void DumpNumaNodeInfo (PVOID Buffer, SIZE_T Size); void __cdecl wmain (int argc, const wchar_t* argv[]) { ULONG HighestNodeNumber; ULONG NumberOfProcessors; if (argc > 1) { (void)swscanf (argv[1], L"%Ix", &AllocationSize); } if (AllocationSize == 0) { AllocationSize = 16*1024*1024; } // // Get the number of processors and system page size. // SYSTEM_INFO SystemInfo; GetSystemInfo (&SystemInfo); NumberOfProcessors = SystemInfo.dwNumberOfProcessors; PageSize = SystemInfo.dwPageSize; // // Get the highest node number. // if (TRUE != GetNumaHighestNodeNumber (&HighestNodeNumber)) { printf ("GetNumaHighestNodeNumber failed: 0x%x\r\n", GetLastError()); goto Exit; } if (HighestNodeNumber == 0) { puts ("Not a NUMA system - exiting"); goto Exit; } // // Allocate array of pointers to memory blocks. // PVOID* Buffers = (PVOID*) malloc (sizeof(PVOID)*NumberOfProcessors); if (Buffers == NULL) { puts ("Allocating array of buffers failed"); goto Exit; } ZeroMemory (Buffers, sizeof(PVOID)*NumberOfProcessors); // // For each processor, get its associated NUMA node // and allocate some memory from it. // for (UCHAR i = 0; i < NumberOfProcessors; i++) { UCHAR NodeNumber; if (TRUE != GetNumaProcessorNode (i, &NodeNumber)) { printf ("GetNumaProcessorNode failed: 0x%x\r\n", GetLastError()); goto Exit; } printf ("CPU %u: node %u\r\n", (ULONG)i, NodeNumber); PCHAR Buffer = (PCHAR)VirtualAllocExNuma( GetCurrentProcess(), NULL, AllocationSize, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE, NodeNumber ); if (Buffer == NULL) { printf ("VirtualAllocExNuma failed: 0x%x, node %u\r\n", GetLastError(), NodeNumber); goto Exit; } PCHAR BufferEnd = Buffer + AllocationSize - 1; SIZE_T NumPages = ((SIZE_T)BufferEnd)/PageSize - ((SIZE_T)Buffer)/PageSize + 1; puts ("Allocated virtual memory:"); printf ("%p - %p (%6Iu pages), preferred node %u\r\n", Buffer, BufferEnd, NumPages, NodeNumber); Buffers[i] = Buffer; // // At this point, virtual pages are allocated but no valid physical // pages are associated with them yet. // // The FillMemory call below will touch every page in the buffer, // faulting them into our working set. When this happens physical // pages will be allocated from the preferred node we specified // in VirtualAllocExNuma, or any node // if the preferred one is out of pages. // FillMemory (Buffer, AllocationSize, 'x'); // // Check the actual node number for the physical pages that are still valid // (if system is low on physical memory, // some pages could have been trimmed already). // DumpNumaNodeInfo (Buffer, AllocationSize); puts(""); } Exit: if (Buffers != NULL) { for (UINT i = 0; i < NumberOfProcessors; i++) { if (Buffers[i] != NULL) { VirtualFree (Buffers[i], 0, MEM_RELEASE); } } free (Buffers); } } void DumpRegion (PVOID StartPtr, PVOID EndPtr, BOOL Valid, DWORD Node) { DWORD_PTR StartPage = ((DWORD_PTR)StartPtr)/PageSize; DWORD_PTR EndPage = ((DWORD_PTR)EndPtr)/PageSize; DWORD_PTR NumPages = (EndPage - StartPage) + 1; if (Valid == TRUE) { printf ("%p - %p (%6Iu pages): node %u\r\n", StartPtr, EndPtr, NumPages, Node); } else { printf ("%p - %p (%6Iu pages): no valid pages\r\n", StartPtr, EndPtr, NumPages); } } void DumpNumaNodeInfo (PVOID Buffer, SIZE_T Size) { DWORD_PTR StartPage = ((DWORD_PTR)Buffer)/PageSize; DWORD_PTR EndPage = ((DWORD_PTR)Buffer + Size - 1)/PageSize; DWORD_PTR NumPages = (EndPage - StartPage) + 1; PCHAR StartPtr = (PCHAR)(PageSize*StartPage); puts ("Checking NUMA node:"); PPSAPI_WORKING_SET_EX_INFORMATION WsInfo = (PPSAPI_WORKING_SET_EX_INFORMATION) malloc (NumPages*sizeof(PSAPI_WORKING_SET_EX_INFORMATION)); if (WsInfo == NULL) { puts ("Could not allocate array of PSAPI_WORKING_SET_EX_INFORMATION structures"); return; } for (DWORD_PTR i = 0; i < NumPages; i++) { WsInfo[i].VirtualAddress = StartPtr + i*PageSize; } BOOL bResult = QueryWorkingSetEx( GetCurrentProcess(), WsInfo, NumPages*sizeof(PSAPI_WORKING_SET_EX_INFORMATION) ); if (bResult != TRUE) { printf ("QueryWorkingSetEx failed: 0x%x\r\n", GetLastError()); free (WsInfo); return; } PCHAR RegionStart; BOOL RegionIsValid; DWORD RegionNode; for (DWORD_PTR i = 0; i < NumPages; i++) { PCHAR Address = (PCHAR)WsInfo[i].VirtualAddress; BOOL IsValid = WsInfo[i].u1.VirtualAttributes.Valid; DWORD Node = WsInfo[i].u1.VirtualAttributes.Node; if (i == 0) { RegionStart = Address; RegionIsValid = IsValid; RegionNode = Node; } if (IsValid != RegionIsValid || Node != RegionNode) { DumpRegion (RegionStart, Address - 1, RegionIsValid, RegionNode); RegionStart = Address; RegionIsValid = IsValid; RegionNode = Node; } if (i == (NumPages - 1)) { DumpRegion (RegionStart, Address + PageSize - 1, IsValid, Node); } } free (WsInfo); } |
#include "stdafx.h" // Moved all standard includes to stdafx.h #pragma comment(lib, "psapi.lib") // Make sure it links by putting the lib specifications here /* pseudo-const */ DWORD PageSize; // written once at startup and never changed // Mark the above as write-once; it is the only global variable // that could possibly make sense void DumpNumaNodeInfo (PVOID Buffer, SIZE_T Size); /**************************************************************************** * ReportError * Inputs: * LPCTSTR msg: First part of message * DWORD err: Error code * Result: void * * Effect: * Issues the error message ****************************************************************************/ // All functions get a header so the code is readable and well-documented static void ReportError(LPCTSTR msg, DWORD err) { LPTSTR errmsg; if(::FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, NULL, // source err, // Message ID 0, // Default language ID (LPTSTR)&errmsg, // buffer 0, // size NULL) == 0) // argument list { /* No message code found */ _tprintf(_T("%s: Error %d (0x%08x)\n"), msg, err, err); return; } /* No message code found */ else { /* has message */ _tprintf(_T("%s: %s\n"), msg, errmsg); LocalFree(errmsg); return; } /* has message */ } // ReportError // The above code is so simple that it should be hyperlinked from every example // that needs to display an error /**************************************************************************** * class BufferInfo ****************************************************************************/ class BufferInfo { public: BufferInfo() { Buffer = NULL; } virtual ~BufferInfo() { if(Buffer != NULL) ::VirtualFree(Buffer, 0, MEM_RELEASE); } PVOID Buffer; }; // There is no need to zero fields and have deletion loops when C++ // makes this so easy! Note the example was written in C++! /**************************************************************************** * _tmain * Inputs: * int argc: Number of pages to allocate * _TCHAR * argv[]: Array of arguments * Result: int * 0 if successful * 1 if error * Effect: * The program * Argv: * [1] If present, the number of pages to allocate, as a decimal number * If argc < 2, defaults to 4096 pages ****************************************************************************/ int _tmain(int argc, _TCHAR* argv[]) // This is the correct prototype for main { ULONG NumberOfProcessors; SIZE_T AllocationSize = 0; // Since this is not used except in this function, there was // no reason to make it a global variable // // Get the number of processors and system page size. // SYSTEM_INFO SystemInfo; GetSystemInfo (&SystemInfo); NumberOfProcessors = SystemInfo.dwNumberOfProcessors; PageSize = SystemInfo.dwPageSize; if (argc > 1) { AllocationSize = _ttoi(argv[1]) * PageSize; } if (AllocationSize == 0) { // If not specified, default to 4096 pages AllocationSize = 4096 * PageSize; } //**************************************************************** // Get the highest node number. //**************************************************************** ULONG HighestNodeNumber; // Since this variable is not needed until the next line // it makes no sense to have declared it at the block head if (!GetNumaHighestNodeNumber (&HighestNodeNumber)) { DWORD err = GetLastError(); ReportError(_T("GetNumaHighestNodeNumber failed"), err); return 1; } if (HighestNodeNumber == 0) { _putts (_T("Not a NUMA system - exiting\n")); return 1; } //**************************************************************** // Allocate array of pointers to memory blocks. //**************************************************************** std::vector |
VS2008 source code for the Microsoft example (rewritten to look like
professional code)
![[Dividing Line Image]](div.gif){kind=small}
The views expressed in these essays are those of the author, and in no way represent, nor are they endorsed by, Microsoft.