#include <cstdint>
#include <iostream>
#include <numaif.h>
#include <sys/mman.h>
#include <fcntl.h>
#include <errno.h>
#include <unistd.h>
#include <string.h>
#include <limits>
int main(int argc, char** argv) {
const constexpr uint64_t size = 16lu * 1024 * 1024;
const constexpr uint32_t nPages = size / (4lu * 1024 * 1024);
int32_t status[nPages];
std::fill_n(status, nPages, std::numeric_limits<int32_t>::min());
void* pages[nPages];
auto fd = shm_open("test_shm", O_RDWR|O_CREAT, 0666);
void* ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
if (ptr == MAP_FAILED) {
if (fd > 0) close(fd);
throw "failed to map hugepages";
}
for (uint32_t i = 0; i < nPages; i++) {
pages[i] = (char*)ptr + 4 * 1024 * 1024;
}
if (0 != move_pages(0, nPages, pages, nullptr, status, 0)) {
std::cout << "failed to inquiry pages because " << strerror(errno) << std::endl;
}
else {
for (uint32_t i = 0; i < nPages; i++) {
std::cout << "page # " << i << " locates at numa node " << status[i] << std::endl;
}
}
munmap(ptr, size);
close(fd);
}
它打印:
根据manpage，它指出:
为什么查询返回成功却打印负值？我的机器只有2个NUMA-0和1。
内核版本:3.10.0-862.2.3.el7.x86_64
这是大页面的版本:
#include <cstdint>
#include <iostream>
#include <numaif.h>
#include <sys/mman.h>
#include <fcntl.h>
#include <errno.h>
#include <unistd.h>
#include <string.h>
#include <limits>
int main(int argc, char** argv) {
const int32_t dst_node = strtoul(argv[1], nullptr, 10);
const constexpr uint64_t size = 4lu * 1024 * 1024;
const constexpr uint64_t pageSize = 2lu * 1024 * 1024;
const constexpr uint32_t nPages = size / pageSize;
int32_t status[nPages];
std::fill_n(status, nPages, std::numeric_limits<int32_t>::min());
void* pages[nPages];
int32_t dst_nodes[nPages];
void* ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE | MAP_HUGETLB, -1, 0);
if (ptr == MAP_FAILED) {
throw "failed to map hugepages";
}
memset(ptr, 0x41, nPages*pageSize);
for (uint32_t i = 0; i < nPages; i++) {
pages[i] = &((char*)ptr)[i*pageSize];
dst_nodes[i] = dst_node;
}
std::cout << "Before moving" << std::endl;
if (0 != move_pages(0, nPages, pages, nullptr, status, 0)) {
std::cout << "failed to inquiry pages because " << strerror(errno) << std::endl;
}
else {
for (uint32_t i = 0; i < nPages; i++) {
std::cout << "page # " << i << " locates at numa node " << status[i] << std::endl;
}
}
// real move
if (0 != move_pages(0, nPages, pages, dst_nodes, status, MPOL_MF_MOVE_ALL)) {
std::cout << "failed to move pages because " << strerror(errno) << std::endl;
exit(-1);
}
const constexpr uint64_t smallPageSize = 4lu * 1024;
const constexpr uint32_t nSmallPages = size / smallPageSize;
void* smallPages[nSmallPages];
int32_t smallStatus[nSmallPages] = {std::numeric_limits<int32_t>::min()};
for (uint32_t i = 0; i < nSmallPages; i++) {
smallPages[i] = &((char*)ptr)[i*smallPageSize];
}
std::cout << "after moving" << std::endl;
if (0 != move_pages(0, nSmallPages, smallPages, nullptr, smallStatus, 0)) {
std::cout << "failed to inquiry pages because " << strerror(errno) << std::endl;
}
else {
for (uint32_t i = 0; i < nSmallPages; i++) {
std::cout << "page # " << i << " locates at numa node " << smallStatus[i] << std::endl;
}
}
}
有趣的是，随着移动大页面后，
move_pages()似乎可以理解大页面，我根据小页面大小进行查询，并填充了预期的NUMA ID。 最佳答案
您对shm_open和mmap的使用可能不会获得所需的大页面。move_pages syscall(和libnuma包装器)适用于x86_64的4096字节标准页面。
并且您以错误的方式使用了move_pages以及错误的第三个参数“pages”。它不应该是指向内存的指针。但是指向数组的指针，数组本身将包含nPages指针:
http://man7.org/linux/man-pages/man2/move_pages.2.html
long move_pages(int pid, unsigned long count, void **pages,
const int *nodes, int *status, int flags);
pages is an array of pointers to the pages that should be moved.
These are pointers that should be aligned to page boundaries.
Addresses are specified as seen by the process specified by pid.
如果在“页面”中没有正确的指针，根据
errno 14(来自moreutils包)，您将得到-14，即EFAULT。//https://stackoverflow.com/questions/54546367/fail-to-query-via-move-pages
//g++ 54546367.move_pages.cc -o 54546367.move_pages -lnuma -lrt
#include <cstdint>
#include <iostream>
#include <numaif.h>
#include <sys/mman.h>
#include <fcntl.h>
#include <errno.h>
#include <unistd.h>
#include <string.h>
#include <limits>
int main(int argc, char** argv) {
const constexpr uint64_t size = 256lu * 1024;// * 1024;
const constexpr uint32_t nPages = size / (4lu * 1024);
void * pages[nPages];
int32_t status[nPages];
std::fill_n(status, nPages, std::numeric_limits<int32_t>::min());
// auto fd = shm_open("test_shm", O_RDWR|O_CREAT, 0666);
// void* ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
void* ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, 0, 0);
std::cout << "Ptr is " << ptr << std::endl;
if (ptr == MAP_FAILED) {
// if (fd > 0) close(fd);
throw "failed to map hugepages";
}
memset(ptr, 0x41, nPages*4096);
for(uint32_t i = 0; i<nPages; i++) {
pages[i] = &((char*)ptr)[i*4096];
}
if (0 != move_pages(0, nPages, pages, nullptr, status, 0)) {
std::cout << "failed to inquiry pages because " << strerror(errno) << std::endl;
}
else {
for (uint32_t i = 0; i < nPages; i++) {
std::cout << "page # " << i << " locates at numa node " << status[i] << std::endl;
}
}
munmap(ptr, size);
// close(fd);
}
使用NUMA机器时，它在启动时输出与
taskset -c 7 ./54546367.move_pages相同的节点，在numactl -i all ./54546367.move_pages时交错输出(0 1 0 1)。关于c++ - 无法通过move_pages()查询，我们在Stack Overflow上找到一个类似的问题：https://stackoverflow.com/questions/54546367/