| Author | Tokens | Token Proportion | Commits | Commit Proportion |
|---|---|---|---|---|
| Ralph Campbell | 9699 | 88.46% | 7 | 33.33% |
| Alex Sierra | 799 | 7.29% | 3 | 14.29% |
| Tarun Sahu | 264 | 2.41% | 1 | 4.76% |
| Alistair Popple | 178 | 1.62% | 3 | 14.29% |
| Ayush Jain | 11 | 0.10% | 1 | 4.76% |
| John Hubbard | 5 | 0.05% | 2 | 9.52% |
| Jakub Kiciński | 4 | 0.04% | 1 | 4.76% |
| Axel Rasmussen | 2 | 0.02% | 1 | 4.76% |
| Andrew Morton | 1 | 0.01% | 1 | 4.76% |
| Donet Tom | 1 | 0.01% | 1 | 4.76% |
| Total | 10964 | 21 |
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059
// SPDX-License-Identifier: GPL-2.0 /* * HMM stands for Heterogeneous Memory Management, it is a helper layer inside * the linux kernel to help device drivers mirror a process address space in * the device. This allows the device to use the same address space which * makes communication and data exchange a lot easier. * * This framework's sole purpose is to exercise various code paths inside * the kernel to make sure that HMM performs as expected and to flush out any * bugs. */ #include "../kselftest_harness.h" #include <errno.h> #include <fcntl.h> #include <stdio.h> #include <stdlib.h> #include <stdint.h> #include <unistd.h> #include <strings.h> #include <time.h> #include <pthread.h> #include <sys/types.h> #include <sys/stat.h> #include <sys/mman.h> #include <sys/ioctl.h> /* * This is a private UAPI to the kernel test module so it isn't exported * in the usual include/uapi/... directory. */ #include <lib/test_hmm_uapi.h> #include <mm/gup_test.h> struct hmm_buffer { void *ptr; void *mirror; unsigned long size; int fd; uint64_t cpages; uint64_t faults; }; enum { HMM_PRIVATE_DEVICE_ONE, HMM_PRIVATE_DEVICE_TWO, HMM_COHERENCE_DEVICE_ONE, HMM_COHERENCE_DEVICE_TWO, }; #define TWOMEG (1 << 21) #define HMM_BUFFER_SIZE (1024 << 12) #define HMM_PATH_MAX 64 #define NTIMES 10 #define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1))) /* Just the flags we need, copied from mm.h: */ #ifndef FOLL_WRITE #define FOLL_WRITE 0x01 /* check pte is writable */ #endif #ifndef FOLL_LONGTERM #define FOLL_LONGTERM 0x100 /* mapping lifetime is indefinite */ #endif FIXTURE(hmm) { int fd; unsigned int page_size; unsigned int page_shift; }; FIXTURE_VARIANT(hmm) { int device_number; }; FIXTURE_VARIANT_ADD(hmm, hmm_device_private) { .device_number = HMM_PRIVATE_DEVICE_ONE, }; FIXTURE_VARIANT_ADD(hmm, hmm_device_coherent) { .device_number = HMM_COHERENCE_DEVICE_ONE, }; FIXTURE(hmm2) { int fd0; int fd1; unsigned int page_size; unsigned int page_shift; }; FIXTURE_VARIANT(hmm2) { int device_number0; int device_number1; }; FIXTURE_VARIANT_ADD(hmm2, hmm2_device_private) { .device_number0 = HMM_PRIVATE_DEVICE_ONE, .device_number1 = HMM_PRIVATE_DEVICE_TWO, }; FIXTURE_VARIANT_ADD(hmm2, hmm2_device_coherent) { .device_number0 = HMM_COHERENCE_DEVICE_ONE, .device_number1 = HMM_COHERENCE_DEVICE_TWO, }; static int hmm_open(int unit) { char pathname[HMM_PATH_MAX]; int fd; snprintf(pathname, sizeof(pathname), "/dev/hmm_dmirror%d", unit); fd = open(pathname, O_RDWR, 0); if (fd < 0) fprintf(stderr, "could not open hmm dmirror driver (%s)\n", pathname); return fd; } static bool hmm_is_coherent_type(int dev_num) { return (dev_num >= HMM_COHERENCE_DEVICE_ONE); } FIXTURE_SETUP(hmm) { self->page_size = sysconf(_SC_PAGE_SIZE); self->page_shift = ffs(self->page_size) - 1; self->fd = hmm_open(variant->device_number); if (self->fd < 0 && hmm_is_coherent_type(variant->device_number)) SKIP(return, "DEVICE_COHERENT not available"); ASSERT_GE(self->fd, 0); } FIXTURE_SETUP(hmm2) { self->page_size = sysconf(_SC_PAGE_SIZE); self->page_shift = ffs(self->page_size) - 1; self->fd0 = hmm_open(variant->device_number0); if (self->fd0 < 0 && hmm_is_coherent_type(variant->device_number0)) SKIP(return, "DEVICE_COHERENT not available"); ASSERT_GE(self->fd0, 0); self->fd1 = hmm_open(variant->device_number1); ASSERT_GE(self->fd1, 0); } FIXTURE_TEARDOWN(hmm) { int ret = close(self->fd); ASSERT_EQ(ret, 0); self->fd = -1; } FIXTURE_TEARDOWN(hmm2) { int ret = close(self->fd0); ASSERT_EQ(ret, 0); self->fd0 = -1; ret = close(self->fd1); ASSERT_EQ(ret, 0); self->fd1 = -1; } static int hmm_dmirror_cmd(int fd, unsigned long request, struct hmm_buffer *buffer, unsigned long npages) { struct hmm_dmirror_cmd cmd; int ret; /* Simulate a device reading system memory. */ cmd.addr = (__u64)buffer->ptr; cmd.ptr = (__u64)buffer->mirror; cmd.npages = npages; for (;;) { ret = ioctl(fd, request, &cmd); if (ret == 0) break; if (errno == EINTR) continue; return -errno; } buffer->cpages = cmd.cpages; buffer->faults = cmd.faults; return 0; } static void hmm_buffer_free(struct hmm_buffer *buffer) { if (buffer == NULL) return; if (buffer->ptr) munmap(buffer->ptr, buffer->size); free(buffer->mirror); free(buffer); } /* * Create a temporary file that will be deleted on close. */ static int hmm_create_file(unsigned long size) { char path[HMM_PATH_MAX]; int fd; strcpy(path, "/tmp"); fd = open(path, O_TMPFILE | O_EXCL | O_RDWR, 0600); if (fd >= 0) { int r; do { r = ftruncate(fd, size); } while (r == -1 && errno == EINTR); if (!r) return fd; close(fd); } return -1; } /* * Return a random unsigned number. */ static unsigned int hmm_random(void) { static int fd = -1; unsigned int r; if (fd < 0) { fd = open("/dev/urandom", O_RDONLY); if (fd < 0) { fprintf(stderr, "%s:%d failed to open /dev/urandom\n", __FILE__, __LINE__); return ~0U; } } read(fd, &r, sizeof(r)); return r; } static void hmm_nanosleep(unsigned int n) { struct timespec t; t.tv_sec = 0; t.tv_nsec = n; nanosleep(&t, NULL); } static int hmm_migrate_sys_to_dev(int fd, struct hmm_buffer *buffer, unsigned long npages) { return hmm_dmirror_cmd(fd, HMM_DMIRROR_MIGRATE_TO_DEV, buffer, npages); } static int hmm_migrate_dev_to_sys(int fd, struct hmm_buffer *buffer, unsigned long npages) { return hmm_dmirror_cmd(fd, HMM_DMIRROR_MIGRATE_TO_SYS, buffer, npages); } /* * Simple NULL test of device open/close. */ TEST_F(hmm, open_close) { } /* * Read private anonymous memory. */ TEST_F(hmm, anon_read) { struct hmm_buffer *buffer; unsigned long npages; unsigned long size; unsigned long i; int *ptr; int ret; int val; npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; ASSERT_NE(npages, 0); size = npages << self->page_shift; buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); buffer->fd = -1; buffer->size = size; buffer->mirror = malloc(size); ASSERT_NE(buffer->mirror, NULL); buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); ASSERT_NE(buffer->ptr, MAP_FAILED); /* * Initialize buffer in system memory but leave the first two pages * zero (pte_none and pfn_zero). */ i = 2 * self->page_size / sizeof(*ptr); for (ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; /* Set buffer permission to read-only. */ ret = mprotect(buffer->ptr, size, PROT_READ); ASSERT_EQ(ret, 0); /* Populate the CPU page table with a special zero page. */ val = *(int *)(buffer->ptr + self->page_size); ASSERT_EQ(val, 0); /* Simulate a device reading system memory. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); ASSERT_EQ(buffer->faults, 1); /* Check what the device read. */ ptr = buffer->mirror; for (i = 0; i < 2 * self->page_size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], 0); for (; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); hmm_buffer_free(buffer); } /* * Read private anonymous memory which has been protected with * mprotect() PROT_NONE. */ TEST_F(hmm, anon_read_prot) { struct hmm_buffer *buffer; unsigned long npages; unsigned long size; unsigned long i; int *ptr; int ret; npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; ASSERT_NE(npages, 0); size = npages << self->page_shift; buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); buffer->fd = -1; buffer->size = size; buffer->mirror = malloc(size); ASSERT_NE(buffer->mirror, NULL); buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); ASSERT_NE(buffer->ptr, MAP_FAILED); /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; /* Initialize mirror buffer so we can verify it isn't written. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ptr[i] = -i; /* Protect buffer from reading. */ ret = mprotect(buffer->ptr, size, PROT_NONE); ASSERT_EQ(ret, 0); /* Simulate a device reading system memory. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); ASSERT_EQ(ret, -EFAULT); /* Allow CPU to read the buffer so we can check it. */ ret = mprotect(buffer->ptr, size, PROT_READ); ASSERT_EQ(ret, 0); for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); /* Check what the device read. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], -i); hmm_buffer_free(buffer); } /* * Write private anonymous memory. */ TEST_F(hmm, anon_write) { struct hmm_buffer *buffer; unsigned long npages; unsigned long size; unsigned long i; int *ptr; int ret; npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; ASSERT_NE(npages, 0); size = npages << self->page_shift; buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); buffer->fd = -1; buffer->size = size; buffer->mirror = malloc(size); ASSERT_NE(buffer->mirror, NULL); buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); ASSERT_NE(buffer->ptr, MAP_FAILED); /* Initialize data that the device will write to buffer->ptr. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ptr[i] = i; /* Simulate a device writing system memory. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); ASSERT_EQ(buffer->faults, 1); /* Check what the device wrote. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); hmm_buffer_free(buffer); } /* * Write private anonymous memory which has been protected with * mprotect() PROT_READ. */ TEST_F(hmm, anon_write_prot) { struct hmm_buffer *buffer; unsigned long npages; unsigned long size; unsigned long i; int *ptr; int ret; npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; ASSERT_NE(npages, 0); size = npages << self->page_shift; buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); buffer->fd = -1; buffer->size = size; buffer->mirror = malloc(size); ASSERT_NE(buffer->mirror, NULL); buffer->ptr = mmap(NULL, size, PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); ASSERT_NE(buffer->ptr, MAP_FAILED); /* Simulate a device reading a zero page of memory. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, 1); ASSERT_EQ(buffer->faults, 1); /* Initialize data that the device will write to buffer->ptr. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ptr[i] = i; /* Simulate a device writing system memory. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); ASSERT_EQ(ret, -EPERM); /* Check what the device wrote. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], 0); /* Now allow writing and see that the zero page is replaced. */ ret = mprotect(buffer->ptr, size, PROT_WRITE | PROT_READ); ASSERT_EQ(ret, 0); /* Simulate a device writing system memory. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); ASSERT_EQ(buffer->faults, 1); /* Check what the device wrote. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); hmm_buffer_free(buffer); } /* * Check that a device writing an anonymous private mapping * will copy-on-write if a child process inherits the mapping. */ TEST_F(hmm, anon_write_child) { struct hmm_buffer *buffer; unsigned long npages; unsigned long size; unsigned long i; int *ptr; pid_t pid; int child_fd; int ret; npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; ASSERT_NE(npages, 0); size = npages << self->page_shift; buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); buffer->fd = -1; buffer->size = size; buffer->mirror = malloc(size); ASSERT_NE(buffer->mirror, NULL); buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); ASSERT_NE(buffer->ptr, MAP_FAILED); /* Initialize buffer->ptr so we can tell if it is written. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; /* Initialize data that the device will write to buffer->ptr. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ptr[i] = -i; pid = fork(); if (pid == -1) ASSERT_EQ(pid, 0); if (pid != 0) { waitpid(pid, &ret, 0); ASSERT_EQ(WIFEXITED(ret), 1); /* Check that the parent's buffer did not change. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); return; } /* Check that we see the parent's values. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], -i); /* The child process needs its own mirror to its own mm. */ child_fd = hmm_open(0); ASSERT_GE(child_fd, 0); /* Simulate a device writing system memory. */ ret = hmm_dmirror_cmd(child_fd, HMM_DMIRROR_WRITE, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); ASSERT_EQ(buffer->faults, 1); /* Check what the device wrote. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], -i); close(child_fd); exit(0); } /* * Check that a device writing an anonymous shared mapping * will not copy-on-write if a child process inherits the mapping. */ TEST_F(hmm, anon_write_child_shared) { struct hmm_buffer *buffer; unsigned long npages; unsigned long size; unsigned long i; int *ptr; pid_t pid; int child_fd; int ret; npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; ASSERT_NE(npages, 0); size = npages << self->page_shift; buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); buffer->fd = -1; buffer->size = size; buffer->mirror = malloc(size); ASSERT_NE(buffer->mirror, NULL); buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, buffer->fd, 0); ASSERT_NE(buffer->ptr, MAP_FAILED); /* Initialize buffer->ptr so we can tell if it is written. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; /* Initialize data that the device will write to buffer->ptr. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ptr[i] = -i; pid = fork(); if (pid == -1) ASSERT_EQ(pid, 0); if (pid != 0) { waitpid(pid, &ret, 0); ASSERT_EQ(WIFEXITED(ret), 1); /* Check that the parent's buffer did change. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], -i); return; } /* Check that we see the parent's values. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], -i); /* The child process needs its own mirror to its own mm. */ child_fd = hmm_open(0); ASSERT_GE(child_fd, 0); /* Simulate a device writing system memory. */ ret = hmm_dmirror_cmd(child_fd, HMM_DMIRROR_WRITE, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); ASSERT_EQ(buffer->faults, 1); /* Check what the device wrote. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], -i); close(child_fd); exit(0); } /* * Write private anonymous huge page. */ TEST_F(hmm, anon_write_huge) { struct hmm_buffer *buffer; unsigned long npages; unsigned long size; unsigned long i; void *old_ptr; void *map; int *ptr; int ret; size = 2 * TWOMEG; buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); buffer->fd = -1; buffer->size = size; buffer->mirror = malloc(size); ASSERT_NE(buffer->mirror, NULL); buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); ASSERT_NE(buffer->ptr, MAP_FAILED); size = TWOMEG; npages = size >> self->page_shift; map = (void *)ALIGN((uintptr_t)buffer->ptr, size); ret = madvise(map, size, MADV_HUGEPAGE); ASSERT_EQ(ret, 0); old_ptr = buffer->ptr; buffer->ptr = map; /* Initialize data that the device will write to buffer->ptr. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ptr[i] = i; /* Simulate a device writing system memory. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); ASSERT_EQ(buffer->faults, 1); /* Check what the device wrote. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); buffer->ptr = old_ptr; hmm_buffer_free(buffer); } /* * Read numeric data from raw and tagged kernel status files. Used to read * /proc and /sys data (without a tag) and from /proc/meminfo (with a tag). */ static long file_read_ulong(char *file, const char *tag) { int fd; char buf[2048]; int len; char *p, *q; long val; fd = open(file, O_RDONLY); if (fd < 0) { /* Error opening the file */ return -1; } len = read(fd, buf, sizeof(buf)); close(fd); if (len < 0) { /* Error in reading the file */ return -1; } if (len == sizeof(buf)) { /* Error file is too large */ return -1; } buf[len] = '\0'; /* Search for a tag if provided */ if (tag) { p = strstr(buf, tag); if (!p) return -1; /* looks like the line we want isn't there */ p += strlen(tag); } else p = buf; val = strtol(p, &q, 0); if (*q != ' ') { /* Error parsing the file */ return -1; } return val; } /* * Write huge TLBFS page. */ TEST_F(hmm, anon_write_hugetlbfs) { struct hmm_buffer *buffer; unsigned long npages; unsigned long size; unsigned long default_hsize; unsigned long i; int *ptr; int ret; default_hsize = file_read_ulong("/proc/meminfo", "Hugepagesize:"); if (default_hsize < 0 || default_hsize*1024 < default_hsize) SKIP(return, "Huge page size could not be determined"); default_hsize = default_hsize*1024; /* KB to B */ size = ALIGN(TWOMEG, default_hsize); npages = size >> self->page_shift; buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); if (buffer->ptr == MAP_FAILED) { free(buffer); SKIP(return, "Huge page could not be allocated"); } buffer->fd = -1; buffer->size = size; buffer->mirror = malloc(size); ASSERT_NE(buffer->mirror, NULL); /* Initialize data that the device will write to buffer->ptr. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ptr[i] = i; /* Simulate a device writing system memory. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); ASSERT_EQ(buffer->faults, 1); /* Check what the device wrote. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); munmap(buffer->ptr, buffer->size); buffer->ptr = NULL; hmm_buffer_free(buffer); } /* * Read mmap'ed file memory. */ TEST_F(hmm, file_read) { struct hmm_buffer *buffer; unsigned long npages; unsigned long size; unsigned long i; int *ptr; int ret; int fd; ssize_t len; npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; ASSERT_NE(npages, 0); size = npages << self->page_shift; fd = hmm_create_file(size); ASSERT_GE(fd, 0); buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); buffer->fd = fd; buffer->size = size; buffer->mirror = malloc(size); ASSERT_NE(buffer->mirror, NULL); /* Write initial contents of the file. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ptr[i] = i; len = pwrite(fd, buffer->mirror, size, 0); ASSERT_EQ(len, size); memset(buffer->mirror, 0, size); buffer->ptr = mmap(NULL, size, PROT_READ, MAP_SHARED, buffer->fd, 0); ASSERT_NE(buffer->ptr, MAP_FAILED); /* Simulate a device reading system memory. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); ASSERT_EQ(buffer->faults, 1); /* Check what the device read. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); hmm_buffer_free(buffer); } /* * Write mmap'ed file memory. */ TEST_F(hmm, file_write) { struct hmm_buffer *buffer; unsigned long npages; unsigned long size; unsigned long i; int *ptr; int ret; int fd; ssize_t len; npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; ASSERT_NE(npages, 0); size = npages << self->page_shift; fd = hmm_create_file(size); ASSERT_GE(fd, 0); buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); buffer->fd = fd; buffer->size = size; buffer->mirror = malloc(size); ASSERT_NE(buffer->mirror, NULL); buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, buffer->fd, 0); ASSERT_NE(buffer->ptr, MAP_FAILED); /* Initialize data that the device will write to buffer->ptr. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ptr[i] = i; /* Simulate a device writing system memory. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); ASSERT_EQ(buffer->faults, 1); /* Check what the device wrote. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); /* Check that the device also wrote the file. */ len = pread(fd, buffer->mirror, size, 0); ASSERT_EQ(len, size); for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); hmm_buffer_free(buffer); } /* * Migrate anonymous memory to device private memory. */ TEST_F(hmm, migrate) { struct hmm_buffer *buffer; unsigned long npages; unsigned long size; unsigned long i; int *ptr; int ret; npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; ASSERT_NE(npages, 0); size = npages << self->page_shift; buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); buffer->fd = -1; buffer->size = size; buffer->mirror = malloc(size); ASSERT_NE(buffer->mirror, NULL); buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); ASSERT_NE(buffer->ptr, MAP_FAILED); /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; /* Migrate memory to device. */ ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); /* Check what the device read. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); hmm_buffer_free(buffer); } /* * Migrate anonymous memory to device private memory and fault some of it back * to system memory, then try migrating the resulting mix of system and device * private memory to the device. */ TEST_F(hmm, migrate_fault) { struct hmm_buffer *buffer; unsigned long npages; unsigned long size; unsigned long i; int *ptr; int ret; npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; ASSERT_NE(npages, 0); size = npages << self->page_shift; buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); buffer->fd = -1; buffer->size = size; buffer->mirror = malloc(size); ASSERT_NE(buffer->mirror, NULL); buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); ASSERT_NE(buffer->ptr, MAP_FAILED); /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; /* Migrate memory to device. */ ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); /* Check what the device read. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); /* Fault half the pages back to system memory and check them. */ for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i) ASSERT_EQ(ptr[i], i); /* Migrate memory to the device again. */ ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); /* Check what the device read. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); hmm_buffer_free(buffer); } TEST_F(hmm, migrate_release) { struct hmm_buffer *buffer; unsigned long npages; unsigned long size; unsigned long i; int *ptr; int ret; npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; ASSERT_NE(npages, 0); size = npages << self->page_shift; buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); buffer->fd = -1; buffer->size = size; buffer->mirror = malloc(size); ASSERT_NE(buffer->mirror, NULL); buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); ASSERT_NE(buffer->ptr, MAP_FAILED); /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; /* Migrate memory to device. */ ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); /* Check what the device read. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); /* Release device memory. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_RELEASE, buffer, npages); ASSERT_EQ(ret, 0); /* Fault pages back to system memory and check them. */ for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i) ASSERT_EQ(ptr[i], i); hmm_buffer_free(buffer); } /* * Migrate anonymous shared memory to device private memory. */ TEST_F(hmm, migrate_shared) { struct hmm_buffer *buffer; unsigned long npages; unsigned long size; int ret; npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; ASSERT_NE(npages, 0); size = npages << self->page_shift; buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); buffer->fd = -1; buffer->size = size; buffer->mirror = malloc(size); ASSERT_NE(buffer->mirror, NULL); buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, buffer->fd, 0); ASSERT_NE(buffer->ptr, MAP_FAILED); /* Migrate memory to device. */ ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); ASSERT_EQ(ret, -ENOENT); hmm_buffer_free(buffer); } /* * Try to migrate various memory types to device private memory. */ TEST_F(hmm2, migrate_mixed) { struct hmm_buffer *buffer; unsigned long npages; unsigned long size; int *ptr; unsigned char *p; int ret; int val; npages = 6; size = npages << self->page_shift; buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); buffer->fd = -1; buffer->size = size; buffer->mirror = malloc(size); ASSERT_NE(buffer->mirror, NULL); /* Reserve a range of addresses. */ buffer->ptr = mmap(NULL, size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); ASSERT_NE(buffer->ptr, MAP_FAILED); p = buffer->ptr; /* Migrating a protected area should be an error. */ ret = hmm_migrate_sys_to_dev(self->fd1, buffer, npages); ASSERT_EQ(ret, -EINVAL); /* Punch a hole after the first page address. */ ret = munmap(buffer->ptr + self->page_size, self->page_size); ASSERT_EQ(ret, 0); /* We expect an error if the vma doesn't cover the range. */ ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 3); ASSERT_EQ(ret, -EINVAL); /* Page 2 will be a read-only zero page. */ ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size, PROT_READ); ASSERT_EQ(ret, 0); ptr = (int *)(buffer->ptr + 2 * self->page_size); val = *ptr + 3; ASSERT_EQ(val, 3); /* Page 3 will be read-only. */ ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, PROT_READ | PROT_WRITE); ASSERT_EQ(ret, 0); ptr = (int *)(buffer->ptr + 3 * self->page_size); *ptr = val; ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, PROT_READ); ASSERT_EQ(ret, 0); /* Page 4-5 will be read-write. */ ret = mprotect(buffer->ptr + 4 * self->page_size, 2 * self->page_size, PROT_READ | PROT_WRITE); ASSERT_EQ(ret, 0); ptr = (int *)(buffer->ptr + 4 * self->page_size); *ptr = val; ptr = (int *)(buffer->ptr + 5 * self->page_size); *ptr = val; /* Now try to migrate pages 2-5 to device 1. */ buffer->ptr = p + 2 * self->page_size; ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 4); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, 4); /* Page 5 won't be migrated to device 0 because it's on device 1. */ buffer->ptr = p + 5 * self->page_size; ret = hmm_migrate_sys_to_dev(self->fd0, buffer, 1); ASSERT_EQ(ret, -ENOENT); buffer->ptr = p; buffer->ptr = p; hmm_buffer_free(buffer); } /* * Migrate anonymous memory to device memory and back to system memory * multiple times. In case of private zone configuration, this is done * through fault pages accessed by CPU. In case of coherent zone configuration, * the pages from the device should be explicitly migrated back to system memory. * The reason is Coherent device zone has coherent access by CPU, therefore * it will not generate any page fault. */ TEST_F(hmm, migrate_multiple) { struct hmm_buffer *buffer; unsigned long npages; unsigned long size; unsigned long i; unsigned long c; int *ptr; int ret; npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; ASSERT_NE(npages, 0); size = npages << self->page_shift; for (c = 0; c < NTIMES; c++) { buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); buffer->fd = -1; buffer->size = size; buffer->mirror = malloc(size); ASSERT_NE(buffer->mirror, NULL); buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); ASSERT_NE(buffer->ptr, MAP_FAILED); /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; /* Migrate memory to device. */ ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); /* Check what the device read. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); /* Migrate back to system memory and check them. */ if (hmm_is_coherent_type(variant->device_number)) { ret = hmm_migrate_dev_to_sys(self->fd, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); } for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); hmm_buffer_free(buffer); } } /* * Read anonymous memory multiple times. */ TEST_F(hmm, anon_read_multiple) { struct hmm_buffer *buffer; unsigned long npages; unsigned long size; unsigned long i; unsigned long c; int *ptr; int ret; npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; ASSERT_NE(npages, 0); size = npages << self->page_shift; for (c = 0; c < NTIMES; c++) { buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); buffer->fd = -1; buffer->size = size; buffer->mirror = malloc(size); ASSERT_NE(buffer->mirror, NULL); buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); ASSERT_NE(buffer->ptr, MAP_FAILED); /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i + c; /* Simulate a device reading system memory. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); ASSERT_EQ(buffer->faults, 1); /* Check what the device read. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i + c); hmm_buffer_free(buffer); } } void *unmap_buffer(void *p) { struct hmm_buffer *buffer = p; /* Delay for a bit and then unmap buffer while it is being read. */ hmm_nanosleep(hmm_random() % 32000); munmap(buffer->ptr + buffer->size / 2, buffer->size / 2); buffer->ptr = NULL; return NULL; } /* * Try reading anonymous memory while it is being unmapped. */ TEST_F(hmm, anon_teardown) { unsigned long npages; unsigned long size; unsigned long c; void *ret; npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; ASSERT_NE(npages, 0); size = npages << self->page_shift; for (c = 0; c < NTIMES; ++c) { pthread_t thread; struct hmm_buffer *buffer; unsigned long i; int *ptr; int rc; buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); buffer->fd = -1; buffer->size = size; buffer->mirror = malloc(size); ASSERT_NE(buffer->mirror, NULL); buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); ASSERT_NE(buffer->ptr, MAP_FAILED); /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i + c; rc = pthread_create(&thread, NULL, unmap_buffer, buffer); ASSERT_EQ(rc, 0); /* Simulate a device reading system memory. */ rc = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); if (rc == 0) { ASSERT_EQ(buffer->cpages, npages); ASSERT_EQ(buffer->faults, 1); /* Check what the device read. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i + c); } pthread_join(thread, &ret); hmm_buffer_free(buffer); } } /* * Test memory snapshot without faulting in pages accessed by the device. */ TEST_F(hmm, mixedmap) { struct hmm_buffer *buffer; unsigned long npages; unsigned long size; unsigned char *m; int ret; npages = 1; size = npages << self->page_shift; buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); buffer->fd = -1; buffer->size = size; buffer->mirror = malloc(npages); ASSERT_NE(buffer->mirror, NULL); /* Reserve a range of addresses. */ buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE, self->fd, 0); ASSERT_NE(buffer->ptr, MAP_FAILED); /* Simulate a device snapshotting CPU pagetables. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); /* Check what the device saw. */ m = buffer->mirror; ASSERT_EQ(m[0], HMM_DMIRROR_PROT_READ); hmm_buffer_free(buffer); } /* * Test memory snapshot without faulting in pages accessed by the device. */ TEST_F(hmm2, snapshot) { struct hmm_buffer *buffer; unsigned long npages; unsigned long size; int *ptr; unsigned char *p; unsigned char *m; int ret; int val; npages = 7; size = npages << self->page_shift; buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); buffer->fd = -1; buffer->size = size; buffer->mirror = malloc(npages); ASSERT_NE(buffer->mirror, NULL); /* Reserve a range of addresses. */ buffer->ptr = mmap(NULL, size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); ASSERT_NE(buffer->ptr, MAP_FAILED); p = buffer->ptr; /* Punch a hole after the first page address. */ ret = munmap(buffer->ptr + self->page_size, self->page_size); ASSERT_EQ(ret, 0); /* Page 2 will be read-only zero page. */ ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size, PROT_READ); ASSERT_EQ(ret, 0); ptr = (int *)(buffer->ptr + 2 * self->page_size); val = *ptr + 3; ASSERT_EQ(val, 3); /* Page 3 will be read-only. */ ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, PROT_READ | PROT_WRITE); ASSERT_EQ(ret, 0); ptr = (int *)(buffer->ptr + 3 * self->page_size); *ptr = val; ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, PROT_READ); ASSERT_EQ(ret, 0); /* Page 4-6 will be read-write. */ ret = mprotect(buffer->ptr + 4 * self->page_size, 3 * self->page_size, PROT_READ | PROT_WRITE); ASSERT_EQ(ret, 0); ptr = (int *)(buffer->ptr + 4 * self->page_size); *ptr = val; /* Page 5 will be migrated to device 0. */ buffer->ptr = p + 5 * self->page_size; ret = hmm_migrate_sys_to_dev(self->fd0, buffer, 1); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, 1); /* Page 6 will be migrated to device 1. */ buffer->ptr = p + 6 * self->page_size; ret = hmm_migrate_sys_to_dev(self->fd1, buffer, 1); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, 1); /* Simulate a device snapshotting CPU pagetables. */ buffer->ptr = p; ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_SNAPSHOT, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); /* Check what the device saw. */ m = buffer->mirror; ASSERT_EQ(m[0], HMM_DMIRROR_PROT_ERROR); ASSERT_EQ(m[1], HMM_DMIRROR_PROT_ERROR); ASSERT_EQ(m[2], HMM_DMIRROR_PROT_ZERO | HMM_DMIRROR_PROT_READ); ASSERT_EQ(m[3], HMM_DMIRROR_PROT_READ); ASSERT_EQ(m[4], HMM_DMIRROR_PROT_WRITE); if (!hmm_is_coherent_type(variant->device_number0)) { ASSERT_EQ(m[5], HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL | HMM_DMIRROR_PROT_WRITE); ASSERT_EQ(m[6], HMM_DMIRROR_PROT_NONE); } else { ASSERT_EQ(m[5], HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | HMM_DMIRROR_PROT_WRITE); ASSERT_EQ(m[6], HMM_DMIRROR_PROT_DEV_COHERENT_REMOTE | HMM_DMIRROR_PROT_WRITE); } hmm_buffer_free(buffer); } /* * Test the hmm_range_fault() HMM_PFN_PMD flag for large pages that * should be mapped by a large page table entry. */ TEST_F(hmm, compound) { struct hmm_buffer *buffer; unsigned long npages; unsigned long size; unsigned long default_hsize; int *ptr; unsigned char *m; int ret; unsigned long i; /* Skip test if we can't allocate a hugetlbfs page. */ default_hsize = file_read_ulong("/proc/meminfo", "Hugepagesize:"); if (default_hsize < 0 || default_hsize*1024 < default_hsize) SKIP(return, "Huge page size could not be determined"); default_hsize = default_hsize*1024; /* KB to B */ size = ALIGN(TWOMEG, default_hsize); npages = size >> self->page_shift; buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); if (buffer->ptr == MAP_FAILED) { free(buffer); return; } buffer->size = size; buffer->mirror = malloc(npages); ASSERT_NE(buffer->mirror, NULL); /* Initialize the pages the device will snapshot in buffer->ptr. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; /* Simulate a device snapshotting CPU pagetables. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); /* Check what the device saw. */ m = buffer->mirror; for (i = 0; i < npages; ++i) ASSERT_EQ(m[i], HMM_DMIRROR_PROT_WRITE | HMM_DMIRROR_PROT_PMD); /* Make the region read-only. */ ret = mprotect(buffer->ptr, size, PROT_READ); ASSERT_EQ(ret, 0); /* Simulate a device snapshotting CPU pagetables. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); /* Check what the device saw. */ m = buffer->mirror; for (i = 0; i < npages; ++i) ASSERT_EQ(m[i], HMM_DMIRROR_PROT_READ | HMM_DMIRROR_PROT_PMD); munmap(buffer->ptr, buffer->size); buffer->ptr = NULL; hmm_buffer_free(buffer); } /* * Test two devices reading the same memory (double mapped). */ TEST_F(hmm2, double_map) { struct hmm_buffer *buffer; unsigned long npages; unsigned long size; unsigned long i; int *ptr; int ret; npages = 6; size = npages << self->page_shift; buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); buffer->fd = -1; buffer->size = size; buffer->mirror = malloc(size); ASSERT_NE(buffer->mirror, NULL); /* Reserve a range of addresses. */ buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); ASSERT_NE(buffer->ptr, MAP_FAILED); /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; /* Make region read-only. */ ret = mprotect(buffer->ptr, size, PROT_READ); ASSERT_EQ(ret, 0); /* Simulate device 0 reading system memory. */ ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); ASSERT_EQ(buffer->faults, 1); /* Check what the device read. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); /* Simulate device 1 reading system memory. */ ret = hmm_dmirror_cmd(self->fd1, HMM_DMIRROR_READ, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); ASSERT_EQ(buffer->faults, 1); /* Check what the device read. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); /* Migrate pages to device 1 and try to read from device 0. */ ret = hmm_migrate_sys_to_dev(self->fd1, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); ASSERT_EQ(buffer->faults, 1); /* Check what device 0 read. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); hmm_buffer_free(buffer); } /* * Basic check of exclusive faulting. */ TEST_F(hmm, exclusive) { struct hmm_buffer *buffer; unsigned long npages; unsigned long size; unsigned long i; int *ptr; int ret; npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; ASSERT_NE(npages, 0); size = npages << self->page_shift; buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); buffer->fd = -1; buffer->size = size; buffer->mirror = malloc(size); ASSERT_NE(buffer->mirror, NULL); buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); ASSERT_NE(buffer->ptr, MAP_FAILED); /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; /* Map memory exclusively for device access. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); /* Check what the device read. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); /* Fault pages back to system memory and check them. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i]++, i); for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i+1); /* Check atomic access revoked */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_CHECK_EXCLUSIVE, buffer, npages); ASSERT_EQ(ret, 0); hmm_buffer_free(buffer); } TEST_F(hmm, exclusive_mprotect) { struct hmm_buffer *buffer; unsigned long npages; unsigned long size; unsigned long i; int *ptr; int ret; npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; ASSERT_NE(npages, 0); size = npages << self->page_shift; buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); buffer->fd = -1; buffer->size = size; buffer->mirror = malloc(size); ASSERT_NE(buffer->mirror, NULL); buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); ASSERT_NE(buffer->ptr, MAP_FAILED); /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; /* Map memory exclusively for device access. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); /* Check what the device read. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); ret = mprotect(buffer->ptr, size, PROT_READ); ASSERT_EQ(ret, 0); /* Simulate a device writing system memory. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); ASSERT_EQ(ret, -EPERM); hmm_buffer_free(buffer); } /* * Check copy-on-write works. */ TEST_F(hmm, exclusive_cow) { struct hmm_buffer *buffer; unsigned long npages; unsigned long size; unsigned long i; int *ptr; int ret; npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; ASSERT_NE(npages, 0); size = npages << self->page_shift; buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); buffer->fd = -1; buffer->size = size; buffer->mirror = malloc(size); ASSERT_NE(buffer->mirror, NULL); buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); ASSERT_NE(buffer->ptr, MAP_FAILED); /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; /* Map memory exclusively for device access. */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); fork(); /* Fault pages back to system memory and check them. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i]++, i); for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i+1); hmm_buffer_free(buffer); } static int gup_test_exec(int gup_fd, unsigned long addr, int cmd, int npages, int size, int flags) { struct gup_test gup = { .nr_pages_per_call = npages, .addr = addr, .gup_flags = FOLL_WRITE | flags, .size = size, }; if (ioctl(gup_fd, cmd, &gup)) { perror("ioctl on error\n"); return errno; } return 0; } /* * Test get user device pages through gup_test. Setting PIN_LONGTERM flag. * This should trigger a migration back to system memory for both, private * and coherent type pages. * This test makes use of gup_test module. Make sure GUP_TEST_CONFIG is added * to your configuration before you run it. */ TEST_F(hmm, hmm_gup_test) { struct hmm_buffer *buffer; int gup_fd; unsigned long npages; unsigned long size; unsigned long i; int *ptr; int ret; unsigned char *m; gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR); if (gup_fd == -1) SKIP(return, "Skipping test, could not find gup_test driver"); npages = 4; size = npages << self->page_shift; buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); buffer->fd = -1; buffer->size = size; buffer->mirror = malloc(size); ASSERT_NE(buffer->mirror, NULL); buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); ASSERT_NE(buffer->ptr, MAP_FAILED); /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; /* Migrate memory to device. */ ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); /* Check what the device read. */ for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); ASSERT_EQ(gup_test_exec(gup_fd, (unsigned long)buffer->ptr, GUP_BASIC_TEST, 1, self->page_size, 0), 0); ASSERT_EQ(gup_test_exec(gup_fd, (unsigned long)buffer->ptr + 1 * self->page_size, GUP_FAST_BENCHMARK, 1, self->page_size, 0), 0); ASSERT_EQ(gup_test_exec(gup_fd, (unsigned long)buffer->ptr + 2 * self->page_size, PIN_FAST_BENCHMARK, 1, self->page_size, FOLL_LONGTERM), 0); ASSERT_EQ(gup_test_exec(gup_fd, (unsigned long)buffer->ptr + 3 * self->page_size, PIN_LONGTERM_BENCHMARK, 1, self->page_size, 0), 0); /* Take snapshot to CPU pagetables */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); m = buffer->mirror; if (hmm_is_coherent_type(variant->device_number)) { ASSERT_EQ(HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | HMM_DMIRROR_PROT_WRITE, m[0]); ASSERT_EQ(HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | HMM_DMIRROR_PROT_WRITE, m[1]); } else { ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[0]); ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[1]); } ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[2]); ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[3]); /* * Check again the content on the pages. Make sure there's no * corrupted data. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ASSERT_EQ(ptr[i], i); close(gup_fd); hmm_buffer_free(buffer); } /* * Test copy-on-write in device pages. * In case of writing to COW private page(s), a page fault will migrate pages * back to system memory first. Then, these pages will be duplicated. In case * of COW device coherent type, pages are duplicated directly from device * memory. */ TEST_F(hmm, hmm_cow_in_device) { struct hmm_buffer *buffer; unsigned long npages; unsigned long size; unsigned long i; int *ptr; int ret; unsigned char *m; pid_t pid; int status; npages = 4; size = npages << self->page_shift; buffer = malloc(sizeof(*buffer)); ASSERT_NE(buffer, NULL); buffer->fd = -1; buffer->size = size; buffer->mirror = malloc(size); ASSERT_NE(buffer->mirror, NULL); buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0); ASSERT_NE(buffer->ptr, MAP_FAILED); /* Initialize buffer in system memory. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; /* Migrate memory to device. */ ret = hmm_migrate_sys_to_dev(self->fd, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); pid = fork(); if (pid == -1) ASSERT_EQ(pid, 0); if (!pid) { /* Child process waitd for SIGTERM from the parent. */ while (1) { } perror("Should not reach this\n"); exit(0); } /* Parent process writes to COW pages(s) and gets a * new copy in system. In case of device private pages, * this write causes a migration to system mem first. */ for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) ptr[i] = i; /* Terminate child and wait */ EXPECT_EQ(0, kill(pid, SIGTERM)); EXPECT_EQ(pid, waitpid(pid, &status, 0)); EXPECT_NE(0, WIFSIGNALED(status)); EXPECT_EQ(SIGTERM, WTERMSIG(status)); /* Take snapshot to CPU pagetables */ ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); ASSERT_EQ(ret, 0); ASSERT_EQ(buffer->cpages, npages); m = buffer->mirror; for (i = 0; i < npages; i++) ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[i]); hmm_buffer_free(buffer); } TEST_HARNESS_MAIN
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1