| Author | Tokens | Token Proportion | Commits | Commit Proportion |
|---|---|---|---|---|
| Mathieu Desnoyers | 3687 | 98.42% | 7 | 77.78% |
| Peter Oskolkov | 51 | 1.36% | 1 | 11.11% |
| Vasily Gorbik | 8 | 0.21% | 1 | 11.11% |
| Total | 3746 | 9 |
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993
/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ /* * rseq-x86-bits.h * * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */ #include "rseq-bits-template.h" #ifdef __x86_64__ #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) static inline __attribute__((always_inline)) int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) { RSEQ_INJECT_C(9) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) RSEQ_INJECT_ASM(3) "cmpq %[v], %[expect]\n\t" "jnz %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) "cmpq %[v], %[expect]\n\t" "jnz %l[error2]\n\t" #endif /* final store */ "movq %[newv], %[v]\n\t" "2:\n\t" RSEQ_INJECT_ASM(5) RSEQ_ASM_DEFINE_ABORT(4, "", abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [rseq_offset] "r" (rseq_offset), [v] "m" (*v), [expect] "r" (expect), [newv] "r" (newv) : "memory", "cc", "rax" RSEQ_INJECT_CLOBBER : abort, cmpfail #ifdef RSEQ_COMPARE_TWICE , error1, error2 #endif ); rseq_after_asm_goto(); return 0; abort: rseq_after_asm_goto(); RSEQ_INJECT_FAILED return -1; cmpfail: rseq_after_asm_goto(); return 1; #ifdef RSEQ_COMPARE_TWICE error1: rseq_after_asm_goto(); rseq_bug("cpu_id comparison failed"); error2: rseq_after_asm_goto(); rseq_bug("expected value comparison failed"); #endif } /* * Compare @v against @expectnot. When it does _not_ match, load @v * into @load, and store the content of *@v + voffp into @v. */ static inline __attribute__((always_inline)) int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, long voffp, intptr_t *load, int cpu) { RSEQ_INJECT_C(9) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) RSEQ_INJECT_ASM(3) "movq %[v], %%rbx\n\t" "cmpq %%rbx, %[expectnot]\n\t" "je %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) "movq %[v], %%rbx\n\t" "cmpq %%rbx, %[expectnot]\n\t" "je %l[error2]\n\t" #endif "movq %%rbx, %[load]\n\t" "addq %[voffp], %%rbx\n\t" "movq (%%rbx), %%rbx\n\t" /* final store */ "movq %%rbx, %[v]\n\t" "2:\n\t" RSEQ_INJECT_ASM(5) RSEQ_ASM_DEFINE_ABORT(4, "", abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [rseq_offset] "r" (rseq_offset), /* final store input */ [v] "m" (*v), [expectnot] "r" (expectnot), [voffp] "er" (voffp), [load] "m" (*load) : "memory", "cc", "rax", "rbx" RSEQ_INJECT_CLOBBER : abort, cmpfail #ifdef RSEQ_COMPARE_TWICE , error1, error2 #endif ); rseq_after_asm_goto(); return 0; abort: rseq_after_asm_goto(); RSEQ_INJECT_FAILED return -1; cmpfail: rseq_after_asm_goto(); return 1; #ifdef RSEQ_COMPARE_TWICE error1: rseq_after_asm_goto(); rseq_bug("cpu_id comparison failed"); error2: rseq_after_asm_goto(); rseq_bug("expected value comparison failed"); #endif } static inline __attribute__((always_inline)) int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) { RSEQ_INJECT_C(9) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) RSEQ_INJECT_ASM(3) #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) #endif /* final store */ "addq %[count], %[v]\n\t" "2:\n\t" RSEQ_INJECT_ASM(4) RSEQ_ASM_DEFINE_ABORT(4, "", abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [rseq_offset] "r" (rseq_offset), /* final store input */ [v] "m" (*v), [count] "er" (count) : "memory", "cc", "rax" RSEQ_INJECT_CLOBBER : abort #ifdef RSEQ_COMPARE_TWICE , error1 #endif ); rseq_after_asm_goto(); return 0; abort: rseq_after_asm_goto(); RSEQ_INJECT_FAILED return -1; #ifdef RSEQ_COMPARE_TWICE error1: rseq_after_asm_goto(); rseq_bug("cpu_id comparison failed"); #endif } #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV /* * pval = *(ptr+off) * *pval += inc; */ static inline __attribute__((always_inline)) int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, long off, intptr_t inc, int cpu) { RSEQ_INJECT_C(9) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) RSEQ_INJECT_ASM(3) #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) #endif /* get p+v */ "movq %[ptr], %%rbx\n\t" "addq %[off], %%rbx\n\t" /* get pv */ "movq (%%rbx), %%rcx\n\t" /* *pv += inc */ "addq %[inc], (%%rcx)\n\t" "2:\n\t" RSEQ_INJECT_ASM(4) RSEQ_ASM_DEFINE_ABORT(4, "", abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [rseq_offset] "r" (rseq_offset), /* final store input */ [ptr] "m" (*ptr), [off] "er" (off), [inc] "er" (inc) : "memory", "cc", "rax", "rbx", "rcx" RSEQ_INJECT_CLOBBER : abort #ifdef RSEQ_COMPARE_TWICE , error1 #endif ); return 0; abort: RSEQ_INJECT_FAILED return -1; #ifdef RSEQ_COMPARE_TWICE error1: rseq_bug("cpu_id comparison failed"); #endif } static inline __attribute__((always_inline)) int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t *v2, intptr_t expect2, intptr_t newv, int cpu) { RSEQ_INJECT_C(9) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) RSEQ_INJECT_ASM(3) "cmpq %[v], %[expect]\n\t" "jnz %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) "cmpq %[v2], %[expect2]\n\t" "jnz %l[cmpfail]\n\t" RSEQ_INJECT_ASM(5) #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) "cmpq %[v], %[expect]\n\t" "jnz %l[error2]\n\t" "cmpq %[v2], %[expect2]\n\t" "jnz %l[error3]\n\t" #endif /* final store */ "movq %[newv], %[v]\n\t" "2:\n\t" RSEQ_INJECT_ASM(6) RSEQ_ASM_DEFINE_ABORT(4, "", abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [rseq_offset] "r" (rseq_offset), /* cmp2 input */ [v2] "m" (*v2), [expect2] "r" (expect2), /* final store input */ [v] "m" (*v), [expect] "r" (expect), [newv] "r" (newv) : "memory", "cc", "rax" RSEQ_INJECT_CLOBBER : abort, cmpfail #ifdef RSEQ_COMPARE_TWICE , error1, error2, error3 #endif ); rseq_after_asm_goto(); return 0; abort: rseq_after_asm_goto(); RSEQ_INJECT_FAILED return -1; cmpfail: rseq_after_asm_goto(); return 1; #ifdef RSEQ_COMPARE_TWICE error1: rseq_after_asm_goto(); rseq_bug("cpu_id comparison failed"); error2: rseq_after_asm_goto(); rseq_bug("1st expected value comparison failed"); error3: rseq_after_asm_goto(); rseq_bug("2nd expected value comparison failed"); #endif } #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) static inline __attribute__((always_inline)) int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, intptr_t *v2, intptr_t newv2, intptr_t newv, int cpu) { RSEQ_INJECT_C(9) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) RSEQ_INJECT_ASM(3) "cmpq %[v], %[expect]\n\t" "jnz %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) "cmpq %[v], %[expect]\n\t" "jnz %l[error2]\n\t" #endif /* try store */ "movq %[newv2], %[v2]\n\t" RSEQ_INJECT_ASM(5) /* final store */ "movq %[newv], %[v]\n\t" "2:\n\t" RSEQ_INJECT_ASM(6) RSEQ_ASM_DEFINE_ABORT(4, "", abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [rseq_offset] "r" (rseq_offset), /* try store input */ [v2] "m" (*v2), [newv2] "r" (newv2), /* final store input */ [v] "m" (*v), [expect] "r" (expect), [newv] "r" (newv) : "memory", "cc", "rax" RSEQ_INJECT_CLOBBER : abort, cmpfail #ifdef RSEQ_COMPARE_TWICE , error1, error2 #endif ); rseq_after_asm_goto(); return 0; abort: rseq_after_asm_goto(); RSEQ_INJECT_FAILED return -1; cmpfail: rseq_after_asm_goto(); return 1; #ifdef RSEQ_COMPARE_TWICE error1: rseq_after_asm_goto(); rseq_bug("cpu_id comparison failed"); error2: rseq_after_asm_goto(); rseq_bug("expected value comparison failed"); #endif } static inline __attribute__((always_inline)) int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, void *dst, void *src, size_t len, intptr_t newv, int cpu) { uint64_t rseq_scratch[3]; RSEQ_INJECT_C(9) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) #endif "movq %[src], %[rseq_scratch0]\n\t" "movq %[dst], %[rseq_scratch1]\n\t" "movq %[len], %[rseq_scratch2]\n\t" /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) RSEQ_INJECT_ASM(3) "cmpq %[v], %[expect]\n\t" "jnz 5f\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 6f) "cmpq %[v], %[expect]\n\t" "jnz 7f\n\t" #endif /* try memcpy */ "test %[len], %[len]\n\t" \ "jz 333f\n\t" \ "222:\n\t" \ "movb (%[src]), %%al\n\t" \ "movb %%al, (%[dst])\n\t" \ "inc %[src]\n\t" \ "inc %[dst]\n\t" \ "dec %[len]\n\t" \ "jnz 222b\n\t" \ "333:\n\t" \ RSEQ_INJECT_ASM(5) /* final store */ "movq %[newv], %[v]\n\t" "2:\n\t" RSEQ_INJECT_ASM(6) /* teardown */ "movq %[rseq_scratch2], %[len]\n\t" "movq %[rseq_scratch1], %[dst]\n\t" "movq %[rseq_scratch0], %[src]\n\t" RSEQ_ASM_DEFINE_ABORT(4, "movq %[rseq_scratch2], %[len]\n\t" "movq %[rseq_scratch1], %[dst]\n\t" "movq %[rseq_scratch0], %[src]\n\t", abort) RSEQ_ASM_DEFINE_CMPFAIL(5, "movq %[rseq_scratch2], %[len]\n\t" "movq %[rseq_scratch1], %[dst]\n\t" "movq %[rseq_scratch0], %[src]\n\t", cmpfail) #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_DEFINE_CMPFAIL(6, "movq %[rseq_scratch2], %[len]\n\t" "movq %[rseq_scratch1], %[dst]\n\t" "movq %[rseq_scratch0], %[src]\n\t", error1) RSEQ_ASM_DEFINE_CMPFAIL(7, "movq %[rseq_scratch2], %[len]\n\t" "movq %[rseq_scratch1], %[dst]\n\t" "movq %[rseq_scratch0], %[src]\n\t", error2) #endif : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [rseq_offset] "r" (rseq_offset), /* final store input */ [v] "m" (*v), [expect] "r" (expect), [newv] "r" (newv), /* try memcpy input */ [dst] "r" (dst), [src] "r" (src), [len] "r" (len), [rseq_scratch0] "m" (rseq_scratch[0]), [rseq_scratch1] "m" (rseq_scratch[1]), [rseq_scratch2] "m" (rseq_scratch[2]) : "memory", "cc", "rax" RSEQ_INJECT_CLOBBER : abort, cmpfail #ifdef RSEQ_COMPARE_TWICE , error1, error2 #endif ); rseq_after_asm_goto(); return 0; abort: rseq_after_asm_goto(); RSEQ_INJECT_FAILED return -1; cmpfail: rseq_after_asm_goto(); return 1; #ifdef RSEQ_COMPARE_TWICE error1: rseq_after_asm_goto(); rseq_bug("cpu_id comparison failed"); error2: rseq_after_asm_goto(); rseq_bug("expected value comparison failed"); #endif } #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ #elif defined(__i386__) #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) static inline __attribute__((always_inline)) int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) { RSEQ_INJECT_C(9) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) RSEQ_INJECT_ASM(3) "cmpl %[v], %[expect]\n\t" "jnz %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) "cmpl %[v], %[expect]\n\t" "jnz %l[error2]\n\t" #endif /* final store */ "movl %[newv], %[v]\n\t" "2:\n\t" RSEQ_INJECT_ASM(5) RSEQ_ASM_DEFINE_ABORT(4, "", abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [rseq_offset] "r" (rseq_offset), [v] "m" (*v), [expect] "r" (expect), [newv] "r" (newv) : "memory", "cc", "eax" RSEQ_INJECT_CLOBBER : abort, cmpfail #ifdef RSEQ_COMPARE_TWICE , error1, error2 #endif ); rseq_after_asm_goto(); return 0; abort: rseq_after_asm_goto(); RSEQ_INJECT_FAILED return -1; cmpfail: rseq_after_asm_goto(); return 1; #ifdef RSEQ_COMPARE_TWICE error1: rseq_after_asm_goto(); rseq_bug("cpu_id comparison failed"); error2: rseq_after_asm_goto(); rseq_bug("expected value comparison failed"); #endif } /* * Compare @v against @expectnot. When it does _not_ match, load @v * into @load, and store the content of *@v + voffp into @v. */ static inline __attribute__((always_inline)) int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, long voffp, intptr_t *load, int cpu) { RSEQ_INJECT_C(9) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) RSEQ_INJECT_ASM(3) "movl %[v], %%ebx\n\t" "cmpl %%ebx, %[expectnot]\n\t" "je %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) "movl %[v], %%ebx\n\t" "cmpl %%ebx, %[expectnot]\n\t" "je %l[error2]\n\t" #endif "movl %%ebx, %[load]\n\t" "addl %[voffp], %%ebx\n\t" "movl (%%ebx), %%ebx\n\t" /* final store */ "movl %%ebx, %[v]\n\t" "2:\n\t" RSEQ_INJECT_ASM(5) RSEQ_ASM_DEFINE_ABORT(4, "", abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [rseq_offset] "r" (rseq_offset), /* final store input */ [v] "m" (*v), [expectnot] "r" (expectnot), [voffp] "ir" (voffp), [load] "m" (*load) : "memory", "cc", "eax", "ebx" RSEQ_INJECT_CLOBBER : abort, cmpfail #ifdef RSEQ_COMPARE_TWICE , error1, error2 #endif ); rseq_after_asm_goto(); return 0; abort: rseq_after_asm_goto(); RSEQ_INJECT_FAILED return -1; cmpfail: rseq_after_asm_goto(); return 1; #ifdef RSEQ_COMPARE_TWICE error1: rseq_after_asm_goto(); rseq_bug("cpu_id comparison failed"); error2: rseq_after_asm_goto(); rseq_bug("expected value comparison failed"); #endif } static inline __attribute__((always_inline)) int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) { RSEQ_INJECT_C(9) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) RSEQ_INJECT_ASM(3) #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) #endif /* final store */ "addl %[count], %[v]\n\t" "2:\n\t" RSEQ_INJECT_ASM(4) RSEQ_ASM_DEFINE_ABORT(4, "", abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [rseq_offset] "r" (rseq_offset), /* final store input */ [v] "m" (*v), [count] "ir" (count) : "memory", "cc", "eax" RSEQ_INJECT_CLOBBER : abort #ifdef RSEQ_COMPARE_TWICE , error1 #endif ); rseq_after_asm_goto(); return 0; abort: rseq_after_asm_goto(); RSEQ_INJECT_FAILED return -1; #ifdef RSEQ_COMPARE_TWICE error1: rseq_after_asm_goto(); rseq_bug("cpu_id comparison failed"); #endif } static inline __attribute__((always_inline)) int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t *v2, intptr_t expect2, intptr_t newv, int cpu) { RSEQ_INJECT_C(9) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) RSEQ_INJECT_ASM(3) "cmpl %[v], %[expect]\n\t" "jnz %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) "cmpl %[expect2], %[v2]\n\t" "jnz %l[cmpfail]\n\t" RSEQ_INJECT_ASM(5) #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) "cmpl %[v], %[expect]\n\t" "jnz %l[error2]\n\t" "cmpl %[expect2], %[v2]\n\t" "jnz %l[error3]\n\t" #endif "movl %[newv], %%eax\n\t" /* final store */ "movl %%eax, %[v]\n\t" "2:\n\t" RSEQ_INJECT_ASM(6) RSEQ_ASM_DEFINE_ABORT(4, "", abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [rseq_offset] "r" (rseq_offset), /* cmp2 input */ [v2] "m" (*v2), [expect2] "r" (expect2), /* final store input */ [v] "m" (*v), [expect] "r" (expect), [newv] "m" (newv) : "memory", "cc", "eax" RSEQ_INJECT_CLOBBER : abort, cmpfail #ifdef RSEQ_COMPARE_TWICE , error1, error2, error3 #endif ); rseq_after_asm_goto(); return 0; abort: rseq_after_asm_goto(); RSEQ_INJECT_FAILED return -1; cmpfail: rseq_after_asm_goto(); return 1; #ifdef RSEQ_COMPARE_TWICE error1: rseq_after_asm_goto(); rseq_bug("cpu_id comparison failed"); error2: rseq_after_asm_goto(); rseq_bug("1st expected value comparison failed"); error3: rseq_after_asm_goto(); rseq_bug("2nd expected value comparison failed"); #endif } #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) static inline __attribute__((always_inline)) int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, intptr_t *v2, intptr_t newv2, intptr_t newv, int cpu) { RSEQ_INJECT_C(9) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) RSEQ_INJECT_ASM(3) "movl %[expect], %%eax\n\t" "cmpl %[v], %%eax\n\t" "jnz %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) "movl %[expect], %%eax\n\t" "cmpl %[v], %%eax\n\t" "jnz %l[error2]\n\t" #endif /* try store */ "movl %[newv2], %[v2]\n\t" RSEQ_INJECT_ASM(5) #ifdef RSEQ_TEMPLATE_MO_RELEASE "lock; addl $0,-128(%%esp)\n\t" #endif /* final store */ "movl %[newv], %[v]\n\t" "2:\n\t" RSEQ_INJECT_ASM(6) RSEQ_ASM_DEFINE_ABORT(4, "", abort) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [rseq_offset] "r" (rseq_offset), /* try store input */ [v2] "m" (*v2), [newv2] "r" (newv2), /* final store input */ [v] "m" (*v), [expect] "m" (expect), [newv] "r" (newv) : "memory", "cc", "eax" RSEQ_INJECT_CLOBBER : abort, cmpfail #ifdef RSEQ_COMPARE_TWICE , error1, error2 #endif ); rseq_after_asm_goto(); return 0; abort: rseq_after_asm_goto(); RSEQ_INJECT_FAILED return -1; cmpfail: rseq_after_asm_goto(); return 1; #ifdef RSEQ_COMPARE_TWICE error1: rseq_after_asm_goto(); rseq_bug("cpu_id comparison failed"); error2: rseq_after_asm_goto(); rseq_bug("expected value comparison failed"); #endif } /* TODO: implement a faster memcpy. */ static inline __attribute__((always_inline)) int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, void *dst, void *src, size_t len, intptr_t newv, int cpu) { uint32_t rseq_scratch[3]; RSEQ_INJECT_C(9) __asm__ __volatile__ goto ( RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) #endif "movl %[src], %[rseq_scratch0]\n\t" "movl %[dst], %[rseq_scratch1]\n\t" "movl %[len], %[rseq_scratch2]\n\t" /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) RSEQ_INJECT_ASM(3) "movl %[expect], %%eax\n\t" "cmpl %%eax, %[v]\n\t" "jnz 5f\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 6f) "movl %[expect], %%eax\n\t" "cmpl %%eax, %[v]\n\t" "jnz 7f\n\t" #endif /* try memcpy */ "test %[len], %[len]\n\t" \ "jz 333f\n\t" \ "222:\n\t" \ "movb (%[src]), %%al\n\t" \ "movb %%al, (%[dst])\n\t" \ "inc %[src]\n\t" \ "inc %[dst]\n\t" \ "dec %[len]\n\t" \ "jnz 222b\n\t" \ "333:\n\t" \ RSEQ_INJECT_ASM(5) #ifdef RSEQ_TEMPLATE_MO_RELEASE "lock; addl $0,-128(%%esp)\n\t" #endif "movl %[newv], %%eax\n\t" /* final store */ "movl %%eax, %[v]\n\t" "2:\n\t" RSEQ_INJECT_ASM(6) /* teardown */ "movl %[rseq_scratch2], %[len]\n\t" "movl %[rseq_scratch1], %[dst]\n\t" "movl %[rseq_scratch0], %[src]\n\t" RSEQ_ASM_DEFINE_ABORT(4, "movl %[rseq_scratch2], %[len]\n\t" "movl %[rseq_scratch1], %[dst]\n\t" "movl %[rseq_scratch0], %[src]\n\t", abort) RSEQ_ASM_DEFINE_CMPFAIL(5, "movl %[rseq_scratch2], %[len]\n\t" "movl %[rseq_scratch1], %[dst]\n\t" "movl %[rseq_scratch0], %[src]\n\t", cmpfail) #ifdef RSEQ_COMPARE_TWICE RSEQ_ASM_DEFINE_CMPFAIL(6, "movl %[rseq_scratch2], %[len]\n\t" "movl %[rseq_scratch1], %[dst]\n\t" "movl %[rseq_scratch0], %[src]\n\t", error1) RSEQ_ASM_DEFINE_CMPFAIL(7, "movl %[rseq_scratch2], %[len]\n\t" "movl %[rseq_scratch1], %[dst]\n\t" "movl %[rseq_scratch0], %[src]\n\t", error2) #endif : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [rseq_offset] "r" (rseq_offset), /* final store input */ [v] "m" (*v), [expect] "m" (expect), [newv] "m" (newv), /* try memcpy input */ [dst] "r" (dst), [src] "r" (src), [len] "r" (len), [rseq_scratch0] "m" (rseq_scratch[0]), [rseq_scratch1] "m" (rseq_scratch[1]), [rseq_scratch2] "m" (rseq_scratch[2]) : "memory", "cc", "eax" RSEQ_INJECT_CLOBBER : abort, cmpfail #ifdef RSEQ_COMPARE_TWICE , error1, error2 #endif ); rseq_after_asm_goto(); return 0; abort: rseq_after_asm_goto(); RSEQ_INJECT_FAILED return -1; cmpfail: rseq_after_asm_goto(); return 1; #ifdef RSEQ_COMPARE_TWICE error1: rseq_after_asm_goto(); rseq_bug("cpu_id comparison failed"); error2: rseq_after_asm_goto(); rseq_bug("expected value comparison failed"); #endif } #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ #endif #include "rseq-bits-reset.h"
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1