| /* sse.h |
| |
| Streaming SIMD Extenstions (a.k.a. Katmai New Instructions) |
| GCC interface library for IA32. |
| |
| To use this library, simply include this header file |
| and compile with GCC. You MUST have inlining enabled |
| in order for sse_ok() to work; this can be done by |
| simply using -O on the GCC command line. |
| |
| Compiling with -DSSE_TRACE will cause detailed trace |
| output to be sent to stderr for each sse operation. |
| This adds lots of code, and obviously slows execution to |
| a crawl, but can be very useful for debugging. |
| |
| THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY |
| EXPRESS OR IMPLIED WARRANTIES, INCLUDING, WITHOUT |
| LIMITATION, THE IMPLIED WARRANTIES OF MERCHANTABILITY |
| AND FITNESS FOR ANY PARTICULAR PURPOSE. |
| |
| 1999 by R. Fisher |
| Based on libmmx by H. Dietz and R. Fisher |
| |
| Notes: |
| This is still extremely alpha. |
| Because this library depends on an assembler which understands the |
| SSE opcodes, you probably won't be able to use this yet. |
| For now, do not use TRACE versions. These both make use |
| of the MMX registers, not the SSE registers. This will be resolved |
| at a later date. |
| ToDo: |
| Rewrite TRACE macros |
| Major Debugging Work |
| */ |
| |
| #ifndef _SSE_H |
| #define _SSE_H |
| |
| |
| |
| /* The type of an value that fits in an SSE register |
| (note that long long constant values MUST be suffixed |
| by LL and unsigned long long values by ULL, lest |
| they be truncated by the compiler) |
| */ |
| typedef union { |
| float sf[4]; /* Single-precision (32-bit) value */ |
| } __attribute__ ((aligned (16))) sse_t; /* On a 16 byte (128-bit) boundary */ |
| |
| |
| #if 0 |
| /* Function to test if multimedia instructions are supported... |
| */ |
| inline extern int |
| mm_support(void) |
| { |
| /* Returns 1 if MMX instructions are supported, |
| 3 if Cyrix MMX and Extended MMX instructions are supported |
| 5 if AMD MMX and 3DNow! instructions are supported |
| 9 if MMX and SSE instructions are supported |
| 0 if hardware does not support any of these |
| */ |
| register int rval = 0; |
| |
| __asm__ __volatile__ ( |
| /* See if CPUID instruction is supported ... */ |
| /* ... Get copies of EFLAGS into eax and ecx */ |
| "pushf\n\t" |
| "popl %%eax\n\t" |
| "movl %%eax, %%ecx\n\t" |
| |
| /* ... Toggle the ID bit in one copy and store */ |
| /* to the EFLAGS reg */ |
| "xorl $0x200000, %%eax\n\t" |
| "push %%eax\n\t" |
| "popf\n\t" |
| |
| /* ... Get the (hopefully modified) EFLAGS */ |
| "pushf\n\t" |
| "popl %%eax\n\t" |
| |
| /* ... Compare and test result */ |
| "xorl %%eax, %%ecx\n\t" |
| "testl $0x200000, %%ecx\n\t" |
| "jz NotSupported1\n\t" /* CPUID not supported */ |
| |
| |
| /* Get standard CPUID information, and |
| go to a specific vendor section */ |
| "movl $0, %%eax\n\t" |
| "cpuid\n\t" |
| |
| /* Check for Intel */ |
| "cmpl $0x756e6547, %%ebx\n\t" |
| "jne TryAMD\n\t" |
| "cmpl $0x49656e69, %%edx\n\t" |
| "jne TryAMD\n\t" |
| "cmpl $0x6c65746e, %%ecx\n" |
| "jne TryAMD\n\t" |
| "jmp Intel\n\t" |
| |
| /* Check for AMD */ |
| "\nTryAMD:\n\t" |
| "cmpl $0x68747541, %%ebx\n\t" |
| "jne TryCyrix\n\t" |
| "cmpl $0x69746e65, %%edx\n\t" |
| "jne TryCyrix\n\t" |
| "cmpl $0x444d4163, %%ecx\n" |
| "jne TryCyrix\n\t" |
| "jmp AMD\n\t" |
| |
| /* Check for Cyrix */ |
| "\nTryCyrix:\n\t" |
| "cmpl $0x69727943, %%ebx\n\t" |
| "jne NotSupported2\n\t" |
| "cmpl $0x736e4978, %%edx\n\t" |
| "jne NotSupported3\n\t" |
| "cmpl $0x64616574, %%ecx\n\t" |
| "jne NotSupported4\n\t" |
| /* Drop through to Cyrix... */ |
| |
| |
| /* Cyrix Section */ |
| /* See if extended CPUID level 80000001 is supported */ |
| /* The value of CPUID/80000001 for the 6x86MX is undefined |
| according to the Cyrix CPU Detection Guide (Preliminary |
| Rev. 1.01 table 1), so we'll check the value of eax for |
| CPUID/0 to see if standard CPUID level 2 is supported. |
| According to the table, the only CPU which supports level |
| 2 is also the only one which supports extended CPUID levels. |
| */ |
| "cmpl $0x2, %%eax\n\t" |
| "jne MMXtest\n\t" /* Use standard CPUID instead */ |
| |
| /* Extended CPUID supported (in theory), so get extended |
| features */ |
| "movl $0x80000001, %%eax\n\t" |
| "cpuid\n\t" |
| "testl $0x00800000, %%eax\n\t" /* Test for MMX */ |
| "jz NotSupported5\n\t" /* MMX not supported */ |
| "testl $0x01000000, %%eax\n\t" /* Test for Ext'd MMX */ |
| "jnz EMMXSupported\n\t" |
| "movl $1, %0:\n\n\t" /* MMX Supported */ |
| "jmp Return\n\n" |
| "EMMXSupported:\n\t" |
| "movl $3, %0:\n\n\t" /* EMMX and MMX Supported */ |
| "jmp Return\n\t" |
| |
| |
| /* AMD Section */ |
| "AMD:\n\t" |
| |
| /* See if extended CPUID is supported */ |
| "movl $0x80000000, %%eax\n\t" |
| "cpuid\n\t" |
| "cmpl $0x80000000, %%eax\n\t" |
| "jl MMXtest\n\t" /* Use standard CPUID instead */ |
| |
| /* Extended CPUID supported, so get extended features */ |
| "movl $0x80000001, %%eax\n\t" |
| "cpuid\n\t" |
| "testl $0x00800000, %%edx\n\t" /* Test for MMX */ |
| "jz NotSupported6\n\t" /* MMX not supported */ |
| "testl $0x80000000, %%edx\n\t" /* Test for 3DNow! */ |
| "jnz ThreeDNowSupported\n\t" |
| "movl $1, %0:\n\n\t" /* MMX Supported */ |
| "jmp Return\n\n" |
| "ThreeDNowSupported:\n\t" |
| "movl $5, %0:\n\n\t" /* 3DNow! and MMX Supported */ |
| "jmp Return\n\t" |
| |
| |
| /* Intel Section */ |
| "Intel:\n\t" |
| |
| /* Check for SSE */ |
| "SSEtest:\n\t" |
| "movl $1, %%eax\n\t" |
| "cpuid\n\t" |
| "testl $0x02000000, %%edx\n\t" /* Test for SSE */ |
| "jz MMXtest\n\t" /* SSE Not supported */ |
| "movl $9, %0:\n\n\t" /* SSE Supported */ |
| "jmp Return\n\t" |
| |
| /* Check for MMX */ |
| "MMXtest:\n\t" |
| "movl $1, %%eax\n\t" |
| "cpuid\n\t" |
| "testl $0x00800000, %%edx\n\t" /* Test for MMX */ |
| "jz NotSupported7\n\t" /* MMX Not supported */ |
| "movl $1, %0:\n\n\t" /* MMX Supported */ |
| "jmp Return\n\t" |
| |
| /* Nothing supported */ |
| "\nNotSupported1:\n\t" |
| "#movl $101, %0:\n\n\t" |
| "\nNotSupported2:\n\t" |
| "#movl $102, %0:\n\n\t" |
| "\nNotSupported3:\n\t" |
| "#movl $103, %0:\n\n\t" |
| "\nNotSupported4:\n\t" |
| "#movl $104, %0:\n\n\t" |
| "\nNotSupported5:\n\t" |
| "#movl $105, %0:\n\n\t" |
| "\nNotSupported6:\n\t" |
| "#movl $106, %0:\n\n\t" |
| "\nNotSupported7:\n\t" |
| "#movl $107, %0:\n\n\t" |
| "movl $0, %0:\n\n\t" |
| |
| "Return:\n\t" |
| : "=a" (rval) |
| : /* no input */ |
| : "eax", "ebx", "ecx", "edx" |
| ); |
| |
| /* Return */ |
| return(rval); |
| } |
| |
| /* Function to test if sse instructions are supported... |
| */ |
| inline extern int |
| sse_ok(void) |
| { |
| /* Returns 1 if SSE instructions are supported, 0 otherwise */ |
| return ( (mm_support() & 0x8) >> 3 ); |
| } |
| #endif |
| |
| |
| |
| /* Helper functions for the instruction macros that follow... |
| (note that memory-to-register, m2r, instructions are nearly |
| as efficient as register-to-register, r2r, instructions; |
| however, memory-to-memory instructions are really simulated |
| as a convenience, and are only 1/3 as efficient) |
| */ |
| #ifdef SSE_TRACE |
| |
| /* Include the stuff for printing a trace to stderr... |
| */ |
| |
| #include <stdio.h> |
| |
| #define sse_i2r(op, imm, reg) \ |
| { \ |
| sse_t sse_trace; \ |
| sse_trace.uq = (imm); \ |
| fprintf(stderr, #op "_i2r(" #imm "=0x%08x%08x, ", \ |
| sse_trace.d[1], sse_trace.d[0]); \ |
| __asm__ __volatile__ ("movq %%" #reg ", %0" \ |
| : "=X" (sse_trace) \ |
| : /* nothing */ ); \ |
| fprintf(stderr, #reg "=0x%08x%08x) => ", \ |
| sse_trace.d[1], sse_trace.d[0]); \ |
| __asm__ __volatile__ (#op " %0, %%" #reg \ |
| : /* nothing */ \ |
| : "X" (imm)); \ |
| __asm__ __volatile__ ("movq %%" #reg ", %0" \ |
| : "=X" (sse_trace) \ |
| : /* nothing */ ); \ |
| fprintf(stderr, #reg "=0x%08x%08x\n", \ |
| sse_trace.d[1], sse_trace.d[0]); \ |
| } |
| |
| #define sse_m2r(op, mem, reg) \ |
| { \ |
| sse_t sse_trace; \ |
| sse_trace = (mem); \ |
| fprintf(stderr, #op "_m2r(" #mem "=0x%08x%08x, ", \ |
| sse_trace.d[1], sse_trace.d[0]); \ |
| __asm__ __volatile__ ("movq %%" #reg ", %0" \ |
| : "=X" (sse_trace) \ |
| : /* nothing */ ); \ |
| fprintf(stderr, #reg "=0x%08x%08x) => ", \ |
| sse_trace.d[1], sse_trace.d[0]); \ |
| __asm__ __volatile__ (#op " %0, %%" #reg \ |
| : /* nothing */ \ |
| : "X" (mem)); \ |
| __asm__ __volatile__ ("movq %%" #reg ", %0" \ |
| : "=X" (sse_trace) \ |
| : /* nothing */ ); \ |
| fprintf(stderr, #reg "=0x%08x%08x\n", \ |
| sse_trace.d[1], sse_trace.d[0]); \ |
| } |
| |
| #define sse_r2m(op, reg, mem) \ |
| { \ |
| sse_t sse_trace; \ |
| __asm__ __volatile__ ("movq %%" #reg ", %0" \ |
| : "=X" (sse_trace) \ |
| : /* nothing */ ); \ |
| fprintf(stderr, #op "_r2m(" #reg "=0x%08x%08x, ", \ |
| sse_trace.d[1], sse_trace.d[0]); \ |
| sse_trace = (mem); \ |
| fprintf(stderr, #mem "=0x%08x%08x) => ", \ |
| sse_trace.d[1], sse_trace.d[0]); \ |
| __asm__ __volatile__ (#op " %%" #reg ", %0" \ |
| : "=X" (mem) \ |
| : /* nothing */ ); \ |
| sse_trace = (mem); \ |
| fprintf(stderr, #mem "=0x%08x%08x\n", \ |
| sse_trace.d[1], sse_trace.d[0]); \ |
| } |
| |
| #define sse_r2r(op, regs, regd) \ |
| { \ |
| sse_t sse_trace; \ |
| __asm__ __volatile__ ("movq %%" #regs ", %0" \ |
| : "=X" (sse_trace) \ |
| : /* nothing */ ); \ |
| fprintf(stderr, #op "_r2r(" #regs "=0x%08x%08x, ", \ |
| sse_trace.d[1], sse_trace.d[0]); \ |
| __asm__ __volatile__ ("movq %%" #regd ", %0" \ |
| : "=X" (sse_trace) \ |
| : /* nothing */ ); \ |
| fprintf(stderr, #regd "=0x%08x%08x) => ", \ |
| sse_trace.d[1], sse_trace.d[0]); \ |
| __asm__ __volatile__ (#op " %" #regs ", %" #regd); \ |
| __asm__ __volatile__ ("movq %%" #regd ", %0" \ |
| : "=X" (sse_trace) \ |
| : /* nothing */ ); \ |
| fprintf(stderr, #regd "=0x%08x%08x\n", \ |
| sse_trace.d[1], sse_trace.d[0]); \ |
| } |
| |
| #define sse_m2m(op, mems, memd) \ |
| { \ |
| sse_t sse_trace; \ |
| sse_trace = (mems); \ |
| fprintf(stderr, #op "_m2m(" #mems "=0x%08x%08x, ", \ |
| sse_trace.d[1], sse_trace.d[0]); \ |
| sse_trace = (memd); \ |
| fprintf(stderr, #memd "=0x%08x%08x) => ", \ |
| sse_trace.d[1], sse_trace.d[0]); \ |
| __asm__ __volatile__ ("movq %0, %%mm0\n\t" \ |
| #op " %1, %%mm0\n\t" \ |
| "movq %%mm0, %0" \ |
| : "=X" (memd) \ |
| : "X" (mems)); \ |
| sse_trace = (memd); \ |
| fprintf(stderr, #memd "=0x%08x%08x\n", \ |
| sse_trace.d[1], sse_trace.d[0]); \ |
| } |
| |
| #else |
| |
| /* These macros are a lot simpler without the tracing... |
| */ |
| |
| #define sse_i2r(op, imm, reg) \ |
| __asm__ __volatile__ (#op " %0, %%" #reg \ |
| : /* nothing */ \ |
| : "X" (imm) ) |
| |
| #define sse_m2r(op, mem, reg) \ |
| __asm__ __volatile__ (#op " %0, %%" #reg \ |
| : /* nothing */ \ |
| : "X" (mem)) |
| |
| #define sse_r2m(op, reg, mem) \ |
| __asm__ __volatile__ (#op " %%" #reg ", %0" \ |
| : "=X" (mem) \ |
| : /* nothing */ ) |
| |
| #define sse_r2r(op, regs, regd) \ |
| __asm__ __volatile__ (#op " %" #regs ", %" #regd) |
| |
| #define sse_r2ri(op, regs, regd, imm) \ |
| __asm__ __volatile__ (#op " %0, %%" #regs ", %%" #regd \ |
| : /* nothing */ \ |
| : "X" (imm) ) |
| |
| /* Load data from mems to xmmreg, operate on xmmreg, and store data to memd */ |
| #define sse_m2m(op, mems, memd, xmmreg) \ |
| __asm__ __volatile__ ("movups %0, %%xmm0\n\t" \ |
| #op " %1, %%xmm0\n\t" \ |
| "movups %%mm0, %0" \ |
| : "=X" (memd) \ |
| : "X" (mems)) |
| |
| #define sse_m2ri(op, mem, reg, subop) \ |
| __asm__ __volatile__ (#op " %0, %%" #reg ", " #subop \ |
| : /* nothing */ \ |
| : "X" (mem)) |
| |
| #define sse_m2mi(op, mems, memd, xmmreg, subop) \ |
| __asm__ __volatile__ ("movups %0, %%xmm0\n\t" \ |
| #op " %1, %%xmm0, " #subop "\n\t" \ |
| "movups %%mm0, %0" \ |
| : "=X" (memd) \ |
| : "X" (mems)) |
| #endif |
| |
| |
| |
| |
| /* 1x128 MOVe Aligned four Packed Single-fp |
| */ |
| #define movaps_m2r(var, reg) sse_m2r(movaps, var, reg) |
| #define movaps_r2m(reg, var) sse_r2m(movaps, reg, var) |
| #define movaps_r2r(regs, regd) sse_r2r(movaps, regs, regd) |
| #define movaps(vars, vard) \ |
| __asm__ __volatile__ ("movaps %1, %%mm0\n\t" \ |
| "movaps %%mm0, %0" \ |
| : "=X" (vard) \ |
| : "X" (vars)) |
| |
| |
| /* 1x128 MOVe aligned Non-Temporal four Packed Single-fp |
| */ |
| #define movntps_r2m(xmmreg, var) sse_r2m(movntps, xmmreg, var) |
| |
| |
| /* 1x64 MOVe Non-Temporal Quadword |
| */ |
| #define movntq_r2m(mmreg, var) sse_r2m(movntq, mmreg, var) |
| |
| |
| /* 1x128 MOVe Unaligned four Packed Single-fp |
| */ |
| #define movups_m2r(var, reg) sse_m2r(movups, var, reg) |
| #define movups_r2m(reg, var) sse_r2m(movups, reg, var) |
| #define movups_r2r(regs, regd) sse_r2r(movups, regs, regd) |
| #define movups(vars, vard) \ |
| __asm__ __volatile__ ("movups %1, %%mm0\n\t" \ |
| "movups %%mm0, %0" \ |
| : "=X" (vard) \ |
| : "X" (vars)) |
| |
| |
| /* MOVe High to Low Packed Single-fp |
| high half of 4x32f (x) -> low half of 4x32f (y) |
| */ |
| #define movhlps_r2r(regs, regd) sse_r2r(movhlps, regs, regd) |
| |
| |
| /* MOVe Low to High Packed Single-fp |
| low half of 4x32f (x) -> high half of 4x32f (y) |
| */ |
| #define movlhps_r2r(regs, regd) sse_r2r(movlhps, regs, regd) |
| |
| |
| /* MOVe High Packed Single-fp |
| 2x32f -> high half of 4x32f |
| */ |
| #define movhps_m2r(var, reg) sse_m2r(movhps, var, reg) |
| #define movhps_r2m(reg, var) sse_r2m(movhps, reg, var) |
| #define movhps(vars, vard) \ |
| __asm__ __volatile__ ("movhps %1, %%mm0\n\t" \ |
| "movhps %%mm0, %0" \ |
| : "=X" (vard) \ |
| : "X" (vars)) |
| |
| |
| /* MOVe Low Packed Single-fp |
| 2x32f -> low half of 4x32f |
| */ |
| #define movlps_m2r(var, reg) sse_m2r(movlps, var, reg) |
| #define movlps_r2m(reg, var) sse_r2m(movlps, reg, var) |
| #define movlps(vars, vard) \ |
| __asm__ __volatile__ ("movlps %1, %%mm0\n\t" \ |
| "movlps %%mm0, %0" \ |
| : "=X" (vard) \ |
| : "X" (vars)) |
| |
| |
| /* MOVe Scalar Single-fp |
| lowest field of 4x32f (x) -> lowest field of 4x32f (y) |
| */ |
| #define movss_m2r(var, reg) sse_m2r(movss, var, reg) |
| #define movss_r2m(reg, var) sse_r2m(movss, reg, var) |
| #define movss_r2r(regs, regd) sse_r2r(movss, regs, regd) |
| #define movss(vars, vard) \ |
| __asm__ __volatile__ ("movss %1, %%mm0\n\t" \ |
| "movss %%mm0, %0" \ |
| : "=X" (vard) \ |
| : "X" (vars)) |
| |
| |
| /* 4x16 Packed SHUFfle Word |
| */ |
| #define pshufw_m2r(var, reg, index) sse_m2ri(pshufw, var, reg, index) |
| #define pshufw_r2r(regs, regd, index) sse_r2ri(pshufw, regs, regd, index) |
| |
| |
| /* 1x128 SHUFfle Packed Single-fp |
| */ |
| #define shufps_m2r(var, reg, index) sse_m2ri(shufps, var, reg, index) |
| #define shufps_r2r(regs, regd, index) sse_r2ri(shufps, regs, regd, index) |
| |
| |
| /* ConVerT Packed signed Int32 to(2) Packed Single-fp |
| */ |
| #define cvtpi2ps_m2r(var, xmmreg) sse_m2r(cvtpi2ps, var, xmmreg) |
| #define cvtpi2ps_r2r(mmreg, xmmreg) sse_r2r(cvtpi2ps, mmreg, xmmreg) |
| |
| |
| /* ConVerT Packed Single-fp to(2) Packed signed Int32 |
| */ |
| #define cvtps2pi_m2r(var, mmreg) sse_m2r(cvtps2pi, var, mmreg) |
| #define cvtps2pi_r2r(xmmreg, mmreg) sse_r2r(cvtps2pi, mmreg, xmmreg) |
| |
| |
| /* ConVerT with Truncate Packed Single-fp to(2) Packed Int32 |
| */ |
| #define cvttps2pi_m2r(var, mmreg) sse_m2r(cvttps2pi, var, mmreg) |
| #define cvttps2pi_r2r(xmmreg, mmreg) sse_r2r(cvttps2pi, mmreg, xmmreg) |
| |
| |
| /* ConVerT Signed Int32 to(2) Single-fp (Scalar) |
| */ |
| #define cvtsi2ss_m2r(var, xmmreg) sse_m2r(cvtsi2ss, var, xmmreg) |
| #define cvtsi2ss_r2r(reg, xmmreg) sse_r2r(cvtsi2ss, reg, xmmreg) |
| |
| |
| /* ConVerT Scalar Single-fp to(2) Signed Int32 |
| */ |
| #define cvtss2si_m2r(var, reg) sse_m2r(cvtss2si, var, reg) |
| #define cvtss2si_r2r(xmmreg, reg) sse_r2r(cvtss2si, xmmreg, reg) |
| |
| |
| /* ConVerT with Truncate Scalar Single-fp to(2) Signed Int32 |
| */ |
| #define cvttss2si_m2r(var, reg) sse_m2r(cvtss2si, var, reg) |
| #define cvttss2si_r2r(xmmreg, reg) sse_r2r(cvtss2si, xmmreg, reg) |
| |
| |
| /* Parallel EXTRact Word from 4x16 |
| */ |
| #define pextrw_r2r(mmreg, reg, field) sse_r2ri(pextrw, mmreg, reg, field) |
| |
| |
| /* Parallel INSeRt Word from 4x16 |
| */ |
| #define pinsrw_r2r(reg, mmreg, field) sse_r2ri(pinsrw, reg, mmreg, field) |
| |
| |
| |
| /* MOVe MaSK from Packed Single-fp |
| */ |
| #ifdef SSE_TRACE |
| #define movmskps(xmmreg, reg) \ |
| { \ |
| fprintf(stderr, "movmskps()\n"); \ |
| __asm__ __volatile__ ("movmskps %" #xmmreg ", %" #reg) \ |
| } |
| #else |
| #define movmskps(xmmreg, reg) \ |
| __asm__ __volatile__ ("movmskps %" #xmmreg ", %" #reg) |
| #endif |
| |
| |
| /* Parallel MOVe MaSK from mmx reg to 32-bit reg |
| */ |
| #ifdef SSE_TRACE |
| #define pmovmskb(mmreg, reg) \ |
| { \ |
| fprintf(stderr, "movmskps()\n"); \ |
| __asm__ __volatile__ ("movmskps %" #mmreg ", %" #reg) \ |
| } |
| #else |
| #define pmovmskb(mmreg, reg) \ |
| __asm__ __volatile__ ("movmskps %" #mmreg ", %" #reg) |
| #endif |
| |
| |
| /* MASKed MOVe from 8x8 to memory pointed to by (e)di register |
| */ |
| #define maskmovq(mmregs, fieldreg) sse_r2ri(maskmovq, mmregs, fieldreg) |
| |
| |
| |
| |
| /* 4x32f Parallel ADDs |
| */ |
| #define addps_m2r(var, reg) sse_m2r(addps, var, reg) |
| #define addps_r2r(regs, regd) sse_r2r(addps, regs, regd) |
| #define addps(vars, vard, xmmreg) sse_m2m(addps, vars, vard, xmmreg) |
| |
| |
| /* Lowest Field of 4x32f Parallel ADDs |
| */ |
| #define addss_m2r(var, reg) sse_m2r(addss, var, reg) |
| #define addss_r2r(regs, regd) sse_r2r(addss, regs, regd) |
| #define addss(vars, vard, xmmreg) sse_m2m(addss, vars, vard, xmmreg) |
| |
| |
| /* 4x32f Parallel SUBs |
| */ |
| #define subps_m2r(var, reg) sse_m2r(subps, var, reg) |
| #define subps_r2r(regs, regd) sse_r2r(subps, regs, regd) |
| #define subps(vars, vard, xmmreg) sse_m2m(subps, vars, vard, xmmreg) |
| |
| |
| /* Lowest Field of 4x32f Parallel SUBs |
| */ |
| #define subss_m2r(var, reg) sse_m2r(subss, var, reg) |
| #define subss_r2r(regs, regd) sse_r2r(subss, regs, regd) |
| #define subss(vars, vard, xmmreg) sse_m2m(subss, vars, vard, xmmreg) |
| |
| |
| /* 8x8u -> 4x16u Packed Sum of Absolute Differences |
| */ |
| #define psadbw_m2r(var, reg) sse_m2r(psadbw, var, reg) |
| #define psadbw_r2r(regs, regd) sse_r2r(psadbw, regs, regd) |
| #define psadbw(vars, vard, mmreg) sse_m2m(psadbw, vars, vard, mmreg) |
| |
| |
| /* 4x16u Parallel MUL High Unsigned |
| */ |
| #define pmulhuw_m2r(var, reg) sse_m2r(pmulhuw, var, reg) |
| #define pmulhuw_r2r(regs, regd) sse_r2r(pmulhuw, regs, regd) |
| #define pmulhuw(vars, vard, mmreg) sse_m2m(pmulhuw, vars, vard, mmreg) |
| |
| |
| /* 4x32f Parallel MULs |
| */ |
| #define mulps_m2r(var, reg) sse_m2r(mulps, var, reg) |
| #define mulps_r2r(regs, regd) sse_r2r(mulps, regs, regd) |
| #define mulps(vars, vard, xmmreg) sse_m2m(mulps, vars, vard, xmmreg) |
| |
| |
| /* Lowest Field of 4x32f Parallel MULs |
| */ |
| #define mulss_m2r(var, reg) sse_m2r(mulss, var, reg) |
| #define mulss_r2r(regs, regd) sse_r2r(mulss, regs, regd) |
| #define mulss(vars, vard, xmmreg) sse_m2m(mulss, vars, vard, xmmreg) |
| |
| |
| /* 4x32f Parallel DIVs |
| */ |
| #define divps_m2r(var, reg) sse_m2r(divps, var, reg) |
| #define divps_r2r(regs, regd) sse_r2r(divps, regs, regd) |
| #define divps(vars, vard, xmmreg) sse_m2m(divps, vars, vard, xmmreg) |
| |
| |
| /* Lowest Field of 4x32f Parallel DIVs |
| */ |
| #define divss_m2r(var, reg) sse_m2r(divss, var, reg) |
| #define divss_r2r(regs, regd) sse_r2r(divss, regs, regd) |
| #define divss(vars, vard, xmmreg) sse_m2m(divss, vars, vard, xmmreg) |
| |
| |
| /* 4x32f Parallel Reciprocals |
| */ |
| #define rcpps_m2r(var, reg) sse_m2r(rcpps, var, reg) |
| #define rcpps_r2r(regs, regd) sse_r2r(rcpps, regs, regd) |
| #define rcpps(vars, vard, xmmreg) sse_m2m(rcpps, vars, vard, xmmreg) |
| |
| |
| /* Lowest Field of 4x32f Parallel Reciprocals |
| */ |
| #define rcpss_m2r(var, reg) sse_m2r(rcpss, var, reg) |
| #define rcpss_r2r(regs, regd) sse_r2r(rcpss, regs, regd) |
| #define rcpss(vars, vard, xmmreg) sse_m2m(rcpss, vars, vard, xmmreg) |
| |
| |
| /* 4x32f Parallel Square Root of Reciprocals |
| */ |
| #define rsqrtps_m2r(var, reg) sse_m2r(rsqrtps, var, reg) |
| #define rsqrtps_r2r(regs, regd) sse_r2r(rsqrtps, regs, regd) |
| #define rsqrtps(vars, vard, xmmreg) sse_m2m(rsqrtps, vars, vard, xmmreg) |
| |
| |
| /* Lowest Field of 4x32f Parallel Square Root of Reciprocals |
| */ |
| #define rsqrtss_m2r(var, reg) sse_m2r(rsqrtss, var, reg) |
| #define rsqrtss_r2r(regs, regd) sse_r2r(rsqrtss, regs, regd) |
| #define rsqrtss(vars, vard, xmmreg) sse_m2m(rsqrtss, vars, vard, xmmreg) |
| |
| |
| /* 4x32f Parallel Square Roots |
| */ |
| #define sqrtps_m2r(var, reg) sse_m2r(sqrtps, var, reg) |
| #define sqrtps_r2r(regs, regd) sse_r2r(sqrtps, regs, regd) |
| #define sqrtps(vars, vard, xmmreg) sse_m2m(sqrtps, vars, vard, xmmreg) |
| |
| |
| /* Lowest Field of 4x32f Parallel Square Roots |
| */ |
| #define sqrtss_m2r(var, reg) sse_m2r(sqrtss, var, reg) |
| #define sqrtss_r2r(regs, regd) sse_r2r(sqrtss, regs, regd) |
| #define sqrtss(vars, vard, xmmreg) sse_m2m(sqrtss, vars, vard, xmmreg) |
| |
| |
| /* 8x8u and 4x16u Parallel AVeraGe |
| */ |
| #define pavgb_m2r(var, reg) sse_m2r(pavgb, var, reg) |
| #define pavgb_r2r(regs, regd) sse_r2r(pavgb, regs, regd) |
| #define pavgb(vars, vard, mmreg) sse_m2m(pavgb, vars, vard, mmreg) |
| |
| #define pavgw_m2r(var, reg) sse_m2r(pavgw, var, reg) |
| #define pavgw_r2r(regs, regd) sse_r2r(pavgw, regs, regd) |
| #define pavgw(vars, vard, mmreg) sse_m2m(pavgw, vars, vard, mmreg) |
| |
| |
| /* 1x128 bitwise AND |
| */ |
| #define andps_m2r(var, reg) sse_m2r(andps, var, reg) |
| #define andps_r2r(regs, regd) sse_r2r(andps, regs, regd) |
| #define andps(vars, vard, xmmreg) sse_m2m(andps, vars, vard, xmmreg) |
| |
| |
| /* 1x128 bitwise AND with Not the destination |
| */ |
| #define andnps_m2r(var, reg) sse_m2r(andnps, var, reg) |
| #define andnps_r2r(regs, regd) sse_r2r(andnps, regs, regd) |
| #define andnps(vars, vard, xmmreg) sse_m2m(andnps, vars, vard, xmmreg) |
| |
| |
| /* 1x128 bitwise OR |
| */ |
| #define orps_m2r(var, reg) sse_m2r(orps, var, reg) |
| #define orps_r2r(regs, regd) sse_r2r(orps, regs, regd) |
| #define orps(vars, vard, xmmreg) sse_m2m(orps, vars, vard, xmmreg) |
| |
| |
| /* 1x128 bitwise eXclusive OR |
| */ |
| #define xorps_m2r(var, reg) sse_m2r(xorps, var, reg) |
| #define xorps_r2r(regs, regd) sse_r2r(xorps, regs, regd) |
| #define xorps(vars, vard, xmmreg) sse_m2m(xorps, vars, vard, xmmreg) |
| |
| |
| /* 8x8u, 4x16, and 4x32f Parallel Maximum |
| */ |
| #define pmaxub_m2r(var, reg) sse_m2r(pmaxub, var, reg) |
| #define pmaxub_r2r(regs, regd) sse_r2r(pmaxub, regs, regd) |
| #define pmaxub(vars, vard, mmreg) sse_m2m(pmaxub, vars, vard, mmreg) |
| |
| #define pmaxsw_m2r(var, reg) sse_m2r(pmaxsw, var, reg) |
| #define pmaxsw_r2r(regs, regd) sse_r2r(pmaxsw, regs, regd) |
| #define pmaxsw(vars, vard, mmreg) sse_m2m(pmaxsw, vars, vard, mmreg) |
| |
| #define maxps_m2r(var, reg) sse_m2r(maxps, var, reg) |
| #define maxps_r2r(regs, regd) sse_r2r(maxps, regs, regd) |
| #define maxps(vars, vard, xmmreg) sse_m2m(maxps, vars, vard, xmmreg) |
| |
| |
| /* Lowest Field of 4x32f Parallel Maximum |
| */ |
| #define maxss_m2r(var, reg) sse_m2r(maxss, var, reg) |
| #define maxss_r2r(regs, regd) sse_r2r(maxss, regs, regd) |
| #define maxss(vars, vard, xmmreg) sse_m2m(maxss, vars, vard, xmmreg) |
| |
| |
| /* 8x8u, 4x16, and 4x32f Parallel Minimum |
| */ |
| #define pminub_m2r(var, reg) sse_m2r(pminub, var, reg) |
| #define pminub_r2r(regs, regd) sse_r2r(pminub, regs, regd) |
| #define pminub(vars, vard, mmreg) sse_m2m(pminub, vars, vard, mmreg) |
| |
| #define pminsw_m2r(var, reg) sse_m2r(pminsw, var, reg) |
| #define pminsw_r2r(regs, regd) sse_r2r(pminsw, regs, regd) |
| #define pminsw(vars, vard, mmreg) sse_m2m(pminsw, vars, vard, mmreg) |
| |
| #define minps_m2r(var, reg) sse_m2r(minps, var, reg) |
| #define minps_r2r(regs, regd) sse_r2r(minps, regs, regd) |
| #define minps(vars, vard, xmmreg) sse_m2m(minps, vars, vard, xmmreg) |
| |
| |
| /* Lowest Field of 4x32f Parallel Minimum |
| */ |
| #define minss_m2r(var, reg) sse_m2r(minss, var, reg) |
| #define minss_r2r(regs, regd) sse_r2r(minss, regs, regd) |
| #define minss(vars, vard, xmmreg) sse_m2m(minss, vars, vard, xmmreg) |
| |
| |
| /* 4x32f Parallel CoMPares |
| (resulting fields are either 0 or -1) |
| */ |
| #define cmpps_m2r(var, reg, op) sse_m2ri(cmpps, var, reg, op) |
| #define cmpps_r2r(regs, regd, op) sse_r2ri(cmpps, regs, regd, op) |
| #define cmpps(vars, vard, op, xmmreg) sse_m2mi(cmpps, vars, vard, xmmreg, op) |
| |
| #define cmpeqps_m2r(var, reg) sse_m2ri(cmpps, var, reg, 0) |
| #define cmpeqps_r2r(regs, regd) sse_r2ri(cmpps, regs, regd, 0) |
| #define cmpeqps(vars, vard, xmmreg) sse_m2mi(cmpps, vars, vard, xmmreg, 0) |
| |
| #define cmpltps_m2r(var, reg) sse_m2ri(cmpps, var, reg, 1) |
| #define cmpltps_r2r(regs, regd) sse_r2ri(cmpps, regs, regd, 1) |
| #define cmpltps(vars, vard, xmmreg) sse_m2mi(cmpps, vars, vard, xmmreg, 1) |
| |
| #define cmpleps_m2r(var, reg) sse_m2ri(cmpps, var, reg, 2) |
| #define cmpleps_r2r(regs, regd) sse_r2ri(cmpps, regs, regd, 2) |
| #define cmpleps(vars, vard, xmmreg) sse_m2mi(cmpps, vars, vard, xmmreg, 2) |
| |
| #define cmpunordps_m2r(var, reg) sse_m2ri(cmpps, var, reg, 3) |
| #define cmpunordps_r2r(regs, regd) sse_r2ri(cmpps, regs, regd, 3) |
| #define cmpunordps(vars, vard, xmmreg) sse_m2mi(cmpps, vars, vard, xmmreg, 3) |
| |
| #define cmpneqps_m2r(var, reg) sse_m2ri(cmpps, var, reg, 4) |
| #define cmpneqps_r2r(regs, regd) sse_r2ri(cmpps, regs, regd, 4) |
| #define cmpneqps(vars, vard, xmmreg) sse_m2mi(cmpps, vars, vard, xmmreg, 4) |
| |
| #define cmpnltps_m2r(var, reg) sse_m2ri(cmpps, var, reg, 5) |
| #define cmpnltps_r2r(regs, regd) sse_r2ri(cmpps, regs, regd, 5) |
| #define cmpnltps(vars, vard, xmmreg) sse_m2mi(cmpps, vars, vard, xmmreg, 5) |
| |
| #define cmpnleps_m2r(var, reg) sse_m2ri(cmpps, var, reg, 6) |
| #define cmpnleps_r2r(regs, regd) sse_r2ri(cmpps, regs, regd, 6) |
| #define cmpnleps(vars, vard, xmmreg) sse_m2mi(cmpps, vars, vard, xmmreg, 6) |
| |
| #define cmpordps_m2r(var, reg) sse_m2ri(cmpps, var, reg, 7) |
| #define cmpordps_r2r(regs, regd) sse_r2ri(cmpps, regs, regd, 7) |
| #define cmpordps(vars, vard, xmmreg) sse_m2mi(cmpps, vars, vard, xmmreg, 7) |
| |
| |
| /* Lowest Field of 4x32f Parallel CoMPares |
| (resulting fields are either 0 or -1) |
| */ |
| #define cmpss_m2r(var, reg, op) sse_m2ri(cmpss, var, reg, op) |
| #define cmpss_r2r(regs, regd, op) sse_r2ri(cmpss, regs, regd, op) |
| #define cmpss(vars, vard, op, xmmreg) sse_m2mi(cmpss, vars, vard, xmmreg, op) |
| |
| #define cmpeqss_m2r(var, reg) sse_m2ri(cmpss, var, reg, 0) |
| #define cmpeqss_r2r(regs, regd) sse_r2ri(cmpss, regs, regd, 0) |
| #define cmpeqss(vars, vard, xmmreg) sse_m2mi(cmpss, vars, vard, xmmreg, 0) |
| |
| #define cmpltss_m2r(var, reg) sse_m2ri(cmpss, var, reg, 1) |
| #define cmpltss_r2r(regs, regd) sse_r2ri(cmpss, regs, regd, 1) |
| #define cmpltss(vars, vard, xmmreg) sse_m2mi(cmpss, vars, vard, xmmreg, 1) |
| |
| #define cmpless_m2r(var, reg) sse_m2ri(cmpss, var, reg, 2) |
| #define cmpless_r2r(regs, regd) sse_r2ri(cmpss, regs, regd, 2) |
| #define cmpless(vars, vard, xmmreg) sse_m2mi(cmpss, vars, vard, xmmreg, 2) |
| |
| #define cmpunordss_m2r(var, reg) sse_m2ri(cmpss, var, reg, 3) |
| #define cmpunordss_r2r(regs, regd) sse_r2ri(cmpss, regs, regd, 3) |
| #define cmpunordss(vars, vard, xmmreg) sse_m2mi(cmpss, vars, vard, xmmreg, 3) |
| |
| #define cmpneqss_m2r(var, reg) sse_m2ri(cmpss, var, reg, 4) |
| #define cmpneqss_r2r(regs, regd) sse_r2ri(cmpss, regs, regd, 4) |
| #define cmpneqss(vars, vard, xmmreg) sse_m2mi(cmpss, vars, vard, xmmreg, 4) |
| |
| #define cmpnltss_m2r(var, reg) sse_m2ri(cmpss, var, reg, 5) |
| #define cmpnltss_r2r(regs, regd) sse_r2ri(cmpss, regs, regd, 5) |
| #define cmpnltss(vars, vard, xmmreg) sse_m2mi(cmpss, vars, vard, xmmreg, 5) |
| |
| #define cmpnless_m2r(var, reg) sse_m2ri(cmpss, var, reg, 6) |
| #define cmpnless_r2r(regs, regd) sse_r2ri(cmpss, regs, regd, 6) |
| #define cmpnless(vars, vard, xmmreg) sse_m2mi(cmpss, vars, vard, xmmreg, 6) |
| |
| #define cmpordss_m2r(var, reg) sse_m2ri(cmpss, var, reg, 7) |
| #define cmpordss_r2r(regs, regd) sse_r2ri(cmpss, regs, regd, 7) |
| #define cmpordss(vars, vard, xmmreg) sse_m2mi(cmpss, vars, vard, xmmreg, 7) |
| |
| |
| /* Lowest Field of 4x32f Parallel CoMPares to set EFLAGS |
| (resulting fields are either 0 or -1) |
| */ |
| #define comiss_m2r(var, reg) sse_m2r(comiss, var, reg) |
| #define comiss_r2r(regs, regd) sse_r2r(comiss, regs, regd) |
| #define comiss(vars, vard, xmmreg) sse_m2m(comiss, vars, vard, xmmreg) |
| |
| |
| /* Lowest Field of 4x32f Unordered Parallel CoMPares to set EFLAGS |
| (resulting fields are either 0 or -1) |
| */ |
| #define ucomiss_m2r(var, reg) sse_m2r(ucomiss, var, reg) |
| #define ucomiss_r2r(regs, regd) sse_r2r(ucomiss, regs, regd) |
| #define ucomiss(vars, vard, xmmreg) sse_m2m(ucomiss, vars, vard, xmmreg) |
| |
| |
| /* 2-(4x32f) -> 4x32f UNPaCK Low Packed Single-fp |
| (interleaves low half of dest with low half of source |
| as padding in each result field) |
| */ |
| #define unpcklps_m2r(var, reg) sse_m2r(unpcklps, var, reg) |
| #define unpcklps_r2r(regs, regd) sse_r2r(unpcklps, regs, regd) |
| |
| |
| /* 2-(4x32f) -> 4x32f UNPaCK High Packed Single-fp |
| (interleaves high half of dest with high half of source |
| as padding in each result field) |
| */ |
| #define unpckhps_m2r(var, reg) sse_m2r(unpckhps, var, reg) |
| #define unpckhps_r2r(regs, regd) sse_r2r(unpckhps, regs, regd) |
| |
| |
| |
| /* Fp and mmX ReSTORe state |
| */ |
| #ifdef SSE_TRACE |
| #define fxrstor(mem) \ |
| { \ |
| fprintf(stderr, "fxrstor()\n"); \ |
| __asm__ __volatile__ ("fxrstor %0" \ |
| : /* nothing */ \ |
| : "X" (mem)) \ |
| } |
| #else |
| #define fxrstor(mem) \ |
| __asm__ __volatile__ ("fxrstor %0" \ |
| : /* nothing */ \ |
| : "X" (mem)) |
| #endif |
| |
| |
| /* Fp and mmX SAVE state |
| */ |
| #ifdef SSE_TRACE |
| #define fxsave(mem) \ |
| { \ |
| fprintf(stderr, "fxsave()\n"); \ |
| __asm__ __volatile__ ("fxsave %0" \ |
| : /* nothing */ \ |
| : "X" (mem)) \ |
| } |
| #else |
| #define fxsave(mem) \ |
| __asm__ __volatile__ ("fxsave %0" \ |
| : /* nothing */ \ |
| : "X" (mem)) |
| #endif |
| |
| |
| /* STore streaMing simd eXtensions Control/Status Register |
| */ |
| #ifdef SSE_TRACE |
| #define stmxcsr(mem) \ |
| { \ |
| fprintf(stderr, "stmxcsr()\n"); \ |
| __asm__ __volatile__ ("stmxcsr %0" \ |
| : /* nothing */ \ |
| : "X" (mem)) \ |
| } |
| #else |
| #define stmxcsr(mem) \ |
| __asm__ __volatile__ ("stmxcsr %0" \ |
| : /* nothing */ \ |
| : "X" (mem)) |
| #endif |
| |
| |
| /* LoaD streaMing simd eXtensions Control/Status Register |
| */ |
| #ifdef SSE_TRACE |
| #define ldmxcsr(mem) \ |
| { \ |
| fprintf(stderr, "ldmxcsr()\n"); \ |
| __asm__ __volatile__ ("ldmxcsr %0" \ |
| : /* nothing */ \ |
| : "X" (mem)) \ |
| } |
| #else |
| #define ldmxcsr(mem) \ |
| __asm__ __volatile__ ("ldmxcsr %0" \ |
| : /* nothing */ \ |
| : "X" (mem)) |
| #endif |
| |
| |
| /* Store FENCE - enforce ordering of stores before fence vs. stores |
| occuring after fence in source code. |
| */ |
| #ifdef SSE_TRACE |
| #define sfence() \ |
| { \ |
| fprintf(stderr, "sfence()\n"); \ |
| __asm__ __volatile__ ("sfence\n\t") \ |
| } |
| #else |
| #define sfence() \ |
| __asm__ __volatile__ ("sfence\n\t") |
| #endif |
| |
| |
| /* PREFETCH data using T0, T1, T2, or NTA hint |
| T0 = Prefetch into all cache levels |
| T1 = Prefetch into all cache levels except 0th level |
| T2 = Prefetch into all cache levels except 0th and 1st levels |
| NTA = Prefetch data into non-temporal cache structure |
| */ |
| #ifdef SSE_TRACE |
| #else |
| #define prefetch(mem, hint) \ |
| __asm__ __volatile__ ("prefetch" #hint " %0" \ |
| : /* nothing */ \ |
| : "X" (mem)) |
| |
| #define prefetcht0(mem) prefetch(mem, t0) |
| #define prefetcht1(mem) prefetch(mem, t1) |
| #define prefetcht2(mem) prefetch(mem, t2) |
| #define prefetchnta(mem) prefetch(mem, nta) |
| #endif |
| |
| |
| |
| #endif |
