/* * Copyright (C) 2008 The Android Open Source Project * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #ifdef HAVE_32_BYTE_CACHE_LINE #define CACHE_LINE_SIZE 32 #else #define CACHE_LINE_SIZE 64 #endif /* * Optimized memcmp() for Cortex-A9. */ .syntax unified ENTRY(memcmp) pld [r0, #(CACHE_LINE_SIZE * 0)] pld [r0, #(CACHE_LINE_SIZE * 1)] /* take of the case where length is 0 or the buffers are the same */ cmp r0, r1 moveq r0, #0 bxeq lr pld [r1, #(CACHE_LINE_SIZE * 0)] pld [r1, #(CACHE_LINE_SIZE * 1)] /* make sure we have at least 8+4 bytes, this simplify things below * and avoid some overhead for small blocks */ cmp r2, #(8+4) bmi 10f /* * Neon optimization * Comparing 32 bytes at a time */ #if defined(__ARM_NEON__) subs r2, r2, #32 blo 3f /* preload all the cache lines we need. */ pld [r0, #(CACHE_LINE_SIZE * 2)] pld [r1, #(CACHE_LINE_SIZE * 2)] 1: /* The main loop compares 32 bytes at a time */ vld1.8 {d0 - d3}, [r0]! pld [r0, #(CACHE_LINE_SIZE * 2)] vld1.8 {d4 - d7}, [r1]! pld [r1, #(CACHE_LINE_SIZE * 2)] /* Start subtracting the values and merge results */ vsub.i8 q0, q2 vsub.i8 q1, q3 vorr q2, q0, q1 vorr d4, d5 vmov r3, ip, d4 /* Check if there are any differences among the 32 bytes */ orrs r3, ip bne 2f subs r2, r2, #32 bhs 1b b 3f 2: /* Check if the difference was in the first or last 16 bytes */ sub r0, #32 vorr d0, d1 sub r1, #32 vmov r3, ip, d0 orrs r3, ip /* if the first 16 bytes are equal, we only have to rewind 16 bytes */ ittt eq subeq r2, #16 addeq r0, #16 addeq r1, #16 3: /* fix-up the remaining count */ add r2, r2, #32 cmp r2, #(8+4) bmi 10f #endif /* save registers */ stmfd sp!, {r4, lr} .cfi_def_cfa_offset 8 .cfi_rel_offset r4, 0 .cfi_rel_offset lr, 4 /* since r0 hold the result, move the first source * pointer somewhere else */ mov r4, r0 /* align first pointer to word boundary * offset = -src & 3 */ rsb r3, r4, #0 ands r3, r3, #3 beq 0f /* align first pointer */ sub r2, r2, r3 1: ldrb r0, [r4], #1 ldrb ip, [r1], #1 subs r0, r0, ip bne 9f subs r3, r3, #1 bne 1b 0: /* here the first pointer is aligned, and we have at least 4 bytes * to process. */ /* see if the pointers are congruent */ eor r0, r4, r1 ands r0, r0, #3 bne 5f /* congruent case, 32 bytes per iteration * We need to make sure there are at least 32+4 bytes left * because we effectively read ahead one word, and we could * read past the buffer (and segfault) if we're not careful. */ ldr ip, [r1] subs r2, r2, #(32 + 4) bmi 1f 0: pld [r4, #(CACHE_LINE_SIZE * 2)] pld [r1, #(CACHE_LINE_SIZE * 2)] ldr r0, [r4], #4 ldr lr, [r1, #4]! eors r0, r0, ip ldreq r0, [r4], #4 ldreq ip, [r1, #4]! eorseq r0, r0, lr ldreq r0, [r4], #4 ldreq lr, [r1, #4]! eorseq r0, r0, ip ldreq r0, [r4], #4 ldreq ip, [r1, #4]! eorseq r0, r0, lr ldreq r0, [r4], #4 ldreq lr, [r1, #4]! eorseq r0, r0, ip ldreq r0, [r4], #4 ldreq ip, [r1, #4]! eorseq r0, r0, lr ldreq r0, [r4], #4 ldreq lr, [r1, #4]! eorseq r0, r0, ip ldreq r0, [r4], #4 ldreq ip, [r1, #4]! eorseq r0, r0, lr bne 2f subs r2, r2, #32 bhs 0b /* do we have at least 4 bytes left? */ 1: adds r2, r2, #(32 - 4 + 4) bmi 4f /* finish off 4 bytes at a time */ 3: ldr r0, [r4], #4 ldr ip, [r1], #4 eors r0, r0, ip bne 2f subs r2, r2, #4 bhs 3b /* are we done? */ 4: adds r2, r2, #4 moveq r0, #0 beq 9f /* finish off the remaining bytes */ b 8f 2: /* the last 4 bytes are different, restart them */ sub r4, r4, #4 sub r1, r1, #4 mov r2, #4 /* process the last few bytes */ 8: ldrb r0, [r4], #1 ldrb ip, [r1], #1 // stall subs r0, r0, ip bne 9f subs r2, r2, #1 bne 8b 9: /* restore registers and return */ ldmfd sp!, {r4, pc} 10: /* process less than 12 bytes */ cmp r2, #0 moveq r0, #0 bxeq lr mov r3, r0 11: ldrb r0, [r3], #1 ldrb ip, [r1], #1 subs r0, ip bxne lr subs r2, r2, #1 bne 11b bx lr 5: /*************** non-congruent case ***************/ and r0, r1, #3 cmp r0, #2 bne 4f /* here, offset is 2 (16-bits aligned, special cased) */ /* make sure we have at least 16 bytes to process */ subs r2, r2, #16 addmi r2, r2, #16 bmi 8b /* align the unaligned pointer */ bic r1, r1, #3 ldr lr, [r1], #4 6: pld [r1, #(CACHE_LINE_SIZE * 2)] pld [r4, #(CACHE_LINE_SIZE * 2)] mov ip, lr, lsr #16 ldr lr, [r1], #4 ldr r0, [r4], #4 orr ip, ip, lr, lsl #16 eors r0, r0, ip moveq ip, lr, lsr #16 ldreq lr, [r1], #4 ldreq r0, [r4], #4 orreq ip, ip, lr, lsl #16 eorseq r0, r0, ip moveq ip, lr, lsr #16 ldreq lr, [r1], #4 ldreq r0, [r4], #4 orreq ip, ip, lr, lsl #16 eorseq r0, r0, ip moveq ip, lr, lsr #16 ldreq lr, [r1], #4 ldreq r0, [r4], #4 orreq ip, ip, lr, lsl #16 eorseq r0, r0, ip bne 7f subs r2, r2, #16 bhs 6b sub r1, r1, #2 /* are we done? */ adds r2, r2, #16 moveq r0, #0 beq 9b /* finish off the remaining bytes */ b 8b 7: /* fix up the 2 pointers and fallthrough... */ sub r1, r1, #(4+2) sub r4, r4, #4 mov r2, #4 b 8b 4: /*************** offset is 1 or 3 (less optimized) ***************/ stmfd sp!, {r5, r6, r7} // r5 = rhs // r6 = lhs // r7 = scratch mov r5, r0, lsl #3 /* r5 = right shift */ rsb r6, r5, #32 /* r6 = left shift */ /* align the unaligned pointer */ bic r1, r1, #3 ldr r7, [r1], #4 sub r2, r2, #8 6: mov ip, r7, lsr r5 ldr r7, [r1], #4 ldr r0, [r4], #4 orr ip, ip, r7, lsl r6 eors r0, r0, ip moveq ip, r7, lsr r5 ldreq r7, [r1], #4 ldreq r0, [r4], #4 orreq ip, ip, r7, lsl r6 eorseq r0, r0, ip bne 7f subs r2, r2, #8 bhs 6b sub r1, r1, r6, lsr #3 ldmfd sp!, {r5, r6, r7} /* are we done? */ adds r2, r2, #8 moveq r0, #0 beq 9b /* finish off the remaining bytes */ b 8b 7: /* fix up the 2 pointers and fallthrough... */ sub r1, r1, #4 sub r1, r1, r6, lsr #3 sub r4, r4, #4 mov r2, #4 ldmfd sp!, {r5, r6, r7} b 8b END(memcmp)