Loading...
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 | /* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (C) 2013 ARM Ltd. * Copyright (C) 2013 Linaro. * * This code is based on glibc cortex strings work originally authored by Linaro * be found @ * * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ * files/head:/src/aarch64/ */ #include <linux/linkage.h> #include <asm/assembler.h> #include <asm/cache.h> /* * Move a buffer from src to test (alignment handled by the hardware). * If dest <= src, call memcpy, otherwise copy in reverse order. * * Parameters: * x0 - dest * x1 - src * x2 - n * Returns: * x0 - dest */ dstin .req x0 src .req x1 count .req x2 tmp1 .req x3 tmp1w .req w3 tmp2 .req x4 tmp2w .req w4 tmp3 .req x5 tmp3w .req w5 dst .req x6 A_l .req x7 A_h .req x8 B_l .req x9 B_h .req x10 C_l .req x11 C_h .req x12 D_l .req x13 D_h .req x14 .weak memmove SYM_FUNC_START_ALIAS(__memmove) SYM_FUNC_START_PI(memmove) cmp dstin, src b.lo __memcpy add tmp1, src, count cmp dstin, tmp1 b.hs __memcpy /* No overlap. */ add dst, dstin, count add src, src, count cmp count, #16 b.lo .Ltail15 /*probably non-alignment accesses.*/ ands tmp2, src, #15 /* Bytes to reach alignment. */ b.eq .LSrcAligned sub count, count, tmp2 /* * process the aligned offset length to make the src aligned firstly. * those extra instructions' cost is acceptable. It also make the * coming accesses are based on aligned address. */ tbz tmp2, #0, 1f ldrb tmp1w, [src, #-1]! strb tmp1w, [dst, #-1]! 1: tbz tmp2, #1, 2f ldrh tmp1w, [src, #-2]! strh tmp1w, [dst, #-2]! 2: tbz tmp2, #2, 3f ldr tmp1w, [src, #-4]! str tmp1w, [dst, #-4]! 3: tbz tmp2, #3, .LSrcAligned ldr tmp1, [src, #-8]! str tmp1, [dst, #-8]! .LSrcAligned: cmp count, #64 b.ge .Lcpy_over64 /* * Deal with small copies quickly by dropping straight into the * exit block. */ .Ltail63: /* * Copy up to 48 bytes of data. At this point we only need the * bottom 6 bits of count to be accurate. */ ands tmp1, count, #0x30 b.eq .Ltail15 cmp tmp1w, #0x20 b.eq 1f b.lt 2f ldp A_l, A_h, [src, #-16]! stp A_l, A_h, [dst, #-16]! 1: ldp A_l, A_h, [src, #-16]! stp A_l, A_h, [dst, #-16]! 2: ldp A_l, A_h, [src, #-16]! stp A_l, A_h, [dst, #-16]! .Ltail15: tbz count, #3, 1f ldr tmp1, [src, #-8]! str tmp1, [dst, #-8]! 1: tbz count, #2, 2f ldr tmp1w, [src, #-4]! str tmp1w, [dst, #-4]! 2: tbz count, #1, 3f ldrh tmp1w, [src, #-2]! strh tmp1w, [dst, #-2]! 3: tbz count, #0, .Lexitfunc ldrb tmp1w, [src, #-1] strb tmp1w, [dst, #-1] .Lexitfunc: ret .Lcpy_over64: subs count, count, #128 b.ge .Lcpy_body_large /* * Less than 128 bytes to copy, so handle 64 bytes here and then jump * to the tail. */ ldp A_l, A_h, [src, #-16] stp A_l, A_h, [dst, #-16] ldp B_l, B_h, [src, #-32] ldp C_l, C_h, [src, #-48] stp B_l, B_h, [dst, #-32] stp C_l, C_h, [dst, #-48] ldp D_l, D_h, [src, #-64]! stp D_l, D_h, [dst, #-64]! tst count, #0x3f b.ne .Ltail63 ret /* * Critical loop. Start at a new cache line boundary. Assuming * 64 bytes per line this ensures the entire loop is in one line. */ .p2align L1_CACHE_SHIFT .Lcpy_body_large: /* pre-load 64 bytes data. */ ldp A_l, A_h, [src, #-16] ldp B_l, B_h, [src, #-32] ldp C_l, C_h, [src, #-48] ldp D_l, D_h, [src, #-64]! 1: /* * interlace the load of next 64 bytes data block with store of the last * loaded 64 bytes data. */ stp A_l, A_h, [dst, #-16] ldp A_l, A_h, [src, #-16] stp B_l, B_h, [dst, #-32] ldp B_l, B_h, [src, #-32] stp C_l, C_h, [dst, #-48] ldp C_l, C_h, [src, #-48] stp D_l, D_h, [dst, #-64]! ldp D_l, D_h, [src, #-64]! subs count, count, #64 b.ge 1b stp A_l, A_h, [dst, #-16] stp B_l, B_h, [dst, #-32] stp C_l, C_h, [dst, #-48] stp D_l, D_h, [dst, #-64]! tst count, #0x3f b.ne .Ltail63 ret SYM_FUNC_END_PI(memmove) EXPORT_SYMBOL(memmove) SYM_FUNC_END_ALIAS(__memmove) EXPORT_SYMBOL(__memmove) |