blob: 5cff141ada8f22e366c6baac5a1a5ea8854435a4 [file] [log] [blame]
Bruce Beare8ff1a272010-03-04 11:03:37 -08001/*
Varvara Rainchik5a922842014-04-24 15:41:20 +04002Copyright (c) 2014, Intel Corporation
Bruce Beare8ff1a272010-03-04 11:03:37 -08003All rights reserved.
4
5Redistribution and use in source and binary forms, with or without
6modification, are permitted provided that the following conditions are met:
7
8 * Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10
11 * Redistributions in binary form must reproduce the above copyright notice,
12 * this list of conditions and the following disclaimer in the documentation
13 * and/or other materials provided with the distribution.
14
15 * Neither the name of Intel Corporation nor the names of its contributors
16 * may be used to endorse or promote products derived from this software
17 * without specific prior written permission.
18
19THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
20ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
23ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
26ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29*/
30
Elliott Hughes81d6a182016-03-03 16:10:33 -080031#include <private/bionic_asm.h>
Varvara Rainchik5a922842014-04-24 15:41:20 +040032
Elliott Hughesed777142022-07-25 16:25:11 +000033#define FOR_SILVERMONT
Elliott Hughes81d6a182016-03-03 16:10:33 -080034#include "cache.h"
Liubov Dmitrieva0a490662012-01-17 12:55:46 +040035
Bruce Beare8ff1a272010-03-04 11:03:37 -080036#ifndef L
37# define L(label) .L##label
38#endif
39
40#ifndef ALIGN
41# define ALIGN(n) .p2align n
42#endif
43
Bruce Beare8ff1a272010-03-04 11:03:37 -080044#define CFI_PUSH(REG) \
Elliott Hughese412f892016-08-01 14:00:45 -070045 .cfi_adjust_cfa_offset 4; \
46 .cfi_rel_offset REG, 0
Bruce Beare8ff1a272010-03-04 11:03:37 -080047
48#define CFI_POP(REG) \
Elliott Hughese412f892016-08-01 14:00:45 -070049 .cfi_adjust_cfa_offset -4; \
50 .cfi_restore REG
Bruce Beare8ff1a272010-03-04 11:03:37 -080051
Elliott Hughese412f892016-08-01 14:00:45 -070052#define PUSH(REG) pushl REG; CFI_PUSH(REG)
53#define POP(REG) popl REG; CFI_POP(REG)
Bruce Beare8ff1a272010-03-04 11:03:37 -080054
Elliott Hughese412f892016-08-01 14:00:45 -070055#define PARMS 8 /* Preserve EBX. */
56#define DST PARMS
57#define CHR (DST+4)
58#define LEN (CHR+4)
Elliott Hughes81d6a182016-03-03 16:10:33 -080059#define CHK_DST_LEN (LEN+4)
60#define SETRTNVAL movl DST(%esp), %eax
Bruce Beare8ff1a272010-03-04 11:03:37 -080061
Elliott Hughese412f892016-08-01 14:00:45 -070062# define ENTRANCE PUSH(%ebx);
63# define RETURN_END POP(%ebx); ret
64# define RETURN RETURN_END; CFI_PUSH(%ebx)
Bruce Beare8ff1a272010-03-04 11:03:37 -080065# define JMPTBL(I, B) I - B
66
67/* Load an entry in a jump table into EBX and branch to it. TABLE is a
68 jump table with relative offsets. */
69# define BRANCH_TO_JMPTBL_ENTRY(TABLE) \
70 /* We first load PC into EBX. */ \
Varvara Rainchik5a922842014-04-24 15:41:20 +040071 call __x86.get_pc_thunk.bx; \
Bruce Beare8ff1a272010-03-04 11:03:37 -080072 /* Get the address of the jump table. */ \
73 add $(TABLE - .), %ebx; \
74 /* Get the entry and convert the relative offset to the \
75 absolute address. */ \
76 add (%ebx,%ecx,4), %ebx; \
77 add %ecx, %edx; \
Elliott Hughese412f892016-08-01 14:00:45 -070078 /* We loaded the jump table and adjusted EDX. Go. */ \
Bruce Beare8ff1a272010-03-04 11:03:37 -080079 jmp *%ebx
80
Haibo Huangb9244ff2018-08-11 10:12:13 -070081ENTRY(__memset_chk_generic)
Elliott Hughes204990c2016-03-24 22:34:47 -070082 ENTRANCE
Elliott Hughes81d6a182016-03-03 16:10:33 -080083
Elliott Hughes204990c2016-03-24 22:34:47 -070084 movl LEN(%esp), %ecx
85 cmpl CHK_DST_LEN(%esp), %ecx
Lev Rumyantsev3a528f12016-07-29 17:55:42 -070086 jna L(memset_length_loaded)
87
Elliott Hughese412f892016-08-01 14:00:45 -070088 POP(%ebx) // Undo ENTRANCE without returning.
Lev Rumyantsev3a528f12016-07-29 17:55:42 -070089 jmp __memset_chk_fail
Haibo Huangb9244ff2018-08-11 10:12:13 -070090END(__memset_chk_generic)
Elliott Hughes81d6a182016-03-03 16:10:33 -080091
Bruce Beare8ff1a272010-03-04 11:03:37 -080092 .section .text.sse2,"ax",@progbits
Elliott Hughese412f892016-08-01 14:00:45 -070093 ALIGN(4)
Haibo Huangb9244ff2018-08-11 10:12:13 -070094ENTRY(memset_generic)
Bruce Beare8ff1a272010-03-04 11:03:37 -080095 ENTRANCE
96
97 movl LEN(%esp), %ecx
Elliott Hughes204990c2016-03-24 22:34:47 -070098L(memset_length_loaded):
Varvara Rainchik5a922842014-04-24 15:41:20 +040099 cmp $0, %ecx
100 ja L(1byteormore)
101 SETRTNVAL
102 RETURN
103
104L(1byteormore):
Bruce Beare8ff1a272010-03-04 11:03:37 -0800105 movzbl CHR(%esp), %eax
106 movb %al, %ah
107 /* Fill the whole EAX with pattern. */
108 movl %eax, %edx
Varvara Rainchik5a922842014-04-24 15:41:20 +0400109 shl $16, %eax
Bruce Beare8ff1a272010-03-04 11:03:37 -0800110 or %edx, %eax
Elliott Hughes81d6a182016-03-03 16:10:33 -0800111 movl DST(%esp), %edx
Varvara Rainchik5a922842014-04-24 15:41:20 +0400112 cmp $1, %ecx
113 je L(1byte)
114 cmp $16, %ecx
115 jae L(16bytesormore)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800116
Varvara Rainchik5a922842014-04-24 15:41:20 +0400117 cmp $4, %ecx
118 jb L(4bytesless)
119 movl %eax, (%edx)
120 movl %eax, -4(%edx, %ecx)
121 cmp $8, %ecx
122 jb L(8bytesless)
123 movl %eax, 4(%edx)
124 movl %eax, -8(%edx, %ecx)
125L(8bytesless):
126 SETRTNVAL
127 RETURN
Bruce Beare8ff1a272010-03-04 11:03:37 -0800128
Varvara Rainchik5a922842014-04-24 15:41:20 +0400129L(4bytesless):
130 movw %ax, (%edx)
131 movw %ax, -2(%edx, %ecx)
132 SETRTNVAL
133 RETURN
Bruce Beare8ff1a272010-03-04 11:03:37 -0800134
Varvara Rainchik5a922842014-04-24 15:41:20 +0400135L(1byte):
136 movb %al, (%edx)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800137 SETRTNVAL
138 RETURN
139
Elliott Hughese412f892016-08-01 14:00:45 -0700140 ALIGN(4)
Varvara Rainchik5a922842014-04-24 15:41:20 +0400141L(16bytesormore):
Bruce Beare8ff1a272010-03-04 11:03:37 -0800142 movd %eax, %xmm0
Bruce Beare8ff1a272010-03-04 11:03:37 -0800143 pshufd $0, %xmm0, %xmm0
Varvara Rainchik5a922842014-04-24 15:41:20 +0400144
145 cmp $64, %ecx
146 ja L(64bytesmore)
147 movdqu %xmm0, (%edx)
148 movdqu %xmm0, -16(%edx, %ecx)
149 cmp $32, %ecx
150 jbe L(32bytesless)
151 movdqu %xmm0, 16(%edx)
152 movdqu %xmm0, -32(%edx, %ecx)
153L(32bytesless):
154 SETRTNVAL
155 RETURN
156
157L(64bytesmore):
Bruce Beare8ff1a272010-03-04 11:03:37 -0800158 testl $0xf, %edx
159 jz L(aligned_16)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800160L(not_aligned_16):
161 movdqu %xmm0, (%edx)
162 movl %edx, %eax
163 and $-16, %edx
164 add $16, %edx
165 sub %edx, %eax
166 add %eax, %ecx
167 movd %xmm0, %eax
168
Elliott Hughese412f892016-08-01 14:00:45 -0700169 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800170L(aligned_16):
171 cmp $128, %ecx
172 jae L(128bytesormore)
173
174L(aligned_16_less128bytes):
Elliott Hughese412f892016-08-01 14:00:45 -0700175 BRANCH_TO_JMPTBL_ENTRY(L(table_16_128bytes))
Bruce Beare8ff1a272010-03-04 11:03:37 -0800176
Elliott Hughese412f892016-08-01 14:00:45 -0700177 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800178L(128bytesormore):
Elliott Hughese412f892016-08-01 14:00:45 -0700179 PUSH(%ebx)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800180 mov $SHARED_CACHE_SIZE, %ebx
Bruce Beare8ff1a272010-03-04 11:03:37 -0800181 cmp %ebx, %ecx
182 jae L(128bytesormore_nt_start)
183
Elliott Hughese412f892016-08-01 14:00:45 -0700184 POP(%ebx)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800185
Elliott Hughese412f892016-08-01 14:00:45 -0700186 PUSH(%ebx)
Varvara Rainchik5a922842014-04-24 15:41:20 +0400187 mov $DATA_CACHE_SIZE, %ebx
Bruce Beare8ff1a272010-03-04 11:03:37 -0800188
Varvara Rainchik5a922842014-04-24 15:41:20 +0400189 cmp %ebx, %ecx
Bruce Beare8ff1a272010-03-04 11:03:37 -0800190 jae L(128bytes_L2_normal)
191 subl $128, %ecx
192L(128bytesormore_normal):
193 sub $128, %ecx
194 movdqa %xmm0, (%edx)
Varvara Rainchik5a922842014-04-24 15:41:20 +0400195 movaps %xmm0, 0x10(%edx)
196 movaps %xmm0, 0x20(%edx)
197 movaps %xmm0, 0x30(%edx)
198 movaps %xmm0, 0x40(%edx)
199 movaps %xmm0, 0x50(%edx)
200 movaps %xmm0, 0x60(%edx)
201 movaps %xmm0, 0x70(%edx)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800202 lea 128(%edx), %edx
203 jb L(128bytesless_normal)
204
205
206 sub $128, %ecx
207 movdqa %xmm0, (%edx)
Varvara Rainchik5a922842014-04-24 15:41:20 +0400208 movaps %xmm0, 0x10(%edx)
209 movaps %xmm0, 0x20(%edx)
210 movaps %xmm0, 0x30(%edx)
211 movaps %xmm0, 0x40(%edx)
212 movaps %xmm0, 0x50(%edx)
213 movaps %xmm0, 0x60(%edx)
214 movaps %xmm0, 0x70(%edx)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800215 lea 128(%edx), %edx
216 jae L(128bytesormore_normal)
217
218L(128bytesless_normal):
Varvara Rainchik5a922842014-04-24 15:41:20 +0400219 lea 128(%ecx), %ecx
Elliott Hughese412f892016-08-01 14:00:45 -0700220 POP(%ebx)
221 BRANCH_TO_JMPTBL_ENTRY(L(table_16_128bytes))
Bruce Beare8ff1a272010-03-04 11:03:37 -0800222
Elliott Hughese412f892016-08-01 14:00:45 -0700223 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800224L(128bytes_L2_normal):
Varvara Rainchik5a922842014-04-24 15:41:20 +0400225 prefetchnta 0x380(%edx)
226 prefetchnta 0x3c0(%edx)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800227 sub $128, %ecx
228 movdqa %xmm0, (%edx)
229 movaps %xmm0, 0x10(%edx)
230 movaps %xmm0, 0x20(%edx)
231 movaps %xmm0, 0x30(%edx)
232 movaps %xmm0, 0x40(%edx)
233 movaps %xmm0, 0x50(%edx)
234 movaps %xmm0, 0x60(%edx)
235 movaps %xmm0, 0x70(%edx)
236 add $128, %edx
237 cmp $128, %ecx
238 jae L(128bytes_L2_normal)
239
240L(128bytesless_L2_normal):
Elliott Hughese412f892016-08-01 14:00:45 -0700241 POP(%ebx)
242 BRANCH_TO_JMPTBL_ENTRY(L(table_16_128bytes))
Bruce Beare8ff1a272010-03-04 11:03:37 -0800243
244L(128bytesormore_nt_start):
245 sub %ebx, %ecx
Elliott Hughese412f892016-08-01 14:00:45 -0700246 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800247L(128bytesormore_shared_cache_loop):
Varvara Rainchik5a922842014-04-24 15:41:20 +0400248 prefetchnta 0x3c0(%edx)
249 prefetchnta 0x380(%edx)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800250 sub $0x80, %ebx
251 movdqa %xmm0, (%edx)
Varvara Rainchik5a922842014-04-24 15:41:20 +0400252 movaps %xmm0, 0x10(%edx)
253 movaps %xmm0, 0x20(%edx)
254 movaps %xmm0, 0x30(%edx)
255 movaps %xmm0, 0x40(%edx)
256 movaps %xmm0, 0x50(%edx)
257 movaps %xmm0, 0x60(%edx)
258 movaps %xmm0, 0x70(%edx)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800259 add $0x80, %edx
260 cmp $0x80, %ebx
261 jae L(128bytesormore_shared_cache_loop)
262 cmp $0x80, %ecx
263 jb L(shared_cache_loop_end)
Elliott Hughese412f892016-08-01 14:00:45 -0700264 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800265L(128bytesormore_nt):
266 sub $0x80, %ecx
267 movntdq %xmm0, (%edx)
268 movntdq %xmm0, 0x10(%edx)
269 movntdq %xmm0, 0x20(%edx)
270 movntdq %xmm0, 0x30(%edx)
271 movntdq %xmm0, 0x40(%edx)
272 movntdq %xmm0, 0x50(%edx)
273 movntdq %xmm0, 0x60(%edx)
274 movntdq %xmm0, 0x70(%edx)
275 add $0x80, %edx
276 cmp $0x80, %ecx
277 jae L(128bytesormore_nt)
278 sfence
279L(shared_cache_loop_end):
Elliott Hughese412f892016-08-01 14:00:45 -0700280 POP(%ebx)
281 BRANCH_TO_JMPTBL_ENTRY(L(table_16_128bytes))
Bruce Beare8ff1a272010-03-04 11:03:37 -0800282
283
284 .pushsection .rodata.sse2,"a",@progbits
Elliott Hughese412f892016-08-01 14:00:45 -0700285 ALIGN(2)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800286L(table_16_128bytes):
Elliott Hughese412f892016-08-01 14:00:45 -0700287 .int JMPTBL(L(aligned_16_0bytes), L(table_16_128bytes))
288 .int JMPTBL(L(aligned_16_1bytes), L(table_16_128bytes))
289 .int JMPTBL(L(aligned_16_2bytes), L(table_16_128bytes))
290 .int JMPTBL(L(aligned_16_3bytes), L(table_16_128bytes))
291 .int JMPTBL(L(aligned_16_4bytes), L(table_16_128bytes))
292 .int JMPTBL(L(aligned_16_5bytes), L(table_16_128bytes))
293 .int JMPTBL(L(aligned_16_6bytes), L(table_16_128bytes))
294 .int JMPTBL(L(aligned_16_7bytes), L(table_16_128bytes))
295 .int JMPTBL(L(aligned_16_8bytes), L(table_16_128bytes))
296 .int JMPTBL(L(aligned_16_9bytes), L(table_16_128bytes))
297 .int JMPTBL(L(aligned_16_10bytes), L(table_16_128bytes))
298 .int JMPTBL(L(aligned_16_11bytes), L(table_16_128bytes))
299 .int JMPTBL(L(aligned_16_12bytes), L(table_16_128bytes))
300 .int JMPTBL(L(aligned_16_13bytes), L(table_16_128bytes))
301 .int JMPTBL(L(aligned_16_14bytes), L(table_16_128bytes))
302 .int JMPTBL(L(aligned_16_15bytes), L(table_16_128bytes))
303 .int JMPTBL(L(aligned_16_16bytes), L(table_16_128bytes))
304 .int JMPTBL(L(aligned_16_17bytes), L(table_16_128bytes))
305 .int JMPTBL(L(aligned_16_18bytes), L(table_16_128bytes))
306 .int JMPTBL(L(aligned_16_19bytes), L(table_16_128bytes))
307 .int JMPTBL(L(aligned_16_20bytes), L(table_16_128bytes))
308 .int JMPTBL(L(aligned_16_21bytes), L(table_16_128bytes))
309 .int JMPTBL(L(aligned_16_22bytes), L(table_16_128bytes))
310 .int JMPTBL(L(aligned_16_23bytes), L(table_16_128bytes))
311 .int JMPTBL(L(aligned_16_24bytes), L(table_16_128bytes))
312 .int JMPTBL(L(aligned_16_25bytes), L(table_16_128bytes))
313 .int JMPTBL(L(aligned_16_26bytes), L(table_16_128bytes))
314 .int JMPTBL(L(aligned_16_27bytes), L(table_16_128bytes))
315 .int JMPTBL(L(aligned_16_28bytes), L(table_16_128bytes))
316 .int JMPTBL(L(aligned_16_29bytes), L(table_16_128bytes))
317 .int JMPTBL(L(aligned_16_30bytes), L(table_16_128bytes))
318 .int JMPTBL(L(aligned_16_31bytes), L(table_16_128bytes))
319 .int JMPTBL(L(aligned_16_32bytes), L(table_16_128bytes))
320 .int JMPTBL(L(aligned_16_33bytes), L(table_16_128bytes))
321 .int JMPTBL(L(aligned_16_34bytes), L(table_16_128bytes))
322 .int JMPTBL(L(aligned_16_35bytes), L(table_16_128bytes))
323 .int JMPTBL(L(aligned_16_36bytes), L(table_16_128bytes))
324 .int JMPTBL(L(aligned_16_37bytes), L(table_16_128bytes))
325 .int JMPTBL(L(aligned_16_38bytes), L(table_16_128bytes))
326 .int JMPTBL(L(aligned_16_39bytes), L(table_16_128bytes))
327 .int JMPTBL(L(aligned_16_40bytes), L(table_16_128bytes))
328 .int JMPTBL(L(aligned_16_41bytes), L(table_16_128bytes))
329 .int JMPTBL(L(aligned_16_42bytes), L(table_16_128bytes))
330 .int JMPTBL(L(aligned_16_43bytes), L(table_16_128bytes))
331 .int JMPTBL(L(aligned_16_44bytes), L(table_16_128bytes))
332 .int JMPTBL(L(aligned_16_45bytes), L(table_16_128bytes))
333 .int JMPTBL(L(aligned_16_46bytes), L(table_16_128bytes))
334 .int JMPTBL(L(aligned_16_47bytes), L(table_16_128bytes))
335 .int JMPTBL(L(aligned_16_48bytes), L(table_16_128bytes))
336 .int JMPTBL(L(aligned_16_49bytes), L(table_16_128bytes))
337 .int JMPTBL(L(aligned_16_50bytes), L(table_16_128bytes))
338 .int JMPTBL(L(aligned_16_51bytes), L(table_16_128bytes))
339 .int JMPTBL(L(aligned_16_52bytes), L(table_16_128bytes))
340 .int JMPTBL(L(aligned_16_53bytes), L(table_16_128bytes))
341 .int JMPTBL(L(aligned_16_54bytes), L(table_16_128bytes))
342 .int JMPTBL(L(aligned_16_55bytes), L(table_16_128bytes))
343 .int JMPTBL(L(aligned_16_56bytes), L(table_16_128bytes))
344 .int JMPTBL(L(aligned_16_57bytes), L(table_16_128bytes))
345 .int JMPTBL(L(aligned_16_58bytes), L(table_16_128bytes))
346 .int JMPTBL(L(aligned_16_59bytes), L(table_16_128bytes))
347 .int JMPTBL(L(aligned_16_60bytes), L(table_16_128bytes))
348 .int JMPTBL(L(aligned_16_61bytes), L(table_16_128bytes))
349 .int JMPTBL(L(aligned_16_62bytes), L(table_16_128bytes))
350 .int JMPTBL(L(aligned_16_63bytes), L(table_16_128bytes))
351 .int JMPTBL(L(aligned_16_64bytes), L(table_16_128bytes))
352 .int JMPTBL(L(aligned_16_65bytes), L(table_16_128bytes))
353 .int JMPTBL(L(aligned_16_66bytes), L(table_16_128bytes))
354 .int JMPTBL(L(aligned_16_67bytes), L(table_16_128bytes))
355 .int JMPTBL(L(aligned_16_68bytes), L(table_16_128bytes))
356 .int JMPTBL(L(aligned_16_69bytes), L(table_16_128bytes))
357 .int JMPTBL(L(aligned_16_70bytes), L(table_16_128bytes))
358 .int JMPTBL(L(aligned_16_71bytes), L(table_16_128bytes))
359 .int JMPTBL(L(aligned_16_72bytes), L(table_16_128bytes))
360 .int JMPTBL(L(aligned_16_73bytes), L(table_16_128bytes))
361 .int JMPTBL(L(aligned_16_74bytes), L(table_16_128bytes))
362 .int JMPTBL(L(aligned_16_75bytes), L(table_16_128bytes))
363 .int JMPTBL(L(aligned_16_76bytes), L(table_16_128bytes))
364 .int JMPTBL(L(aligned_16_77bytes), L(table_16_128bytes))
365 .int JMPTBL(L(aligned_16_78bytes), L(table_16_128bytes))
366 .int JMPTBL(L(aligned_16_79bytes), L(table_16_128bytes))
367 .int JMPTBL(L(aligned_16_80bytes), L(table_16_128bytes))
368 .int JMPTBL(L(aligned_16_81bytes), L(table_16_128bytes))
369 .int JMPTBL(L(aligned_16_82bytes), L(table_16_128bytes))
370 .int JMPTBL(L(aligned_16_83bytes), L(table_16_128bytes))
371 .int JMPTBL(L(aligned_16_84bytes), L(table_16_128bytes))
372 .int JMPTBL(L(aligned_16_85bytes), L(table_16_128bytes))
373 .int JMPTBL(L(aligned_16_86bytes), L(table_16_128bytes))
374 .int JMPTBL(L(aligned_16_87bytes), L(table_16_128bytes))
375 .int JMPTBL(L(aligned_16_88bytes), L(table_16_128bytes))
376 .int JMPTBL(L(aligned_16_89bytes), L(table_16_128bytes))
377 .int JMPTBL(L(aligned_16_90bytes), L(table_16_128bytes))
378 .int JMPTBL(L(aligned_16_91bytes), L(table_16_128bytes))
379 .int JMPTBL(L(aligned_16_92bytes), L(table_16_128bytes))
380 .int JMPTBL(L(aligned_16_93bytes), L(table_16_128bytes))
381 .int JMPTBL(L(aligned_16_94bytes), L(table_16_128bytes))
382 .int JMPTBL(L(aligned_16_95bytes), L(table_16_128bytes))
383 .int JMPTBL(L(aligned_16_96bytes), L(table_16_128bytes))
384 .int JMPTBL(L(aligned_16_97bytes), L(table_16_128bytes))
385 .int JMPTBL(L(aligned_16_98bytes), L(table_16_128bytes))
386 .int JMPTBL(L(aligned_16_99bytes), L(table_16_128bytes))
387 .int JMPTBL(L(aligned_16_100bytes), L(table_16_128bytes))
388 .int JMPTBL(L(aligned_16_101bytes), L(table_16_128bytes))
389 .int JMPTBL(L(aligned_16_102bytes), L(table_16_128bytes))
390 .int JMPTBL(L(aligned_16_103bytes), L(table_16_128bytes))
391 .int JMPTBL(L(aligned_16_104bytes), L(table_16_128bytes))
392 .int JMPTBL(L(aligned_16_105bytes), L(table_16_128bytes))
393 .int JMPTBL(L(aligned_16_106bytes), L(table_16_128bytes))
394 .int JMPTBL(L(aligned_16_107bytes), L(table_16_128bytes))
395 .int JMPTBL(L(aligned_16_108bytes), L(table_16_128bytes))
396 .int JMPTBL(L(aligned_16_109bytes), L(table_16_128bytes))
397 .int JMPTBL(L(aligned_16_110bytes), L(table_16_128bytes))
398 .int JMPTBL(L(aligned_16_111bytes), L(table_16_128bytes))
399 .int JMPTBL(L(aligned_16_112bytes), L(table_16_128bytes))
400 .int JMPTBL(L(aligned_16_113bytes), L(table_16_128bytes))
401 .int JMPTBL(L(aligned_16_114bytes), L(table_16_128bytes))
402 .int JMPTBL(L(aligned_16_115bytes), L(table_16_128bytes))
403 .int JMPTBL(L(aligned_16_116bytes), L(table_16_128bytes))
404 .int JMPTBL(L(aligned_16_117bytes), L(table_16_128bytes))
405 .int JMPTBL(L(aligned_16_118bytes), L(table_16_128bytes))
406 .int JMPTBL(L(aligned_16_119bytes), L(table_16_128bytes))
407 .int JMPTBL(L(aligned_16_120bytes), L(table_16_128bytes))
408 .int JMPTBL(L(aligned_16_121bytes), L(table_16_128bytes))
409 .int JMPTBL(L(aligned_16_122bytes), L(table_16_128bytes))
410 .int JMPTBL(L(aligned_16_123bytes), L(table_16_128bytes))
411 .int JMPTBL(L(aligned_16_124bytes), L(table_16_128bytes))
412 .int JMPTBL(L(aligned_16_125bytes), L(table_16_128bytes))
413 .int JMPTBL(L(aligned_16_126bytes), L(table_16_128bytes))
414 .int JMPTBL(L(aligned_16_127bytes), L(table_16_128bytes))
Bruce Beare8ff1a272010-03-04 11:03:37 -0800415 .popsection
416
Elliott Hughese412f892016-08-01 14:00:45 -0700417 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800418L(aligned_16_112bytes):
419 movdqa %xmm0, -112(%edx)
420L(aligned_16_96bytes):
421 movdqa %xmm0, -96(%edx)
422L(aligned_16_80bytes):
423 movdqa %xmm0, -80(%edx)
424L(aligned_16_64bytes):
425 movdqa %xmm0, -64(%edx)
426L(aligned_16_48bytes):
427 movdqa %xmm0, -48(%edx)
428L(aligned_16_32bytes):
429 movdqa %xmm0, -32(%edx)
430L(aligned_16_16bytes):
431 movdqa %xmm0, -16(%edx)
432L(aligned_16_0bytes):
433 SETRTNVAL
434 RETURN
435
Elliott Hughese412f892016-08-01 14:00:45 -0700436 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800437L(aligned_16_113bytes):
438 movdqa %xmm0, -113(%edx)
439L(aligned_16_97bytes):
440 movdqa %xmm0, -97(%edx)
441L(aligned_16_81bytes):
442 movdqa %xmm0, -81(%edx)
443L(aligned_16_65bytes):
444 movdqa %xmm0, -65(%edx)
445L(aligned_16_49bytes):
446 movdqa %xmm0, -49(%edx)
447L(aligned_16_33bytes):
448 movdqa %xmm0, -33(%edx)
449L(aligned_16_17bytes):
450 movdqa %xmm0, -17(%edx)
451L(aligned_16_1bytes):
452 movb %al, -1(%edx)
453 SETRTNVAL
454 RETURN
455
Elliott Hughese412f892016-08-01 14:00:45 -0700456 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800457L(aligned_16_114bytes):
458 movdqa %xmm0, -114(%edx)
459L(aligned_16_98bytes):
460 movdqa %xmm0, -98(%edx)
461L(aligned_16_82bytes):
462 movdqa %xmm0, -82(%edx)
463L(aligned_16_66bytes):
464 movdqa %xmm0, -66(%edx)
465L(aligned_16_50bytes):
466 movdqa %xmm0, -50(%edx)
467L(aligned_16_34bytes):
468 movdqa %xmm0, -34(%edx)
469L(aligned_16_18bytes):
470 movdqa %xmm0, -18(%edx)
471L(aligned_16_2bytes):
472 movw %ax, -2(%edx)
473 SETRTNVAL
474 RETURN
475
Elliott Hughese412f892016-08-01 14:00:45 -0700476 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800477L(aligned_16_115bytes):
478 movdqa %xmm0, -115(%edx)
479L(aligned_16_99bytes):
480 movdqa %xmm0, -99(%edx)
481L(aligned_16_83bytes):
482 movdqa %xmm0, -83(%edx)
483L(aligned_16_67bytes):
484 movdqa %xmm0, -67(%edx)
485L(aligned_16_51bytes):
486 movdqa %xmm0, -51(%edx)
487L(aligned_16_35bytes):
488 movdqa %xmm0, -35(%edx)
489L(aligned_16_19bytes):
490 movdqa %xmm0, -19(%edx)
491L(aligned_16_3bytes):
492 movw %ax, -3(%edx)
493 movb %al, -1(%edx)
494 SETRTNVAL
495 RETURN
496
Elliott Hughese412f892016-08-01 14:00:45 -0700497 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800498L(aligned_16_116bytes):
499 movdqa %xmm0, -116(%edx)
500L(aligned_16_100bytes):
501 movdqa %xmm0, -100(%edx)
502L(aligned_16_84bytes):
503 movdqa %xmm0, -84(%edx)
504L(aligned_16_68bytes):
505 movdqa %xmm0, -68(%edx)
506L(aligned_16_52bytes):
507 movdqa %xmm0, -52(%edx)
508L(aligned_16_36bytes):
509 movdqa %xmm0, -36(%edx)
510L(aligned_16_20bytes):
511 movdqa %xmm0, -20(%edx)
512L(aligned_16_4bytes):
513 movl %eax, -4(%edx)
514 SETRTNVAL
515 RETURN
516
Elliott Hughese412f892016-08-01 14:00:45 -0700517 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800518L(aligned_16_117bytes):
519 movdqa %xmm0, -117(%edx)
520L(aligned_16_101bytes):
521 movdqa %xmm0, -101(%edx)
522L(aligned_16_85bytes):
523 movdqa %xmm0, -85(%edx)
524L(aligned_16_69bytes):
525 movdqa %xmm0, -69(%edx)
526L(aligned_16_53bytes):
527 movdqa %xmm0, -53(%edx)
528L(aligned_16_37bytes):
529 movdqa %xmm0, -37(%edx)
530L(aligned_16_21bytes):
531 movdqa %xmm0, -21(%edx)
532L(aligned_16_5bytes):
533 movl %eax, -5(%edx)
534 movb %al, -1(%edx)
535 SETRTNVAL
536 RETURN
537
Elliott Hughese412f892016-08-01 14:00:45 -0700538 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800539L(aligned_16_118bytes):
540 movdqa %xmm0, -118(%edx)
541L(aligned_16_102bytes):
542 movdqa %xmm0, -102(%edx)
543L(aligned_16_86bytes):
544 movdqa %xmm0, -86(%edx)
545L(aligned_16_70bytes):
546 movdqa %xmm0, -70(%edx)
547L(aligned_16_54bytes):
548 movdqa %xmm0, -54(%edx)
549L(aligned_16_38bytes):
550 movdqa %xmm0, -38(%edx)
551L(aligned_16_22bytes):
552 movdqa %xmm0, -22(%edx)
553L(aligned_16_6bytes):
554 movl %eax, -6(%edx)
555 movw %ax, -2(%edx)
556 SETRTNVAL
557 RETURN
558
Elliott Hughese412f892016-08-01 14:00:45 -0700559 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800560L(aligned_16_119bytes):
561 movdqa %xmm0, -119(%edx)
562L(aligned_16_103bytes):
563 movdqa %xmm0, -103(%edx)
564L(aligned_16_87bytes):
565 movdqa %xmm0, -87(%edx)
566L(aligned_16_71bytes):
567 movdqa %xmm0, -71(%edx)
568L(aligned_16_55bytes):
569 movdqa %xmm0, -55(%edx)
570L(aligned_16_39bytes):
571 movdqa %xmm0, -39(%edx)
572L(aligned_16_23bytes):
573 movdqa %xmm0, -23(%edx)
574L(aligned_16_7bytes):
575 movl %eax, -7(%edx)
576 movw %ax, -3(%edx)
577 movb %al, -1(%edx)
578 SETRTNVAL
579 RETURN
580
Elliott Hughese412f892016-08-01 14:00:45 -0700581 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800582L(aligned_16_120bytes):
583 movdqa %xmm0, -120(%edx)
584L(aligned_16_104bytes):
585 movdqa %xmm0, -104(%edx)
586L(aligned_16_88bytes):
587 movdqa %xmm0, -88(%edx)
588L(aligned_16_72bytes):
589 movdqa %xmm0, -72(%edx)
590L(aligned_16_56bytes):
591 movdqa %xmm0, -56(%edx)
592L(aligned_16_40bytes):
593 movdqa %xmm0, -40(%edx)
594L(aligned_16_24bytes):
595 movdqa %xmm0, -24(%edx)
596L(aligned_16_8bytes):
597 movq %xmm0, -8(%edx)
598 SETRTNVAL
599 RETURN
600
Elliott Hughese412f892016-08-01 14:00:45 -0700601 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800602L(aligned_16_121bytes):
603 movdqa %xmm0, -121(%edx)
604L(aligned_16_105bytes):
605 movdqa %xmm0, -105(%edx)
606L(aligned_16_89bytes):
607 movdqa %xmm0, -89(%edx)
608L(aligned_16_73bytes):
609 movdqa %xmm0, -73(%edx)
610L(aligned_16_57bytes):
611 movdqa %xmm0, -57(%edx)
612L(aligned_16_41bytes):
613 movdqa %xmm0, -41(%edx)
614L(aligned_16_25bytes):
615 movdqa %xmm0, -25(%edx)
616L(aligned_16_9bytes):
617 movq %xmm0, -9(%edx)
618 movb %al, -1(%edx)
619 SETRTNVAL
620 RETURN
621
Elliott Hughese412f892016-08-01 14:00:45 -0700622 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800623L(aligned_16_122bytes):
624 movdqa %xmm0, -122(%edx)
625L(aligned_16_106bytes):
626 movdqa %xmm0, -106(%edx)
627L(aligned_16_90bytes):
628 movdqa %xmm0, -90(%edx)
629L(aligned_16_74bytes):
630 movdqa %xmm0, -74(%edx)
631L(aligned_16_58bytes):
632 movdqa %xmm0, -58(%edx)
633L(aligned_16_42bytes):
634 movdqa %xmm0, -42(%edx)
635L(aligned_16_26bytes):
636 movdqa %xmm0, -26(%edx)
637L(aligned_16_10bytes):
638 movq %xmm0, -10(%edx)
639 movw %ax, -2(%edx)
640 SETRTNVAL
641 RETURN
642
Elliott Hughese412f892016-08-01 14:00:45 -0700643 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800644L(aligned_16_123bytes):
645 movdqa %xmm0, -123(%edx)
646L(aligned_16_107bytes):
647 movdqa %xmm0, -107(%edx)
648L(aligned_16_91bytes):
649 movdqa %xmm0, -91(%edx)
650L(aligned_16_75bytes):
651 movdqa %xmm0, -75(%edx)
652L(aligned_16_59bytes):
653 movdqa %xmm0, -59(%edx)
654L(aligned_16_43bytes):
655 movdqa %xmm0, -43(%edx)
656L(aligned_16_27bytes):
657 movdqa %xmm0, -27(%edx)
658L(aligned_16_11bytes):
659 movq %xmm0, -11(%edx)
660 movw %ax, -3(%edx)
661 movb %al, -1(%edx)
662 SETRTNVAL
663 RETURN
664
Elliott Hughese412f892016-08-01 14:00:45 -0700665 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800666L(aligned_16_124bytes):
667 movdqa %xmm0, -124(%edx)
668L(aligned_16_108bytes):
669 movdqa %xmm0, -108(%edx)
670L(aligned_16_92bytes):
671 movdqa %xmm0, -92(%edx)
672L(aligned_16_76bytes):
673 movdqa %xmm0, -76(%edx)
674L(aligned_16_60bytes):
675 movdqa %xmm0, -60(%edx)
676L(aligned_16_44bytes):
677 movdqa %xmm0, -44(%edx)
678L(aligned_16_28bytes):
679 movdqa %xmm0, -28(%edx)
680L(aligned_16_12bytes):
681 movq %xmm0, -12(%edx)
682 movl %eax, -4(%edx)
683 SETRTNVAL
684 RETURN
685
Elliott Hughese412f892016-08-01 14:00:45 -0700686 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800687L(aligned_16_125bytes):
688 movdqa %xmm0, -125(%edx)
689L(aligned_16_109bytes):
690 movdqa %xmm0, -109(%edx)
691L(aligned_16_93bytes):
692 movdqa %xmm0, -93(%edx)
693L(aligned_16_77bytes):
694 movdqa %xmm0, -77(%edx)
695L(aligned_16_61bytes):
696 movdqa %xmm0, -61(%edx)
697L(aligned_16_45bytes):
698 movdqa %xmm0, -45(%edx)
699L(aligned_16_29bytes):
700 movdqa %xmm0, -29(%edx)
701L(aligned_16_13bytes):
702 movq %xmm0, -13(%edx)
703 movl %eax, -5(%edx)
704 movb %al, -1(%edx)
705 SETRTNVAL
706 RETURN
707
Elliott Hughese412f892016-08-01 14:00:45 -0700708 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800709L(aligned_16_126bytes):
710 movdqa %xmm0, -126(%edx)
711L(aligned_16_110bytes):
712 movdqa %xmm0, -110(%edx)
713L(aligned_16_94bytes):
714 movdqa %xmm0, -94(%edx)
715L(aligned_16_78bytes):
716 movdqa %xmm0, -78(%edx)
717L(aligned_16_62bytes):
718 movdqa %xmm0, -62(%edx)
719L(aligned_16_46bytes):
720 movdqa %xmm0, -46(%edx)
721L(aligned_16_30bytes):
722 movdqa %xmm0, -30(%edx)
723L(aligned_16_14bytes):
724 movq %xmm0, -14(%edx)
725 movl %eax, -6(%edx)
726 movw %ax, -2(%edx)
727 SETRTNVAL
728 RETURN
729
Elliott Hughese412f892016-08-01 14:00:45 -0700730 ALIGN(4)
Bruce Beare8ff1a272010-03-04 11:03:37 -0800731L(aligned_16_127bytes):
732 movdqa %xmm0, -127(%edx)
733L(aligned_16_111bytes):
734 movdqa %xmm0, -111(%edx)
735L(aligned_16_95bytes):
736 movdqa %xmm0, -95(%edx)
737L(aligned_16_79bytes):
738 movdqa %xmm0, -79(%edx)
739L(aligned_16_63bytes):
740 movdqa %xmm0, -63(%edx)
741L(aligned_16_47bytes):
742 movdqa %xmm0, -47(%edx)
743L(aligned_16_31bytes):
744 movdqa %xmm0, -31(%edx)
745L(aligned_16_15bytes):
746 movq %xmm0, -15(%edx)
747 movl %eax, -7(%edx)
748 movw %ax, -3(%edx)
749 movb %al, -1(%edx)
750 SETRTNVAL
751 RETURN_END
752
Haibo Huangb9244ff2018-08-11 10:12:13 -0700753END(memset_generic)