blob: 15253da8a681fc6c39f64f0be29c1abf522c2ff3 [file] [log] [blame]
Varvara Rainchika020a242014-04-29 17:44:56 +04001/*
2Copyright (c) 2014, Intel Corporation
3All rights reserved.
4
5Redistribution and use in source and binary forms, with or without
6modification, are permitted provided that the following conditions are met:
7
8 * Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10
11 * Redistributions in binary form must reproduce the above copyright notice,
12 * this list of conditions and the following disclaimer in the documentation
13 * and/or other materials provided with the distribution.
14
15 * Neither the name of Intel Corporation nor the names of its contributors
16 * may be used to endorse or promote products derived from this software
17 * without specific prior written permission.
18
19THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
20ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
23ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
24(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
25LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
26ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
28SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29*/
30
Elliott Hughes61c95fe2016-03-02 16:39:29 -080031#include <private/bionic_asm.h>
Varvara Rainchika020a242014-04-29 17:44:56 +040032
Elliott Hughes61c95fe2016-03-02 16:39:29 -080033#include "cache.h"
Varvara Rainchika020a242014-04-29 17:44:56 +040034
35#ifndef L
36# define L(label) .L##label
37#endif
38
39#ifndef ALIGN
40# define ALIGN(n) .p2align n
41#endif
42
Varvara Rainchika020a242014-04-29 17:44:56 +040043
Elliott Hughes61c95fe2016-03-02 16:39:29 -080044ENTRY(__memset_chk)
45 # %rdi = dst, %rsi = byte, %rdx = n, %rcx = dst_len
46 cmp %rdx, %rcx
47 jl memset
Varvara Rainchika020a242014-04-29 17:44:56 +040048
Elliott Hughes61c95fe2016-03-02 16:39:29 -080049 # TODO: include __memset_chk_fail in the backtrace?
50 call PIC_PLT(__memset_chk_fail)
51END(__memset_chk)
Varvara Rainchika020a242014-04-29 17:44:56 +040052
Varvara Rainchika020a242014-04-29 17:44:56 +040053
54 .section .text.sse2,"ax",@progbits
Elliott Hughes61c95fe2016-03-02 16:39:29 -080055ENTRY(memset)
Varvara Rainchika020a242014-04-29 17:44:56 +040056 movq %rdi, %rax
57#ifdef USE_AS_BZERO_P
58 mov %rsi, %rdx
59 xor %rcx, %rcx
60#else
61 and $0xff, %rsi
62 mov $0x0101010101010101, %rcx
63 imul %rsi, %rcx
64#endif
65 cmpq $16, %rdx
66 jae L(16bytesormore)
67 testb $8, %dl
68 jnz L(8_15bytes)
69 testb $4, %dl
70 jnz L(4_7bytes)
71 testb $2, %dl
72 jnz L(2_3bytes)
73 testb $1, %dl
74 jz L(return)
75 movb %cl, (%rdi)
76L(return):
77 ret
78
79L(8_15bytes):
80 movq %rcx, (%rdi)
81 movq %rcx, -8(%rdi, %rdx)
82 ret
83
84L(4_7bytes):
85 movl %ecx, (%rdi)
86 movl %ecx, -4(%rdi, %rdx)
87 ret
88
89L(2_3bytes):
90 movw %cx, (%rdi)
91 movw %cx, -2(%rdi, %rdx)
92 ret
93
94 ALIGN (4)
95L(16bytesormore):
96#ifdef USE_AS_BZERO_P
97 pxor %xmm0, %xmm0
98#else
99 movd %rcx, %xmm0
100 pshufd $0, %xmm0, %xmm0
101#endif
102 movdqu %xmm0, (%rdi)
103 movdqu %xmm0, -16(%rdi, %rdx)
104 cmpq $32, %rdx
105 jbe L(32bytesless)
106 movdqu %xmm0, 16(%rdi)
107 movdqu %xmm0, -32(%rdi, %rdx)
108 cmpq $64, %rdx
109 jbe L(64bytesless)
110 movdqu %xmm0, 32(%rdi)
111 movdqu %xmm0, 48(%rdi)
112 movdqu %xmm0, -64(%rdi, %rdx)
113 movdqu %xmm0, -48(%rdi, %rdx)
114 cmpq $128, %rdx
115 ja L(128bytesmore)
116L(32bytesless):
117L(64bytesless):
118 ret
119
120 ALIGN (4)
121L(128bytesmore):
122 leaq 64(%rdi), %rcx
123 andq $-64, %rcx
124 movq %rdx, %r8
125 addq %rdi, %rdx
126 andq $-64, %rdx
127 cmpq %rcx, %rdx
128 je L(return)
129
130#ifdef SHARED_CACHE_SIZE
131 cmp $SHARED_CACHE_SIZE, %r8
132#else
133 cmp __x86_64_shared_cache_size(%rip), %r8
134#endif
135 ja L(128bytesmore_nt)
136
137 ALIGN (4)
138L(128bytesmore_normal):
139 movdqa %xmm0, (%rcx)
140 movaps %xmm0, 0x10(%rcx)
141 movaps %xmm0, 0x20(%rcx)
142 movaps %xmm0, 0x30(%rcx)
143 addq $64, %rcx
144 cmpq %rcx, %rdx
145 jne L(128bytesmore_normal)
146 ret
147
148 ALIGN (4)
149L(128bytesmore_nt):
150 movntdq %xmm0, (%rcx)
151 movntdq %xmm0, 0x10(%rcx)
152 movntdq %xmm0, 0x20(%rcx)
153 movntdq %xmm0, 0x30(%rcx)
154 leaq 64(%rcx), %rcx
155 cmpq %rcx, %rdx
156 jne L(128bytesmore_nt)
157 sfence
158 ret
159
Elliott Hughes61c95fe2016-03-02 16:39:29 -0800160END(memset)