libc/arch-mips/string/memmove.c

/*
 * Copyright (c) 2017 Imagination Technologies.
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 *      * Redistributions of source code must retain the above copyright
 *        notice, this list of conditions and the following disclaimer.
 *      * Redistributions in binary form must reproduce the above copyright
 *        notice, this list of conditions and the following disclaimer
 *        in the documentation and/or other materials provided with
 *        the distribution.
 *      * Neither the name of Imagination Technologies nor the names of its
 *        contributors may be used to endorse or promote products derived
 *        from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <string.h>

#if !defined(UNALIGNED_INSTR_SUPPORT)
/* does target have unaligned lw/ld/ualw/uald instructions? */
#define UNALIGNED_INSTR_SUPPORT 0
#if __mips_isa_rev < 6 && !__mips1
#undef UNALIGNED_INSTR_SUPPORT
#define UNALIGNED_INSTR_SUPPORT 1
#endif
#endif

#if !defined(HW_UNALIGNED_SUPPORT)
/* Does target have hardware support for unaligned accesses?  */
#define HW_UNALIGNED_SUPPORT 0
#if __mips_isa_rev >= 6
#undef HW_UNALIGNED_SUPPORT
#define HW_UNALIGNED_SUPPORT 1
#endif
#endif

#define ENABLE_PREFETCH     1

#if ENABLE_PREFETCH
#define PREFETCH(addr)  __builtin_prefetch (addr, 0, 1);
#else
#define PREFETCH(addr)
#endif

#if _MIPS_SIM == _ABIO32
typedef unsigned long reg_t;
typedef struct
{
  reg_t B0:8, B1:8, B2:8, B3:8;
} bits_t;
#else
typedef unsigned long long reg_t;
typedef struct
{
  reg_t B0:8, B1:8, B2:8, B3:8, B4:8, B5:8, B6:8, B7:8;
} bits_t;
#endif

typedef union
{
  reg_t v;
  bits_t b;
} bitfields_t;

#define DO_BYTE(a, i)   \
  a[i] = bw.b.B##i;     \
  len--;                \
  if(!len) return ret;  \

/* This code is called when aligning a pointer, there are remaining bytes
   after doing word compares, or architecture does not have some form
   of unaligned support.  */
static inline void * __attribute__ ((always_inline))
do_bytes (void *a, const void *b, unsigned long len, void *ret)
{
  unsigned char *x = (unsigned char *) a;
  unsigned char *y = (unsigned char *) b;
  unsigned long i;

  /* 'len' might be zero here, so preloading the first two values
     before the loop may access unallocated memory.  */
  for (i = 0; i < len; i++)
  {
    *x = *y;
    x++;
    y++;
  }
  return ret;
}

static inline void * __attribute__ ((always_inline))
do_bytes_backward (void *a, const void *b, unsigned long len, void *ret)
{
  unsigned char *x = (unsigned char *) a;
  unsigned char *y = (unsigned char *) b;
  unsigned long i;

  /* 'len' might be zero here, so preloading the first two values
     before the loop may access unallocated memory.  */
  for (i = 0; i < len; i++) {
    *--x = *--y;
  }
  return ret;
}

static inline void * __attribute__ ((always_inline))
do_bytes_aligned (void *a, const void *b, unsigned long len, void *ret)
{
  unsigned char *x = (unsigned char *) a;

  if(len > 0) {
    bitfields_t bw;
    bw.v = *((reg_t*) b);

#if __mips64
    DO_BYTE(x, 0);
    DO_BYTE(x, 1);
    DO_BYTE(x, 2);
    DO_BYTE(x, 3);
    DO_BYTE(x, 4);
    DO_BYTE(x, 5);
    DO_BYTE(x, 6);
    DO_BYTE(x, 7);
#else
    DO_BYTE(x, 0);
    DO_BYTE(x, 1);
    DO_BYTE(x, 2);
    DO_BYTE(x, 3);
#endif
  }

  return ret;
}

#if !HW_UNALIGNED_SUPPORT
#if UNALIGNED_INSTR_SUPPORT
/* for MIPS GCC, there are no unaligned builtins - so this struct forces
   the compiler to treat the pointer access as unaligned.  */
struct ulw
{
  reg_t uli;
} __attribute__ ((packed));

#define STORE_UNALIGNED_8(a, b)                      \
{                                                    \
  reg_t y0 = b[0], y1 = b[1], y2 = b[2], y3 = b[3];  \
  reg_t y4 = b[4], y5 = b[5], y6 = b[6], y7 = b[7];  \
  a[0].uli = y0;                                     \
  a[1].uli = y1;                                     \
  a[2].uli = y2;                                     \
  a[3].uli = y3;                                     \
  a[4].uli = y4;                                     \
  a[5].uli = y5;                                     \
  a[6].uli = y6;                                     \
  a[7].uli = y7;                                     \
}

#define STORE_UNALIGNED_4(a, b)                      \
{                                                    \
  reg_t y0 = b[0], y1 = b[1], y2 = b[2], y3 = b[3];  \
  a[0].uli = y0;                                     \
  a[1].uli = y1;                                     \
  a[2].uli = y2;                                     \
  a[3].uli = y3;                                     \
}

/* first pointer is not aligned while second pointer is.  */
static void *
unaligned_words_forward (struct ulw *a, const reg_t * b,
                         unsigned long words, unsigned long bytes, void *ret)
{
#if ((_MIPS_SIM == _ABIO32) || _MIPS_TUNE_I6400)
  unsigned long i, words_by_8, words_by_1;
  words_by_1 = words % 8;
  words_by_8 = words >> 3;
  for (; words_by_8 > 0; words_by_8--) {
    if(words_by_8 != 1)
      PREFETCH (b + 8);
    STORE_UNALIGNED_8(a, b);
    a += 8;
    b += 8;
  }
#else
  unsigned long i, words_by_4, words_by_1;
  words_by_1 = words % 4;
  words_by_4 = words >> 2;
  for (; words_by_4 > 0; words_by_4--) {
    if(words_by_4 != 1)
      PREFETCH (b + 4);
    STORE_UNALIGNED_4(a, b);
    a += 4;
    b += 4;
  }
#endif

  /* do remaining words.  */
  for (i = 0; i < words_by_1; i++) {
    a->uli = *b;
    a += 1;
    b += 1;
  }

  /* mop up any remaining bytes.  */
  return do_bytes_aligned (a, b, bytes, ret);
}

static void *
unaligned_words_backward (struct ulw *a, const reg_t * b,
                          unsigned long words, unsigned long bytes, void *ret)
{
#if ((_MIPS_SIM == _ABIO32) || _MIPS_TUNE_I6400)
  unsigned long i, words_by_8, words_by_1;
  words_by_1 = words % 8;
  words_by_8 = words >> 3;
  for (; words_by_8 > 0; words_by_8--) {
    if(words_by_8 != 1)
      PREFETCH (b - 16);
    a -= 8;
    b -= 8;
    STORE_UNALIGNED_8(a, b);
  }
#else
  unsigned long i, words_by_4, words_by_1;
  words_by_1 = words % 4;
  words_by_4 = words >> 2;
  for (; words_by_4 > 0; words_by_4--) {
    if(words_by_4 != 1)
      PREFETCH (b - 8);
    a -= 4;
    b -= 4;
    STORE_UNALIGNED_4(a, b);
  }
#endif

  /* do remaining words.  */
  for (i = 0; i < words_by_1; i++) {
    a -= 1;
    b -= 1;
    a->uli = *b;
  }

  /* mop up any remaining bytes.  */
  return do_bytes_backward (a, b, bytes, ret);
}

#else
/* no HW support or unaligned lw/ld/ualw/uald instructions.  */
static void *
unaligned_words_forward (reg_t * a, const reg_t * b,
                         unsigned long words, unsigned long bytes, void *ret)
{
  return do_bytes_aligned (a, b, (sizeof (reg_t) * words) + bytes, ret);
}

static void *
unaligned_words_backward (reg_t * a, const reg_t * b,
                          unsigned long words, unsigned long bytes, void *ret)
{
  return do_bytes_backward (a, b, (sizeof (reg_t) * words) + bytes, ret);
}

#endif /* UNALIGNED_INSTR_SUPPORT */
#endif /* HW_UNALIGNED_SUPPORT */

/* both pointers are aligned, or first isn't and HW support for unaligned.  */

#define STORE_ALIGNED_8(a, b)                        \
{                                                    \
  reg_t x0 = b[0], x1 = b[1], x2 = b[2], x3 = b[3];  \
  reg_t x4 = b[4], x5 = b[5], x6 = b[6], x7 = b[7];  \
  a[0] = x0;                                         \
  a[1] = x1;                                         \
  a[2] = x2;                                         \
  a[3] = x3;                                         \
  a[4] = x4;                                         \
  a[5] = x5;                                         \
  a[6] = x6;                                         \
  a[7] = x7;                                         \
}

#define STORE_ALIGNED_4(a, b)                        \
{                                                    \
  reg_t x0 = b[0], x1 = b[1], x2 = b[2], x3 = b[3];  \
  a[0] = x0;                                         \
  a[1] = x1;                                         \
  a[2] = x2;                                         \
  a[3] = x3;                                         \
}

static void *
aligned_words_forward (reg_t * a, const reg_t * b,
                       unsigned long words, unsigned long bytes, void *ret)
{
#if ((_MIPS_SIM == _ABIO32) || _MIPS_TUNE_I6400)
  unsigned long i, words_by_8, words_by_1;
  words_by_1 = words % 8;
  words_by_8 = words >> 3;
  for (; words_by_8 > 0; words_by_8--) {
    if(words_by_8 != 1)
      PREFETCH (b + 8);
    STORE_ALIGNED_8(a, b);
    a += 8;
    b += 8;
  }
#else
  unsigned long i, words_by_4, words_by_1;
  words_by_1 = words % 4;
  words_by_4 = words >> 2;
  for (; words_by_4 > 0; words_by_4--) {
    if(words_by_4 != 1)
      PREFETCH (b + 4);
    STORE_ALIGNED_4(a, b);
    a += 4;
    b += 4;
  }
#endif

  /* do remaining words.  */
  for (i = 0; i < words_by_1; i++) {
    *a = *b;
    a += 1;
    b += 1;
  }

  /* mop up any remaining bytes.  */
  return do_bytes_aligned (a, b, bytes, ret);
}


static void *
aligned_words_backward (reg_t * a, const reg_t * b,
                        unsigned long words, unsigned long bytes, void *ret)
{
#if ((_MIPS_SIM == _ABIO32) || _MIPS_TUNE_I6400)
  unsigned long i, words_by_8, words_by_1;
  words_by_1 = words % 8;
  words_by_8 = words >> 3;
  for (; words_by_8 > 0; words_by_8--) {
    if(words_by_8 != 1)
      PREFETCH (b - 16);
    a -= 8;
    b -= 8;
    STORE_ALIGNED_8(a, b);
  }
#else
  unsigned long i, words_by_4, words_by_1;
  words_by_1 = words % 4;
  words_by_4 = words >> 2;
  for (; words_by_4 > 0; words_by_4--) {
    if(words_by_4 != 1)
      PREFETCH (b - 8);
    a -= 4;
    b -= 4;
    STORE_ALIGNED_4(a, b);
  }
#endif

  /* do remaining words.  */
  for (i = 0; i < words_by_1; i++) {
    a -= 1;
    b -= 1;
    *a = *b;
  }

  /* mop up any remaining bytes.  */
  return do_bytes_backward (a, b, bytes, ret);
}

void *
memmove (void *dst0, const void *src0, size_t length) __overloadable
{
  unsigned long bytes, words;
  void *ret = dst0;

  if (length == 0 || dst0 == src0)      /* nothing to do */
    return dst0;

  if ((unsigned long)dst0 < (unsigned long)src0) {
    /* Copy forwards. */
    /* This shouldn't hit that often. */
    if (length < sizeof (reg_t) * 4) {
      return do_bytes (dst0, src0, length, ret);
    }

    /* Align the second pointer to word/dword alignment.
       Note that the pointer is only 32-bits for o32/n32 ABIs. For
       n32, loads are done as 64-bit while address remains 32-bit.   */
    bytes = ((unsigned long) src0) % sizeof (reg_t);
    if (bytes) {
      bytes = sizeof (reg_t) - bytes;
      if (bytes > length)
        bytes = length;
      do_bytes (dst0, src0, bytes, ret);
      if (length == bytes)
        return ret;
      length -= bytes;
      dst0 = (void *) (((unsigned char *) dst0) + bytes);
      src0 = (const void *) (((unsigned char *) src0) + bytes);
    }

    /* Second pointer now aligned.  */
    words = length / sizeof (reg_t);
    bytes = length % sizeof (reg_t);
#if HW_UNALIGNED_SUPPORT
    /* treat possible unaligned first pointer as aligned.  */
    return aligned_words_forward (dst0, src0, words, bytes, ret);
#else
    if (((unsigned long) dst0) % sizeof (reg_t) == 0) {
      return aligned_words_forward (dst0, src0, words, bytes, ret);
    }
    /* need to use unaligned instructions on first pointer.  */
    return unaligned_words_forward (dst0, src0, words, bytes, ret);
#endif
  } else {
    /* Copy backwards. */
    dst0 = (void *) (((unsigned char *) dst0) + length);
    src0 = (const void *) (((unsigned char *) src0) + length);

    /* This shouldn't hit that often. */
    if (length < sizeof (reg_t) * 4) {
      return do_bytes_backward (dst0, src0, length, ret);
    }

    /* Align the second pointer to word/dword alignment.
       Note that the pointer is only 32-bits for o32/n32 ABIs. For
       n32, loads are done as 64-bit while address remains 32-bit.   */
    bytes = ((unsigned long) src0) % sizeof (reg_t);
    if (bytes) {
      if (bytes > length)
        bytes = length;
      do_bytes_backward (dst0, src0, bytes, ret);
      if (length == bytes)
        return ret;
      length -= bytes;
      dst0 = (void *) (((unsigned char *) dst0) - bytes);
      src0 = (const void *) (((unsigned char *) src0) - bytes);
    }

    words = length / sizeof (reg_t);
    bytes = length % sizeof (reg_t);
#if HW_UNALIGNED_SUPPORT
    /* treat possible unaligned first pointer as aligned.  */
    return aligned_words_backward ((void *)dst0, (void *)src0, words, bytes, ret);
#else
    if (((unsigned long) dst0) % sizeof (reg_t) == 0) {
      return aligned_words_backward (dst0, src0, words, bytes, ret);
    }
    /* need to use unaligned instructions on first pointer.  */
    return unaligned_words_backward (dst0, src0, words, bytes, ret);
#endif
  }
}