perf(libc): use builtin implementations where possible
When conditions are right, eg a small memcpy of a known size and
alignment, the compiler may know of a sequence that is optimal for the
given constraints and inline it. If the compiler doesn't find one, it
will emit a call to the generic function (in the libc) which will
implement this in the most generic and unconstrained manner. That
generic function is rarely the most optimal when constraints are known.
So give the compiler a chance to do this. Replace calls to libc
functions that have builtins to the builtin and keep the generic
implementation if it decides to emit a call anyway.
And example of this in action is usage of FEAT_MOPS. When the compiler
is aware of the feature (-march=armv8.8-a) then it will emit the 3 MOPS
instructions instead of calls to our memcpy() and memset()
implementations.
Change-Id: I9860cfada1d941b613ebd4da068e9992c387952e
Signed-off-by: Boyan Karatotev <boyan.karatotev@arm.com>
diff --git a/include/lib/libc/string.h b/include/lib/libc/string.h
index bba9816..098fded 100644
--- a/include/lib/libc/string.h
+++ b/include/lib/libc/string.h
@@ -4,7 +4,7 @@
* SPDX-License-Identifier: BSD-3-Clause
*/
/*
- * Portions copyright (c) 2018-2020, Arm Limited and Contributors.
+ * Portions copyright (c) 2018-2025, Arm Limited and Contributors.
* Portions copyright (c) 2023, Intel Corporation. All rights reserved.
* All rights reserved.
*/
@@ -14,19 +14,26 @@
#include <stddef.h>
-void *memcpy(void *dst, const void *src, size_t len);
+/*
+ * When conditions are right, the compiler may have a baked-in call that can be
+ * inlined and that will be much more optimal than our generic implementation.
+ * When it doesn't, it will emit a call to the original function for which we
+ * provide an implementation.
+ */
+#define memcpy __builtin_memcpy
+#define memset __builtin_memset
+#define memcmp __builtin_memcmp
+#define memchr __builtin_memchr
+#define strcmp __builtin_strcmp
+#define strncmp __builtin_strncmp
+#define strchr __builtin_strchr
+#define strlen __builtin_strlen
+#define strrchr __builtin_strrchr
+
int memcpy_s(void *dst, size_t dsize, void *src, size_t ssize);
void *memmove(void *dst, const void *src, size_t len);
-int memcmp(const void *s1, const void *s2, size_t len);
-int strcmp(const char *s1, const char *s2);
-int strncmp(const char *s1, const char *s2, size_t n);
-void *memchr(const void *src, int c, size_t len);
void *memrchr(const void *src, int c, size_t len);
-char *strchr(const char *s, int c);
-void *memset(void *dst, int val, size_t count);
-size_t strlen(const char *s);
size_t strnlen(const char *s, size_t maxlen);
-char *strrchr(const char *p, int ch);
size_t strlcpy(char * dst, const char * src, size_t dsize);
size_t strlcat(char * dst, const char * src, size_t dsize);
char *strtok_r(char *s, const char *delim, char **last);
diff --git a/include/lib/libc/string_private.h b/include/lib/libc/string_private.h
new file mode 100644
index 0000000..da85fae
--- /dev/null
+++ b/include/lib/libc/string_private.h
@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2025, Arm Limited and Contributors. All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#ifndef STRING_PRIVATE_H
+#define STRING_PRIVATE_H
+
+/* Do not include outside of the libc. Use string.h instead. */
+
+#include <stddef.h>
+
+int memcmp(const void *s1, const void *s2, size_t len);
+int strcmp(const char *s1, const char *s2);
+int strncmp(const char *s1, const char *s2, size_t n);
+void *memchr(const void *src, int c, size_t len);
+char *strchr(const char *s, int c);
+void *memset(void *dst, int val, size_t count);
+size_t strlen(const char *s);
+char *strrchr(const char *p, int ch);
+
+#endif /* STRING_PRIVATE_H */
diff --git a/lib/libc/memchr.c b/lib/libc/memchr.c
index 66d7ba1..e009a5f 100644
--- a/lib/libc/memchr.c
+++ b/lib/libc/memchr.c
@@ -1,11 +1,11 @@
/*
- * Copyright (c) 2013-2019, Arm Limited and Contributors. All rights reserved.
+ * Copyright (c) 2013-2025, Arm Limited and Contributors. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
#include <stddef.h>
-#include <string.h>
+#include <string_private.h>
void *memchr(const void *src, int c, size_t len)
{
diff --git a/lib/libc/memcmp.c b/lib/libc/memcmp.c
index db2701b..1458208 100644
--- a/lib/libc/memcmp.c
+++ b/lib/libc/memcmp.c
@@ -1,11 +1,11 @@
/*
- * Copyright (c) 2013-2019, Arm Limited and Contributors. All rights reserved.
+ * Copyright (c) 2013-2025, Arm Limited and Contributors. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
#include <stddef.h>
-#include <string.h>
+#include <string_private.h>
int memcmp(const void *s1, const void *s2, size_t len)
{
diff --git a/lib/libc/memcpy.c b/lib/libc/memcpy.c
index af9ed45..ca31de5 100644
--- a/lib/libc/memcpy.c
+++ b/lib/libc/memcpy.c
@@ -1,11 +1,11 @@
/*
- * Copyright (c) 2013-2019, Arm Limited and Contributors. All rights reserved.
+ * Copyright (c) 2013-2025, Arm Limited and Contributors. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
#include <stddef.h>
-#include <string.h>
+#include <string_private.h>
void *memcpy(void *dst, const void *src, size_t len)
{
diff --git a/lib/libc/memcpy_s.c b/lib/libc/memcpy_s.c
index 26953bf..ee87637 100644
--- a/lib/libc/memcpy_s.c
+++ b/lib/libc/memcpy_s.c
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2013-2023, Arm Limited and Contributors. All rights reserved.
+ * Copyright (c) 2013-2025, Arm Limited and Contributors. All rights reserved.
* Copyright (c) 2023, Intel Corporation. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -53,12 +53,7 @@
}
}
- /*
- * Start copy process when there is no error
- */
- while (ssize--) {
- d[ssize] = s[ssize];
- }
+ (void)memcpy(dst, src, ssize);
return 0;
}
diff --git a/lib/libc/memset.c b/lib/libc/memset.c
index c5bac8d..2513221 100644
--- a/lib/libc/memset.c
+++ b/lib/libc/memset.c
@@ -1,11 +1,11 @@
/*
- * Copyright (c) 2013-2020, Arm Limited and Contributors. All rights reserved.
+ * Copyright (c) 2013-2025, Arm Limited and Contributors. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
#include <stddef.h>
-#include <string.h>
+#include <string_private.h>
#include <stdint.h>
void *memset(void *dst, int val, size_t count)
diff --git a/lib/libc/strchr.c b/lib/libc/strchr.c
index 1cd03ca..b3bebe4 100644
--- a/lib/libc/strchr.c
+++ b/lib/libc/strchr.c
@@ -30,12 +30,12 @@
*/
/*
- * Portions copyright (c) 2018, Arm Limited and Contributors.
+ * Portions copyright (c) 2018-2025, Arm Limited and Contributors.
* All rights reserved.
*/
#include <stddef.h>
-#include <string.h>
+#include <string_private.h>
char *
strchr(const char *p, int ch)
diff --git a/lib/libc/strcmp.c b/lib/libc/strcmp.c
index 290db4c..5afd0e9 100644
--- a/lib/libc/strcmp.c
+++ b/lib/libc/strcmp.c
@@ -33,11 +33,11 @@
*/
/*
- * Portions copyright (c) 2018, Arm Limited and Contributors.
+ * Portions copyright (c) 2018-2025, Arm Limited and Contributors.
* All rights reserved.
*/
-#include <string.h>
+#include <string_private.h>
/*
* Compare strings.
diff --git a/lib/libc/strlen.c b/lib/libc/strlen.c
index e4b79d9..cc1d1df 100644
--- a/lib/libc/strlen.c
+++ b/lib/libc/strlen.c
@@ -1,10 +1,10 @@
/*
- * Copyright (c) 2018, Arm Limited and Contributors. All rights reserved.
+ * Copyright (c) 2018-2025, Arm Limited and Contributors. All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
-#include <string.h>
+#include <string_private.h>
size_t strlen(const char *s)
{
diff --git a/lib/libc/strncmp.c b/lib/libc/strncmp.c
index f0bbadc..828e0c5 100644
--- a/lib/libc/strncmp.c
+++ b/lib/libc/strncmp.c
@@ -30,11 +30,11 @@
*/
/*
- * Portions copyright (c) 2018, Arm Limited and Contributors.
+ * Portions copyright (c) 2018-2025, Arm Limited and Contributors.
* All rights reserved.
*/
-#include <string.h>
+#include <string_private.h>
int
strncmp(const char *s1, const char *s2, size_t n)
diff --git a/lib/libc/strrchr.c b/lib/libc/strrchr.c
index cd435ff..98f76c6 100644
--- a/lib/libc/strrchr.c
+++ b/lib/libc/strrchr.c
@@ -30,7 +30,7 @@
*/
#include <stddef.h>
-#include <string.h>
+#include <string_private.h>
char *
strrchr(const char *p, int ch)