stdlib: Fix strtol, strtoul

Correct return value on signed long int overflow/underflow.
Correct handling of "0x" prefix not followed by a hex digit.

JIRA: RTOS-869

stdlib/strtoul.c

@@ -19,7 +19,7 @@
 #include <limits.h>
 
 
-unsigned long int strtoul(const char *nptr, char **endptr, int base)
+static unsigned long int strtoul_common(const char *nptr, char **endptr, int base, int isUnsigned)
 {
 	unsigned long int cutoff, result = 0;
 	int cutlim, t, width = 0, negative = 0;
@@ -37,11 +37,12 @@ unsigned long int strtoul(const char *nptr, char **endptr, int base)
 		sptr++;
 	}
 
-	if ((base == 16 || base == 0) && sptr[0] == '0' && (sptr[1] | 0x20) == 'x') {
+	if ((base == 16 || base == 0) && sptr[0] == '0' && (sptr[1] | 0x20) == 'x' && isxdigit(sptr[2])) {
 		base = 16;
 		sptr += 2;
 	}
-	else if (base == 0 && sptr[0] == '0') {
+
+	if (base == 0 && sptr[0] == '0') {
 		base = 8;
 	}
 	else if (base == 0) {
@@ -53,8 +54,15 @@ unsigned long int strtoul(const char *nptr, char **endptr, int base)
 		return 0;
 	}
 
-	cutoff = (unsigned long int)(ULONG_MAX) / (unsigned long int)base;
-	cutlim = (unsigned long int)(ULONG_MAX) - (cutoff * (unsigned long int)base);
+	if (isUnsigned != 0) {
+		cutoff = ULONG_MAX;
+	}
+	else {
+		cutoff = (negative != 0) ? -LONG_MIN : LONG_MAX;
+	}
+
+	cutlim = (int)(cutoff % base);
+	cutoff /= base;
 
 	while (isalnum(*sptr) != 0) {
 		t = *sptr - '0';
@@ -78,7 +86,12 @@ unsigned long int strtoul(const char *nptr, char **endptr, int base)
 
 	if (width < 0) {
 		errno = ERANGE;
-		result = ULONG_MAX;
+		if (isUnsigned != 0) {
+			result = ULONG_MAX;
+		}
+		else {
+			result = (negative != 0) ? LONG_MIN : LONG_MAX;
+		}
 	}
 	else if (width == 0) {
 		errno = EINVAL;
@@ -95,9 +108,15 @@ unsigned long int strtoul(const char *nptr, char **endptr, int base)
 }
 
 
+unsigned long int strtoul(const char *nptr, char **endptr, int base)
+{
+	return strtoul_common(nptr, endptr, base, 1);
+}
+
+
 long int strtol(const char *nptr, char **endptr, int base)
 {
-	return (long int)strtoul(nptr, endptr, base);
+	return (long int)strtoul_common(nptr, endptr, base, 0);
 }