conservatory
/
femtolisp
4
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fixing large int64 comparison bug

This commit is contained in:
Jeff Bezanson 2012-03-15 00:21:39 -04:00
parent c019b3bf2d
commit 72d1ee7522
2 changed files with 11 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
operators.c
View File

@ -206,9 +206,10 @@ int cmp_lt(void *a, numerictype_t atag, void *b, numerictype_t btag)
return 0;
if (atag == T_UINT64) {
// this is safe because if a had been bigger than S64_MAX,
// we would already have concluded that it's bigger than b.
if (btag == T_INT64) {
if (*(int64_t*)b >= 0) {
return (*(uint64_t*)a < (uint64_t)*(int64_t*)b);
}
return ((int64_t)*(uint64_t*)a < *(int64_t*)b);
}
else if (btag == T_DOUBLE) {
@ -218,6 +219,9 @@ int cmp_lt(void *a, numerictype_t atag, void *b, numerictype_t btag)
}
else if (atag == T_INT64) {
if (btag == T_UINT64) {
if (*(int64_t*)a >= 0) {
return ((uint64_t)*(int64_t*)a < *(uint64_t*)b);
}
return (*(int64_t*)a < (int64_t)*(uint64_t*)b);
}
else if (btag == T_DOUBLE) {
@ -225,20 +229,14 @@ int cmp_lt(void *a, numerictype_t atag, void *b, numerictype_t btag)
return (*(int64_t*)a < (int64_t)*(double*)b);
}
}
else if (btag == T_UINT64) {
if (atag == T_INT64) {
return ((int64_t)*(uint64_t*)b > *(int64_t*)a);
}
else if (atag == T_DOUBLE) {
if (btag == T_UINT64) {
if (atag == T_DOUBLE) {
if (da != da) return 0;
return (*(uint64_t*)b > (uint64_t)*(double*)a);
}
}
else if (btag == T_INT64) {
if (atag == T_UINT64) {
return (*(int64_t*)b > (int64_t)*(uint64_t*)a);
}
else if (atag == T_DOUBLE) {
if (atag == T_DOUBLE) {
if (da != da) return 0;
return (*(int64_t*)b > (int64_t)*(double*)a);
}
@ -303,62 +301,3 @@ int cmp_eq(void *a, numerictype_t atag, void *b, numerictype_t btag,
}
return 1;
}
#ifdef ENABLE_LLT_TEST
void test_operators()
{
int8_t i8, i8b;
uint8_t ui8, ui8b;
int16_t i16, i16b;
uint16_t ui16, ui16b;
int32_t i32, i32b;
uint32_t ui32, ui32b;
int64_t i64, i64b;
uint64_t ui64, ui64b;
float f, fb;
double d, db;
ui64 = U64_MAX;
ui64b = U64_MAX-1;
i64 = S64_MIN;
i64b = i64+1;
d = (double)ui64;
db = (double)i64b;
assert(cmp_lt(&i64, T_INT64, &ui64, T_UINT64));
assert(!cmp_lt(&ui64, T_UINT64, &i64, T_INT64));
assert(cmp_lt(&i64, T_INT64, &ui64b, T_UINT64));
assert(!cmp_lt(&ui64b, T_UINT64, &i64, T_INT64));
assert(cmp_lt(&i64, T_INT64, &i64b, T_INT64));
assert(!cmp_lt(&i64b, T_INT64, &i64, T_INT64));
// try to compare a double too big to fit in an int64 with an
// int64 requiring too much precision to fit in a double...
// this case fails but it's very difficult/expensive to support
//assert(cmp_lt(&ui64b, T_UINT64, &d, T_DOUBLE));
i64 = S64_MAX;
ui64 = S64_MAX-1;
assert(cmp_lt(&ui64, T_UINT64, &i64, T_INT64));
assert(!cmp_lt(&i64, T_INT64, &ui64, T_UINT64));
i64 = S64_MAX-1;
ui64 = S64_MAX;
assert(cmp_lt(&i64, T_INT64, &ui64, T_UINT64));
assert(!cmp_lt(&ui64, T_UINT64, &i64, T_INT64));
d = DBL_MAXINT;
i64 = DBL_MAXINT+100;
assert(cmp_lt(&d, T_DOUBLE, &i64, T_INT64));
assert(!cmp_lt(&i64, T_INT64, &d, T_DOUBLE));
i64 = DBL_MAXINT+10;
assert(cmp_lt(&d, T_DOUBLE, &i64, T_INT64));
assert(!cmp_lt(&i64, T_INT64, &d, T_DOUBLE));
i64 = DBL_MAXINT+1;
assert(cmp_lt(&d, T_DOUBLE, &i64, T_INT64));
assert(!cmp_lt(&i64, T_INT64, &d, T_DOUBLE));
assert(!cmp_eq(&d, T_DOUBLE, &i64, T_INT64, 0));
i64 = DBL_MAXINT;
assert(cmp_eq(&d, T_DOUBLE, &i64, T_INT64, 0));
}
#endif

2
tests/unittest.lsp
View File

@ -76,6 +76,8 @@
(assert (equal? (string 'sym #byte(65) #wchar(945) "blah") "symA\u03B1blah"))
(assert (> 9223372036854775808 9223372036854775807))
; NaNs
(assert (equal? +nan.0 +nan.0))
(assert (not (= +nan.0 +nan.0)))