commit 863e80b4451b5102b41bebf9ddca3a420de746fa
author Bram Moolenaar <Bram@vim.org> Sun Jun 04 20:30:00 2017 +0200
committer Bram Moolenaar <Bram@vim.org> Sun Jun 04 20:30:00 2017 +0200
tree b7d40ecfa78b45b2fac1ba856028d8a40423f128
parent 3e54569b17683318e0cb6693ab0024c2ad1e3e8f

patch 8.0.0614: float2nr() is not exactly right

Problem:    float2nr() is not exactly right.
Solution:   Make float2nr() more accurate.  Turn test64 into a new style test.
            (Hirohito Higashi, closes #1688)

src/evalfunc.c

diff --git a/src/evalfunc.c b/src/evalfunc.c
index b16b260..60cd344 100644
--- a/src/evalfunc.c
+++ b/src/evalfunc.c
@@ -3352,9 +3352,9 @@
 
     if (get_float_arg(argvars, &f) == OK)
     {
-	if (f < -VARNUM_MAX)
+	if (f <= -VARNUM_MAX + DBL_EPSILON)
 	    rettv->vval.v_number = -VARNUM_MAX;
-	else if (f > VARNUM_MAX)
+	else if (f >= VARNUM_MAX - DBL_EPSILON)
 	    rettv->vval.v_number = VARNUM_MAX;
 	else
 	    rettv->vval.v_number = (varnumber_T)f;