commit d5cec1f1f055316c353cfa15ad8d5eb0952d50a0
author =?UTF-8?q?Dundar=20G=C3=B6c?= <gocdundar@gmail.com> Sat Jan 29 15:19:23 2022 +0000
committer Bram Moolenaar <Bram@vim.org> Sat Jan 29 15:19:23 2022 +0000
tree 94d3ad44783e3442be7293edb3d5890ac08f0377
parent f12b7815f6b55c3e2f37366aa45e723b1fcb2e9a

patch 8.2.4255: theoretical computation overflow

Problem:    Theoretical computation overflow.
Solution:   Perform multiplication in a wider type. (closes #9657)

src/memfile.c

diff --git a/src/memfile.c b/src/memfile.c
index b5ab2d9..de3dd6e 100644
--- a/src/memfile.c
+++ b/src/memfile.c
@@ -249,7 +249,7 @@
 					    // free entries in used list
     for (hp = mfp->mf_used_first; hp != NULL; hp = nextp)
     {
-	total_mem_used -= hp->bh_page_count * mfp->mf_page_size;
+	total_mem_used -= (long_u)hp->bh_page_count * mfp->mf_page_size;
 	nextp = hp->bh_next;
 	mf_free_bhdr(hp);
     }
@@ -359,7 +359,7 @@
 	}
 	else if (hp == NULL)	    // need to allocate memory for this block
 	{
-	    if ((p = alloc(mfp->mf_page_size * page_count)) == NULL)
+	    if ((p = alloc((size_t)mfp->mf_page_size * page_count)) == NULL)
 		return NULL;
 	    hp = mf_rem_free(mfp);
 	    hp->bh_data = p;
@@ -718,7 +718,7 @@
     else
 	hp->bh_next->bh_prev = hp;
     mfp->mf_used_count += hp->bh_page_count;
-    total_mem_used += hp->bh_page_count * mfp->mf_page_size;
+    total_mem_used += (long_u)hp->bh_page_count * mfp->mf_page_size;
 }
 
 /*
@@ -736,7 +736,7 @@
     else
 	hp->bh_prev->bh_next = hp->bh_next;
     mfp->mf_used_count -= hp->bh_page_count;
-    total_mem_used -= hp->bh_page_count * mfp->mf_page_size;
+    total_mem_used -= (long_u)hp->bh_page_count * mfp->mf_page_size;
 }
 
 /*
@@ -814,7 +814,8 @@
     if (hp->bh_page_count != page_count)
     {
 	vim_free(hp->bh_data);
-	if ((hp->bh_data = alloc(mfp->mf_page_size * page_count)) == NULL)
+	if ((hp->bh_data = alloc((size_t)mfp->mf_page_size * page_count))
+								       == NULL)
 	{
 	    vim_free(hp);
 	    return NULL;
@@ -881,7 +882,8 @@
 
     if ((hp = ALLOC_ONE(bhdr_T)) != NULL)
     {
-	if ((hp->bh_data = alloc(mfp->mf_page_size * page_count)) == NULL)
+	if ((hp->bh_data = alloc((size_t)mfp->mf_page_size * page_count))
+								       == NULL)
 	{
 	    vim_free(hp);	    // not enough memory
 	    return NULL;