Revert r4498 and fix #3305357 properly. The issue was two-fold: (1) the compress buffer allocated by the Tight encoder was not large enough, and (2) Zlib 1.2.5 can sometimes call deflate(..., Z_BLOCK) within the body of deflateParams(), so we need to check avail_in after calling checkCompressionLevel() to ensure that there is still data left to compress before we call deflate() again.
git-svn-id: svn://svn.code.sf.net/p/tigervnc/code/trunk@4617 3789f03b-4d11-0410-bbf8-ca57d06f2519
diff --git a/common/rdr/ZlibOutStream.cxx b/common/rdr/ZlibOutStream.cxx
index c39aa55..c86a5a5 100644
--- a/common/rdr/ZlibOutStream.cxx
+++ b/common/rdr/ZlibOutStream.cxx
@@ -1,4 +1,5 @@
/* Copyright (C) 2002-2005 RealVNC Ltd. All Rights Reserved.
+ * Copyright (C) 2011 D. R. Commander. All Rights Reserved.
*
* This is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -88,6 +89,10 @@
// fprintf(stderr,"zos flush: calling deflate, avail_in %d, avail_out %d\n",
// zs->avail_in,zs->avail_out);
checkCompressionLevel();
+ if (zs->avail_in != 0) {
+ int rc = deflate(zs, Z_SYNC_FLUSH);
+ if (rc != Z_OK) throw Exception("ZlibOutStream: deflate failed");
+ }
// fprintf(stderr,"zos flush: after deflate: %d bytes\n",
// zs->next_out-underlying->getptr());
@@ -119,7 +124,11 @@
// fprintf(stderr,"zos overrun: calling deflate, avail_in %d, avail_out %d\n",
// zs->avail_in,zs->avail_out);
- checkCompressionLevel();
+ checkCompressionLevel();
+ if (zs->avail_in != 0) {
+ int rc = deflate(zs, 0);
+ if (rc != Z_OK) throw Exception("ZlibOutStream: deflate failed");
+ }
// fprintf(stderr,"zos overrun: after deflate: %d bytes\n",
// zs->next_out-underlying->getptr());
@@ -151,15 +160,9 @@
void ZlibOutStream::checkCompressionLevel()
{
if (newLevel != compressionLevel) {
- int rc = deflate(zs, Z_SYNC_FLUSH);
- if (rc != Z_OK)
- throw Exception("ZlibOutStream: deflate failed");
- if (deflateParams (zs, newLevel, Z_DEFAULT_STRATEGY) != Z_OK)
+ if (deflateParams (zs, newLevel, Z_DEFAULT_STRATEGY) != Z_OK) {
throw Exception("ZlibOutStream: deflateParams failed");
+ }
compressionLevel = newLevel;
- } else {
- int rc = deflate(zs, Z_SYNC_FLUSH);
- if (rc != Z_OK)
- throw Exception("ZlibOutStream: deflate failed");
}
}
diff --git a/common/rfb/tightEncode.h b/common/rfb/tightEncode.h
index 2e6d0d8..9877759 100644
--- a/common/rfb/tightEncode.h
+++ b/common/rfb/tightEncode.h
@@ -1,4 +1,5 @@
/* Copyright (C) 2000-2003 Constantin Kaplinsky. All Rights Reserved.
+ * Copyright (C) 2011 D. R. Commander. All Rights Reserved.
*
* This is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -188,7 +189,9 @@
} else {
// FIXME: Using a temporary MemOutStream may be not efficient.
// Maybe use the same static object used in the JPEG coder?
- rdr::MemOutStream mem_os;
+ int maxBeforeSize = s_pconf->maxRectSize * (BPP / 8);
+ int maxAfterSize = maxBeforeSize + (maxBeforeSize + 99) / 100 + 12;
+ rdr::MemOutStream mem_os(maxAfterSize);
zos->setUnderlying(&mem_os);
zos->setCompressionLevel(zlibLevel);
zos->writeBytes(buf, length);