Unfortunately, the previous enhancements made an invalid assumption about the code sizes, which caused "DCT out of range" errors in the decoder.  Was able to fix 64-bit without losing any performance, but unfortunately had to revert 32-bit back to its previous behavior for now.


git-svn-id: svn://svn.code.sf.net/p/tigervnc/code/trunk@3871 3789f03b-4d11-0410-bbf8-ca57d06f2519
diff --git a/common/jpeg/jchuff.c b/common/jpeg/jchuff.c
index e9abb42..daf0736 100644
--- a/common/jpeg/jchuff.c
+++ b/common/jpeg/jchuff.c
@@ -334,7 +334,7 @@
 /***************************************************************/
 
 #define EMIT_BYTE() {                                           \
-  if (0xFF == (*buffer++ =  put_buffer >> (put_bits -= 8)))     \
+  if (0xFF == (*buffer++ =  (unsigned char)(put_buffer >> (put_bits -= 8))))  \
     *buffer++ = 0;                                              \
  }
 
@@ -357,13 +357,6 @@
   }                                                             \
 }
 
-#define CHECKBUF15() {                                          \
-  if (put_bits > 15) {                                          \
-    EMIT_BYTE()                                                 \
-    EMIT_BYTE()                                                 \
-  }                                                             \
-}
-
 #define CHECKBUF47() {                                          \
   if (put_bits > 47) {                                          \
     EMIT_BYTE()                                                 \
@@ -375,11 +368,8 @@
   }                                                             \
 }
 
-#define CHECKBUF55() {                                          \
-  if (put_bits > 55) {                                          \
-    EMIT_BYTE()                                                 \
-    EMIT_BYTE()                                                 \
-    EMIT_BYTE()                                                 \
+#define CHECKBUF31() {                                          \
+  if (put_bits > 31) {                                          \
     EMIT_BYTE()                                                 \
     EMIT_BYTE()                                                 \
     EMIT_BYTE()                                                 \
@@ -397,7 +387,7 @@
 #if __WORDSIZE==64
 
 #define DUMP_BITS(code, size) {                                 \
-  CHECKBUF55()                                                  \
+  CHECKBUF47()                                                  \
   put_bits += size;                                             \
   put_buffer = (put_buffer << size) | code;                     \
  }
@@ -433,13 +423,15 @@
   CHECKBUF15()                                \
  }
 
+int _max=0;
+
 #if __WORDSIZE==64
 
 #define DUMP_VALUE(ht, codevalue, t, nbits) { \
   size = ht->ehufsi[codevalue];               \
   code = ht->ehufco[codevalue];               \
   t &= ~(-1 << nbits);                        \
-  CHECKBUF47()                                \
+  CHECKBUF31()                                \
   DUMP_BITS_NOCHECK(code, size)               \
   DUMP_BITS_NOCHECK(t, nbits)                 \
  }
@@ -451,6 +443,7 @@
   code = ht->ehufco[codevalue];               \
   t &= ~(-1 << nbits);                        \
   DUMP_BITS_NOCHECK(code, size)               \
+  CHECKBUF15()                                \
   DUMP_BITS_NOCHECK(t, nbits)                 \
   CHECKBUF15()                                \
  }
@@ -552,8 +545,8 @@
     sflag = temp >> 31;  \
     temp = (temp ^ sflag) - sflag;  \
     temp2 += sflag;  \
-    for(; r > 15; r -= 16) DUMP_BITS(code_0xf0, size_0xf0)  \
     nbits = jpeg_first_bit_table[temp];  \
+    for(; r > 15; r -= 16) DUMP_BITS(code_0xf0, size_0xf0)  \
     sflag = (r << 4) + nbits;  \
     DUMP_VALUE(actbl, sflag, temp2, nbits)  \
     r = 0;  \