Eliminate GCC signed/unsigned warnings related to encodings: The
encoding in the RFB protocol has always been signed, and signed values
are also used in the specification (ie DesktopName = -307 etc). In the
code, however, unsigned types were used in a number of places, but not
all, which causes warnings. This patch fixes the problem by switching
to signed values everywhere.



git-svn-id: svn://svn.code.sf.net/p/tigervnc/code/trunk@3968 3789f03b-4d11-0410-bbf8-ca57d06f2519
diff --git a/common/rdr/types.h b/common/rdr/types.h
index 6421b13..458a13e 100644
--- a/common/rdr/types.h
+++ b/common/rdr/types.h
@@ -61,6 +61,16 @@
     U32* buf;
   };
 
+  class S32Array {
+  public:
+    S32Array() : buf(0) {}
+    S32Array(S32* a) : buf(a) {} // note: assumes ownership
+    S32Array(int len) : buf(new S32[len]) {}
+    ~S32Array() { delete [] buf; }
+    S32* takeBuf() { S32* tmp = buf; buf = 0; return tmp; }
+    S32* buf;
+  };
+
 } // end of namespace rdr
 
 #endif
diff --git a/common/rfb/CMsgHandler.h b/common/rfb/CMsgHandler.h
index 9205ef5..d8b949b 100644
--- a/common/rfb/CMsgHandler.h
+++ b/common/rfb/CMsgHandler.h
@@ -56,8 +56,8 @@
 
     virtual void framebufferUpdateStart() = 0;
     virtual void framebufferUpdateEnd() = 0;
-    virtual void beginRect(const Rect& r, unsigned int encoding) = 0;
-    virtual void endRect(const Rect& r, unsigned int encoding) = 0;
+    virtual void beginRect(const Rect& r, int encoding) = 0;
+    virtual void endRect(const Rect& r, int encoding) = 0;
 
     virtual void setColourMapEntries(int firstColour, int nColours,
 				     rdr::U16* rgbs) = 0;
diff --git a/common/rfb/CMsgReader.cxx b/common/rfb/CMsgReader.cxx
index a78a1a0..e3b73bf 100644
--- a/common/rfb/CMsgReader.cxx
+++ b/common/rfb/CMsgReader.cxx
@@ -28,14 +28,14 @@
   : imageBufIdealSize(0), handler(handler_), is(is_),
     imageBuf(0), imageBufSize(0)
 {
-  for (unsigned int i = 0; i <= encodingMax; i++) {
+  for (int i = 0; i <= encodingMax; i++) {
     decoders[i] = 0;
   }
 }
 
 CMsgReader::~CMsgReader()
 {
-  for (unsigned int i = 0; i <= encodingMax; i++) {
+  for (int i = 0; i <= encodingMax; i++) {
     delete decoders[i];
   }
   delete [] imageBuf;
@@ -82,7 +82,7 @@
   handler->framebufferUpdateEnd();
 }
 
-void CMsgReader::readRect(const Rect& r, unsigned int encoding)
+void CMsgReader::readRect(const Rect& r, int encoding)
 {
   if ((r.br.x > handler->cp.width) || (r.br.y > handler->cp.height)) {
     fprintf(stderr, "Rect too big: %dx%d at %d,%d exceeds %dx%d\n",
diff --git a/common/rfb/CMsgReader.h b/common/rfb/CMsgReader.h
index 7a611fc..7b36160 100644
--- a/common/rfb/CMsgReader.h
+++ b/common/rfb/CMsgReader.h
@@ -55,7 +55,7 @@
 
     virtual void readFramebufferUpdateStart();
     virtual void readFramebufferUpdateEnd();
-    virtual void readRect(const Rect& r, unsigned int encoding);
+    virtual void readRect(const Rect& r, int encoding);
 
     virtual void readCopyRect(const Rect& r);
 
diff --git a/common/rfb/CMsgReaderV3.cxx b/common/rfb/CMsgReaderV3.cxx
index 5471593..ba516bf 100644
--- a/common/rfb/CMsgReaderV3.cxx
+++ b/common/rfb/CMsgReaderV3.cxx
@@ -72,7 +72,7 @@
     int y = is->readU16();
     int w = is->readU16();
     int h = is->readU16();
-    unsigned int encoding = is->readU32();
+    int encoding = is->readS32();
 
     switch (encoding) {
     case pseudoEncodingDesktopSize:
diff --git a/common/rfb/ConnParams.cxx b/common/rfb/ConnParams.cxx
index 7b27a73..9bfc302 100644
--- a/common/rfb/ConnParams.cxx
+++ b/common/rfb/ConnParams.cxx
@@ -85,11 +85,11 @@
   name_ = strDup(name);
 }
 
-void ConnParams::setEncodings(int nEncodings, const rdr::U32* encodings)
+void ConnParams::setEncodings(int nEncodings, const rdr::S32* encodings)
 {
   if (nEncodings > nEncodings_) {
     delete [] encodings_;
-    encodings_ = new rdr::U32[nEncodings];
+    encodings_ = new rdr::S32[nEncodings];
   }
   nEncodings_ = nEncodings;
   useCopyRect = false;
diff --git a/common/rfb/ConnParams.h b/common/rfb/ConnParams.h
index 7779640..54b5ada 100644
--- a/common/rfb/ConnParams.h
+++ b/common/rfb/ConnParams.h
@@ -66,10 +66,10 @@
     const char* name() { return name_; }
     void setName(const char* name);
 
-    rdr::U32 currentEncoding() { return currentEncoding_; }
+    rdr::S32 currentEncoding() { return currentEncoding_; }
     int nEncodings() { return nEncodings_; }
-    const rdr::U32* encodings() { return encodings_; }
-    void setEncodings(int nEncodings, const rdr::U32* encodings);
+    const rdr::S32* encodings() { return encodings_; }
+    void setEncodings(int nEncodings, const rdr::S32* encodings);
     bool useCopyRect;
 
     bool supportsLocalCursor;
@@ -91,7 +91,7 @@
     PixelFormat pf_;
     char* name_;
     int nEncodings_;
-    rdr::U32* encodings_;
+    rdr::S32* encodings_;
     int currentEncoding_;
     char verStr[13];
     int verStrPos;
diff --git a/common/rfb/Decoder.cxx b/common/rfb/Decoder.cxx
index b6e4fd5..61d8bce 100644
--- a/common/rfb/Decoder.cxx
+++ b/common/rfb/Decoder.cxx
@@ -32,19 +32,19 @@
 
 DecoderCreateFnType Decoder::createFns[encodingMax+1] = { 0 };
 
-bool Decoder::supported(unsigned int encoding)
+bool Decoder::supported(int encoding)
 {
   return encoding <= encodingMax && createFns[encoding];
 }
 
-Decoder* Decoder::createDecoder(unsigned int encoding, CMsgReader* reader)
+Decoder* Decoder::createDecoder(int encoding, CMsgReader* reader)
 {
   if (encoding <= encodingMax && createFns[encoding])
     return (*createFns[encoding])(reader);
   return 0;
 }
 
-void Decoder::registerDecoder(unsigned int encoding,
+void Decoder::registerDecoder(int encoding,
                               DecoderCreateFnType createFn)
 {
   if (encoding > encodingMax)
diff --git a/common/rfb/Decoder.h b/common/rfb/Decoder.h
index 3fdba53..50aee82 100644
--- a/common/rfb/Decoder.h
+++ b/common/rfb/Decoder.h
@@ -32,9 +32,9 @@
     virtual ~Decoder();
     virtual void readRect(const Rect& r, CMsgHandler* handler)=0;
 
-    static bool supported(unsigned int encoding);
-    static Decoder* createDecoder(unsigned int encoding, CMsgReader* reader);
-    static void registerDecoder(unsigned int encoding,
+    static bool supported(int encoding);
+    static Decoder* createDecoder(int encoding, CMsgReader* reader);
+    static void registerDecoder(int encoding,
                                 DecoderCreateFnType createFn);
   private:
     static DecoderCreateFnType createFns[encodingMax+1];
diff --git a/common/rfb/Encoder.cxx b/common/rfb/Encoder.cxx
index 53cb170..b0bd147 100644
--- a/common/rfb/Encoder.cxx
+++ b/common/rfb/Encoder.cxx
@@ -32,19 +32,19 @@
 
 EncoderCreateFnType Encoder::createFns[encodingMax+1] = { 0 };
 
-bool Encoder::supported(unsigned int encoding)
+bool Encoder::supported(int encoding)
 {
   return encoding <= encodingMax && createFns[encoding];
 }
 
-Encoder* Encoder::createEncoder(unsigned int encoding, SMsgWriter* writer)
+Encoder* Encoder::createEncoder(int encoding, SMsgWriter* writer)
 {
   if (encoding <= encodingMax && createFns[encoding])
     return (*createFns[encoding])(writer);
   return 0;
 }
 
-void Encoder::registerEncoder(unsigned int encoding,
+void Encoder::registerEncoder(int encoding,
                               EncoderCreateFnType createFn)
 {
   if (encoding > encodingMax)
@@ -56,7 +56,7 @@
   createFns[encoding] = createFn;
 }
 
-void Encoder::unregisterEncoder(unsigned int encoding)
+void Encoder::unregisterEncoder(int encoding)
 {
   if (encoding > encodingMax)
     throw Exception("Encoder::unregisterEncoder: encoding out of range");
diff --git a/common/rfb/Encoder.h b/common/rfb/Encoder.h
index df50dd6..2a6e2f6 100644
--- a/common/rfb/Encoder.h
+++ b/common/rfb/Encoder.h
@@ -40,11 +40,11 @@
     // rectangle which was updated.
     virtual bool writeRect(const Rect& r, ImageGetter* ig, Rect* actual)=0;
 
-    static bool supported(unsigned int encoding);
-    static Encoder* createEncoder(unsigned int encoding, SMsgWriter* writer);
-    static void registerEncoder(unsigned int encoding,
+    static bool supported(int encoding);
+    static Encoder* createEncoder(int encoding, SMsgWriter* writer);
+    static void registerEncoder(int encoding,
                                 EncoderCreateFnType createFn);
-    static void unregisterEncoder(unsigned int encoding);
+    static void unregisterEncoder(int encoding);
   private:
     static EncoderCreateFnType createFns[encodingMax+1];
   };
diff --git a/common/rfb/SConnection.cxx b/common/rfb/SConnection.cxx
index 084fcd3..30def29 100644
--- a/common/rfb/SConnection.cxx
+++ b/common/rfb/SConnection.cxx
@@ -442,7 +442,7 @@
   CapsList ecaps;
 
   // First, add true encodings.
-  for (unsigned int i = 1; i <= encodingMax; i++) {
+  for (int i = 1; i <= encodingMax; i++) {
     if (Encoder::supported(i)) {
       // FIXME: Capability info should be provided by Encoder objects.
       switch (i) {
diff --git a/common/rfb/SMsgHandler.cxx b/common/rfb/SMsgHandler.cxx
index 8124c50..730bc2c 100644
--- a/common/rfb/SMsgHandler.cxx
+++ b/common/rfb/SMsgHandler.cxx
@@ -39,7 +39,7 @@
   cp.setPF(pf);
 }
 
-void SMsgHandler::setEncodings(int nEncodings, rdr::U32* encodings)
+void SMsgHandler::setEncodings(int nEncodings, rdr::S32* encodings)
 {
   cp.setEncodings(nEncodings, encodings);
   supportsLocalCursor();
diff --git a/common/rfb/SMsgHandler.h b/common/rfb/SMsgHandler.h
index f212edf..a0bc64a 100644
--- a/common/rfb/SMsgHandler.h
+++ b/common/rfb/SMsgHandler.h
@@ -46,7 +46,7 @@
     virtual void clientInit(bool shared);
 
     virtual void setPixelFormat(const PixelFormat& pf);
-    virtual void setEncodings(int nEncodings, rdr::U32* encodings);
+    virtual void setEncodings(int nEncodings, rdr::S32* encodings);
     virtual void framebufferUpdateRequest(const Rect& r, bool incremental) = 0;
     virtual void setDesktopSize(int fb_width, int fb_height,
                                 const ScreenSet& layout) = 0;
diff --git a/common/rfb/SMsgReader.cxx b/common/rfb/SMsgReader.cxx
index 447e0a4..49d8dcf 100644
--- a/common/rfb/SMsgReader.cxx
+++ b/common/rfb/SMsgReader.cxx
@@ -51,7 +51,7 @@
 {
   is->skip(1);
   int nEncodings = is->readU16();
-  rdr::U32Array encodings(nEncodings);
+  rdr::S32Array encodings(nEncodings);
   for (int i = 0; i < nEncodings; i++)
     encodings.buf[i] = is->readU32();
   handler->setEncodings(nEncodings, encodings.buf);
diff --git a/common/rfb/SMsgWriter.cxx b/common/rfb/SMsgWriter.cxx
index 1695161..2262be0 100644
--- a/common/rfb/SMsgWriter.cxx
+++ b/common/rfb/SMsgWriter.cxx
@@ -34,7 +34,7 @@
     currentEncoding(0), updatesSent(0), rawBytesEquivalent(0),
     imageBuf(0), imageBufSize(0)
 {
-  for (unsigned int i = 0; i <= encodingMax; i++) {
+  for (int i = 0; i <= encodingMax; i++) {
     encoders[i] = 0;
     bytesSent[i] = 0;
     rectsSent[i] = 0;
@@ -45,7 +45,7 @@
 {
   vlog.info("framebuffer updates %d",updatesSent);
   int bytes = 0;
-  for (unsigned int i = 0; i <= encodingMax; i++) {
+  for (int i = 0; i <= encodingMax; i++) {
     delete encoders[i];
     if (i != encodingCopyRect)
       bytes += bytesSent[i];
@@ -92,7 +92,7 @@
 
 void SMsgWriter::setupCurrentEncoder()
 {
-  unsigned int encoding = cp->currentEncoding();
+  int encoding = cp->currentEncoding();
 
   // FIXME: Code duplication, see writeRect().
   if (!encoders[encoding]) {
@@ -106,7 +106,7 @@
 
 int SMsgWriter::getNumRects(const Rect &r)
 {
-  unsigned int encoding = cp->currentEncoding();
+  int encoding = cp->currentEncoding();
 
   if (!encoders[encoding])
     setupCurrentEncoder();
@@ -169,7 +169,7 @@
   return writeRect(r, cp->currentEncoding(), ig, actual);
 }
 
-bool SMsgWriter::writeRect(const Rect& r, unsigned int encoding,
+bool SMsgWriter::writeRect(const Rect& r, int encoding,
                            ImageGetter* ig, Rect* actual)
 {
   if (!encoders[encoding]) {
diff --git a/common/rfb/SMsgWriter.h b/common/rfb/SMsgWriter.h
index 44a3915..007d758 100644
--- a/common/rfb/SMsgWriter.h
+++ b/common/rfb/SMsgWriter.h
@@ -145,12 +145,12 @@
     // write the whole rectangle it returns false and sets actual to the actual
     // rectangle which was updated.
     virtual bool writeRect(const Rect& r, ImageGetter* ig, Rect* actual);
-    virtual bool writeRect(const Rect& r, unsigned int encoding,
+    virtual bool writeRect(const Rect& r, int encoding,
                            ImageGetter* ig, Rect* actual);
 
     virtual void writeCopyRect(const Rect& r, int srcX, int srcY);
 
-    virtual void startRect(const Rect& r, unsigned int enc)=0;
+    virtual void startRect(const Rect& r, int enc)=0;
     virtual void endRect()=0;
 
     ConnParams* getConnParams() { return cp; }
@@ -176,7 +176,7 @@
 
     Encoder* encoders[encodingMax+1];
     int lenBeforeRect;
-    unsigned int currentEncoding;
+    int currentEncoding;
     int updatesSent;
     int bytesSent[encodingMax+1];
     int rectsSent[encodingMax+1];
diff --git a/common/rfb/SMsgWriterV3.cxx b/common/rfb/SMsgWriterV3.cxx
index 6f0aed0..08f5a2e 100644
--- a/common/rfb/SMsgWriterV3.cxx
+++ b/common/rfb/SMsgWriterV3.cxx
@@ -221,7 +221,7 @@
   endMsg();
 }
 
-void SMsgWriterV3::startRect(const Rect& r, unsigned int encoding)
+void SMsgWriterV3::startRect(const Rect& r, int encoding)
 {
   if (++nRectsInUpdate > nRectsInHeader && nRectsInHeader)
     throw Exception("SMsgWriterV3::startRect: nRects out of sync");
diff --git a/common/rfb/SMsgWriterV3.h b/common/rfb/SMsgWriterV3.h
index 5e0a3f1..8918a04 100644
--- a/common/rfb/SMsgWriterV3.h
+++ b/common/rfb/SMsgWriterV3.h
@@ -52,7 +52,7 @@
     virtual void writeFramebufferUpdateStart(int nRects);
     virtual void writeFramebufferUpdateStart();
     virtual void writeFramebufferUpdateEnd();
-    virtual void startRect(const Rect& r, unsigned int encoding);
+    virtual void startRect(const Rect& r, int encoding);
     virtual void endRect();
 
   protected:
diff --git a/common/rfb/encodings.cxx b/common/rfb/encodings.cxx
index 6aa81c4..97f547b 100644
--- a/common/rfb/encodings.cxx
+++ b/common/rfb/encodings.cxx
@@ -34,7 +34,7 @@
   return -1;
 }
 
-const char* rfb::encodingName(unsigned int num)
+const char* rfb::encodingName(int num)
 {
   switch (num) {
   case encodingRaw:      return "raw";
diff --git a/common/rfb/encodings.h b/common/rfb/encodings.h
index 17c7f29..16cd73a 100644
--- a/common/rfb/encodings.h
+++ b/common/rfb/encodings.h
@@ -20,30 +20,30 @@
 
 namespace rfb {
 
-  const unsigned int encodingRaw = 0;
-  const unsigned int encodingCopyRect = 1;
-  const unsigned int encodingRRE = 2;
-  const unsigned int encodingCoRRE = 4;
-  const unsigned int encodingHextile = 5;
-  const unsigned int encodingTight = 7;
-  const unsigned int encodingZRLE = 16;
+  const int encodingRaw = 0;
+  const int encodingCopyRect = 1;
+  const int encodingRRE = 2;
+  const int encodingCoRRE = 4;
+  const int encodingHextile = 5;
+  const int encodingTight = 7;
+  const int encodingZRLE = 16;
 
-  const unsigned int encodingMax = 255;
+  const int encodingMax = 255;
 
-  const unsigned int pseudoEncodingXCursor = 0xffffff10;
-  const unsigned int pseudoEncodingCursor = 0xffffff11;
-  const unsigned int pseudoEncodingDesktopSize = 0xffffff21;
-  const unsigned int pseudoEncodingExtendedDesktopSize = 0xfffffecc;
-  const unsigned int pseudoEncodingDesktopName = 0xfffffecdl;
+  const int pseudoEncodingXCursor = -240;
+  const int pseudoEncodingCursor = -239;
+  const int pseudoEncodingDesktopSize = -223;
+  const int pseudoEncodingExtendedDesktopSize = -308;
+  const int pseudoEncodingDesktopName = -307;
 
   // TightVNC-specific
-  const unsigned int pseudoEncodingLastRect = 0xFFFFFF20;
-  const unsigned int pseudoEncodingQualityLevel0 = 0xFFFFFFE0;
-  const unsigned int pseudoEncodingQualityLevel9 = 0xFFFFFFE9;
-  const unsigned int pseudoEncodingCompressLevel0 = 0xFFFFFF00;
-  const unsigned int pseudoEncodingCompressLevel9 = 0xFFFFFF09;
+  const int pseudoEncodingLastRect = -224;
+  const int pseudoEncodingQualityLevel0 = -32;
+  const int pseudoEncodingQualityLevel9 = -23;
+  const int pseudoEncodingCompressLevel0 = -256;
+  const int pseudoEncodingCompressLevel9 = -247;
 
   int encodingNum(const char* name);
-  const char* encodingName(unsigned int num);
+  const char* encodingName(int num);
 }
 #endif