rfb/hextileEncodeBetter.h

/* Copyright (C) 2002-2003 RealVNC Ltd.  All Rights Reserved.
 * Copyright (C) 2005 Constantin Kaplinsky.  All Rights Reserved.
 *    
 * This is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 * 
 * This software is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with this software; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
 * USA.
 */
//
// Hextile encoding function.
//
// This file is #included after having set the following macros:
// BPP                - 8, 16 or 32
// EXTRA_ARGS         - optional extra arguments
// GET_IMAGE_INTO_BUF - gets a rectangle of pixel data into a buffer

#include <map>

#include <rdr/OutStream.h>
#include <rfb/hextileConstants.h>

#include <assert.h>

namespace rfb {

// CONCAT2E concatenates its arguments, expanding them if they are macros

#ifndef CONCAT2E
#define CONCAT2(a,b) a##b
#define CONCAT2E(a,b) CONCAT2(a,b)
#endif

#define PIXEL_T rdr::CONCAT2E(U,BPP)
#define WRITE_PIXEL CONCAT2E(writeOpaque,BPP)
#define HEXTILE_ENCODE CONCAT2E(hextileEncode,BPP)

/********************************************************************/

#define HEXTILE_SUBRECTS_TABLE CONCAT2E(HextileSubrectsTable,BPP)

class HEXTILE_SUBRECTS_TABLE {

 public:

  HEXTILE_SUBRECTS_TABLE ();

  void newTile(const PIXEL_T *src, int w, int h);
  int buildTables();
  int encode(rdr::U8* dst);

  int getNumColors() const { return m_numColors; }
  int getBackground() const { return m_background; }
  int getForeground() const { return m_foreground; }

 protected:

  const PIXEL_T *m_tile;
  int m_width;
  int m_height;
  int m_numSubrects;
  int m_numColors;
  PIXEL_T m_background;
  PIXEL_T m_foreground;

  // FIXME: Comment data structures.
  rdr::U8 m_coords[256 * 2];
  PIXEL_T m_colors[256];

 private:

  /* DEBUG: Check performance for: (1) U8[] and (2) dyn.allocated. */
  bool m_processed[16][16];

  /* FIXME: Use array for (BPP == 8)? */
  /* DEBUG: Use own hashing like in ZRLE? */
  std::map<PIXEL_T,int> m_counts;
};

HEXTILE_SUBRECTS_TABLE::HEXTILE_SUBRECTS_TABLE()
  : m_tile(NULL), m_width(0), m_height(0), m_numSubrects(-1), m_numColors(0),
    m_background(0), m_foreground(0)
{
}

void HEXTILE_SUBRECTS_TABLE::newTile(const PIXEL_T *src, int w, int h)
{
  m_tile = src;
  m_width = w;
  m_height = h;
  m_numSubrects = -1;
  m_numColors = 0;
}

/*
 * Returns estimated encoded data size.
 */

int HEXTILE_SUBRECTS_TABLE::buildTables()
{
  if (m_tile == NULL || m_width == 0 || m_height == 0)
    return -1;

  m_numSubrects = 0;
  memset(m_processed, 0, 16 * 16 * sizeof(bool));
  m_counts.clear();

  int x, y, sx, sy, sw, sh, max_x;
  PIXEL_T color;
  PIXEL_T *colorsPtr = &m_colors[0];
  rdr::U8 *coordsPtr = &m_coords[0];

  for (y = 0; y < m_height; y++) {
    for (x = 0; x < m_width; x++) {
      /* Skip pixels that were processed earlier */
      if (m_processed[y][x]) {
        continue;
      }
      /* Determine dimensions of the horizontal subrect */
      color = m_tile[y * m_width + x];
      for (sx = x + 1; sx < m_width; sx++) {
        if (m_tile[y * m_width + sx] != color)
          break;
      }
      sw = sx - x;
      max_x = sx;
      for (sy = y + 1; sy < m_height; sy++) {
        for (sx = x; sx < max_x; sx++) {
          if (m_tile[sy * m_width + sx] != color)
            goto done;
        }
      }
    done:
      sh = sy - y;

      /* Save properties of this subrect */
      *colorsPtr++ = color;
      *coordsPtr++ = (rdr::U8)((x << 4) | (y & 0x0F));
      *coordsPtr++ = (rdr::U8)(((sw - 1) << 4) | ((sh - 1) & 0x0F));
      m_counts[color] += 1;

      m_numSubrects++;

      /* Mark pixels of this subrect as processed, below this row */
      for (sy = y + 1; sy < y + sh; sy++) {
        for (sx = x; sx < x + sw; sx++)
          m_processed[sy][sx] = true;
      }

      /* Skip processed pixels of this row */
      x += (sw - 1);
    }
  }

  // Choose the best background color
  int maxCount = 0, count;
  std::map<PIXEL_T,int>::iterator i;
  for (i = m_counts.begin(); i != m_counts.end(); i++) {
    color = (*i).first;
    count = (*i).second;
    if (count > maxCount) {
      maxCount = count;
      m_background = color;
    }
  }

  // Save the number of colors
  m_numColors = m_counts.size();

  // Set foreground color if it's a monochrome tile
  if (m_numColors == 2) {
    i = m_counts.begin();
    m_foreground = (*i).first;
    if (m_foreground == m_background) {
      i++;
      m_foreground = (*i).first;
    }
    // Calculate and return encoded data size
    return 1 + 2 * (m_numSubrects - maxCount);
  }

  // Calculate and return encoded data size (colored subrects)
  return 1 + (2 + (BPP/8)) * (m_numSubrects - maxCount);
}

/*
 * Call this function only if there are any subrects in the tile.
 * The buffer size should be enough to store at least that number of
 * bytes returned by buildTables() method.
 * Returns encoded data size, or zero if something is wrong.
 */

int HEXTILE_SUBRECTS_TABLE::encode(rdr::U8 *dst)
{
  if (m_numSubrects == -1 || m_numColors == 0)
    return 0;

  // Zero subrects counter.
  rdr::U8 *numSubrectsPtr = dst;
  *dst++ = 0;

  for (int i = 0; i < m_numSubrects; i++) {
    if (m_colors[i] == m_background)
      continue;

    if (m_numColors > 2) {      /* FIXME: Duplicate */
#if (BPP == 8)
      *dst++ = m_colors[i];
#elif (BPP == 16)
      *dst++ = ((rdr::U8*)&m_colors[i])[0];
      *dst++ = ((rdr::U8*)&m_colors[i])[1];
#elif (BPP == 32)
      *dst++ = ((rdr::U8*)&m_colors[i])[0];
      *dst++ = ((rdr::U8*)&m_colors[i])[1];
      *dst++ = ((rdr::U8*)&m_colors[i])[2];
      *dst++ = ((rdr::U8*)&m_colors[i])[3];
#endif
    }
    *dst++ = m_coords[i * 2];
    *dst++ = m_coords[i * 2 + 1];

    (*numSubrectsPtr)++;
  }

  return (dst - numSubrectsPtr);
}

/*------------------------------------------------------------------*/

void HEXTILE_ENCODE(const Rect& r, rdr::OutStream* os
#ifdef EXTRA_ARGS
                    , EXTRA_ARGS
#endif
                    )
{
  Rect t;
  PIXEL_T buf[256];
  PIXEL_T oldBg = 0, oldFg = 0;
  bool oldBgValid = false;
  bool oldFgValid = false;
  rdr::U8 encoded[256*(BPP/8)];

  HEXTILE_SUBRECTS_TABLE subrects;

  for (t.tl.y = r.tl.y; t.tl.y < r.br.y; t.tl.y += 16) {

    t.br.y = vncmin(r.br.y, t.tl.y + 16);

    for (t.tl.x = r.tl.x; t.tl.x < r.br.x; t.tl.x += 16) {

      t.br.x = vncmin(r.br.x, t.tl.x + 16);

      GET_IMAGE_INTO_BUF(t,buf);

      subrects.newTile(buf, t.width(), t.height());
      int encodedLen = subrects.buildTables();

      // FIXME: Adjust encodedLen comparison!
      if (encodedLen >= t.width() * t.height() * (BPP/8)) {
        os->writeU8(hextileRaw);
        os->writeBytes(buf, t.width() * t.height() * (BPP/8));
        oldBgValid = oldFgValid = false;
        continue;
      }

      int numColors = subrects.getNumColors();
      PIXEL_T bg = subrects.getBackground();
      PIXEL_T fg = subrects.getForeground();

      int tileType = 0;

      if (!oldBgValid || oldBg != bg) {
        tileType |= hextileBgSpecified;
        oldBg = bg;
        oldBgValid = true;
      }

      if (numColors >= 2) {
        tileType |= hextileAnySubrects;
        if (numColors == 2) {
          if (!oldFgValid || oldFg != fg) {
            tileType |= hextileFgSpecified;
            oldFg = fg;
            oldFgValid = true;
          }
        } else {
          tileType |= hextileSubrectsColoured;
          oldFgValid = false;
        }
        int finalEncodedLen = subrects.encode(encoded);
        assert(finalEncodedLen == encodedLen);
        assert(finalEncodedLen <= 256*(BPP/8));
      }

      os->writeU8(tileType);
      if (tileType & hextileBgSpecified) os->WRITE_PIXEL(bg);
      if (tileType & hextileFgSpecified) os->WRITE_PIXEL(fg);
      if (tileType & hextileAnySubrects) os->writeBytes(encoded, encodedLen);
    }
  }
}

#undef PIXEL_T
#undef WRITE_PIXEL
#undef HEXTILE_ENCODE

#undef HEXTILE_SUBRECTS_TABLE
}