Automatic lossless refresh
Resend pixel perfect copies of areas that were previously sent
using a lossy encoder. This is done when there is no normal update
to send, and no congestion.
diff --git a/common/rfb/EncodeManager.cxx b/common/rfb/EncodeManager.cxx
index 0ceec8f..ca7c730 100644
--- a/common/rfb/EncodeManager.cxx
+++ b/common/rfb/EncodeManager.cxx
@@ -17,6 +17,9 @@
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
* USA.
*/
+
+#include <stdlib.h>
+
#include <rfb/EncodeManager.h>
#include <rfb/Encoder.h>
#include <rfb/Palette.h>
@@ -47,6 +50,8 @@
// Don't bother with blocks smaller than this
static const int SolidBlockMinArea = 2048;
+static const int LosslessRefreshMaxArea = 4096;
+
namespace rfb {
enum EncoderClass {
@@ -245,17 +250,42 @@
}
}
+bool EncodeManager::needsLosslessRefresh(const Region& req)
+{
+ return !lossyRegion.intersect(req).is_empty();
+}
+
+void EncodeManager::pruneLosslessRefresh(const Region& limits)
+{
+ lossyRegion.assign_intersect(limits);
+}
+
void EncodeManager::writeUpdate(const UpdateInfo& ui, const PixelBuffer* pb,
const RenderedCursor* renderedCursor)
{
+ doUpdate(true, ui.changed, ui.copied, ui.copy_delta, pb, renderedCursor);
+}
+
+void EncodeManager::writeLosslessRefresh(const Region& req, const PixelBuffer* pb,
+ const RenderedCursor* renderedCursor)
+{
+ doUpdate(false, getLosslessRefresh(req),
+ Region(), Point(), pb, renderedCursor);
+}
+
+void EncodeManager::doUpdate(bool allowLossy, const Region& changed_,
+ const Region& copied, const Point& copyDelta,
+ const PixelBuffer* pb,
+ const RenderedCursor* renderedCursor)
+{
int nRects;
Region changed, cursorRegion;
updates++;
- prepareEncoders();
+ prepareEncoders(allowLossy);
- changed = ui.changed;
+ changed = changed_;
/*
* We need to render the cursor seperately as it has its own
@@ -269,14 +299,14 @@
if (conn->cp.supportsLastRect)
nRects = 0xFFFF;
else {
- nRects = ui.copied.numRects();
+ nRects = copied.numRects();
nRects += computeNumRects(changed);
nRects += computeNumRects(cursorRegion);
}
conn->writer()->writeFramebufferUpdateStart(nRects);
- writeCopyRects(ui);
+ writeCopyRects(copied, copyDelta);
/*
* We start by searching for solid rects, which are then removed
@@ -291,7 +321,7 @@
conn->writer()->writeFramebufferUpdateEnd();
}
-void EncodeManager::prepareEncoders()
+void EncodeManager::prepareEncoders(bool allowLossy)
{
enum EncoderClass solid, bitmap, bitmapRLE;
enum EncoderClass indexed, indexedRLE, fullColour;
@@ -316,7 +346,7 @@
break;
case encodingTight:
if (encoders[encoderTightJPEG]->isSupported() &&
- (conn->cp.pf().bpp >= 16))
+ (conn->cp.pf().bpp >= 16) && allowLossy)
fullColour = encoderTightJPEG;
else
fullColour = encoderTight;
@@ -334,7 +364,7 @@
if (fullColour == encoderRaw) {
if (encoders[encoderTightJPEG]->isSupported() &&
- (conn->cp.pf().bpp >= 16))
+ (conn->cp.pf().bpp >= 16) && allowLossy)
fullColour = encoderTightJPEG;
else if (encoders[encoderZRLE]->isSupported())
fullColour = encoderZRLE;
@@ -374,7 +404,7 @@
// JPEG is the only encoder that can reduce things to grayscale
if ((conn->cp.subsampling == subsampleGray) &&
- encoders[encoderTightJPEG]->isSupported()) {
+ encoders[encoderTightJPEG]->isSupported() && allowLossy) {
solid = bitmap = bitmapRLE = encoderTightJPEG;
indexed = indexedRLE = fullColour = encoderTightJPEG;
}
@@ -398,6 +428,48 @@
}
}
+Region EncodeManager::getLosslessRefresh(const Region& req)
+{
+ std::vector<Rect> rects;
+ Region refresh;
+ size_t area;
+
+ area = 0;
+ lossyRegion.intersect(req).get_rects(&rects);
+ while (!rects.empty()) {
+ size_t idx;
+ Rect rect;
+
+ // Grab a random rect so we don't keep damaging and restoring the
+ // same rect over and over
+ idx = rand() % rects.size();
+
+ rect = rects[idx];
+
+ // Add rects until we exceed the threshold, then include as much as
+ // possible of the final rect
+ if ((area + rect.area()) > LosslessRefreshMaxArea) {
+ // Use the narrowest axis to avoid getting to thin rects
+ if (rect.width() > rect.height()) {
+ int width = (LosslessRefreshMaxArea - area) / rect.height();
+ rect.br.x = rect.tl.x + __rfbmax(1, width);
+ } else {
+ int height = (LosslessRefreshMaxArea - area) / rect.width();
+ rect.br.y = rect.tl.y + __rfbmax(1, height);
+ }
+ refresh.assign_union(Region(rect));
+ break;
+ }
+
+ area += rect.area();
+ refresh.assign_union(Region(rect));
+
+ rects.erase(rects.begin() + idx);
+ }
+
+ return refresh;
+}
+
int EncodeManager::computeNumRects(const Region& changed)
{
int numRects;
@@ -450,6 +522,11 @@
encoder = encoders[klass];
conn->writer()->startRect(rect, encoder->encoding);
+ if (encoder->flags & EncoderLossy)
+ lossyRegion.assign_union(Region(rect));
+ else
+ lossyRegion.assign_subtract(Region(rect));
+
return encoder;
}
@@ -466,14 +543,16 @@
stats[klass][activeType].bytes += length;
}
-void EncodeManager::writeCopyRects(const UpdateInfo& ui)
+void EncodeManager::writeCopyRects(const Region& copied, const Point& delta)
{
std::vector<Rect> rects;
std::vector<Rect>::const_iterator rect;
+ Region lossyCopy;
+
beforeLength = conn->getOutStream()->length();
- ui.copied.get_rects(&rects, ui.copy_delta.x <= 0, ui.copy_delta.y <= 0);
+ copied.get_rects(&rects, delta.x <= 0, delta.y <= 0);
for (rect = rects.begin(); rect != rects.end(); ++rect) {
int equiv;
@@ -482,11 +561,16 @@
equiv = 12 + rect->area() * conn->cp.pf().bpp/8;
copyStats.equivalent += equiv;
- conn->writer()->writeCopyRect(*rect, rect->tl.x - ui.copy_delta.x,
- rect->tl.y - ui.copy_delta.y);
+ conn->writer()->writeCopyRect(*rect, rect->tl.x - delta.x,
+ rect->tl.y - delta.y);
}
copyStats.bytes += conn->getOutStream()->length() - beforeLength;
+
+ lossyCopy = lossyRegion;
+ lossyCopy.translate(delta);
+ lossyCopy.assign_intersect(copied);
+ lossyRegion.assign_union(lossyCopy);
}
void EncodeManager::writeSolidRects(Region *changed, const PixelBuffer* pb)
diff --git a/common/rfb/EncodeManager.h b/common/rfb/EncodeManager.h
index 4319b42..bdff04b 100644
--- a/common/rfb/EncodeManager.h
+++ b/common/rfb/EncodeManager.h
@@ -1,6 +1,6 @@
/* Copyright (C) 2000-2003 Constantin Kaplinsky. All Rights Reserved.
* Copyright (C) 2011 D. R. Commander. All Rights Reserved.
- * Copyright 2014 Pierre Ossman for Cendio AB
+ * Copyright 2014-2018 Pierre Ossman for Cendio AB
*
* This is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -24,6 +24,7 @@
#include <rdr/types.h>
#include <rfb/PixelBuffer.h>
+#include <rfb/Region.h>
namespace rfb {
class SConnection;
@@ -31,7 +32,6 @@
class UpdateInfo;
class PixelBuffer;
class RenderedCursor;
- class Region;
struct Rect;
struct RectInfo;
@@ -46,18 +46,30 @@
// Hack to let ConnParams calculate the client's preferred encoding
static bool supported(int encoding);
+ bool needsLosslessRefresh(const Region& req);
+ void pruneLosslessRefresh(const Region& limits);
+
void writeUpdate(const UpdateInfo& ui, const PixelBuffer* pb,
const RenderedCursor* renderedCursor);
+ void writeLosslessRefresh(const Region& req, const PixelBuffer* pb,
+ const RenderedCursor* renderedCursor);
+
protected:
- void prepareEncoders();
+ void doUpdate(bool allowLossy, const Region& changed,
+ const Region& copied, const Point& copy_delta,
+ const PixelBuffer* pb,
+ const RenderedCursor* renderedCursor);
+ void prepareEncoders(bool allowLossy);
+
+ Region getLosslessRefresh(const Region& req);
int computeNumRects(const Region& changed);
Encoder *startRect(const Rect& rect, int type);
void endRect();
- void writeCopyRects(const UpdateInfo& ui);
+ void writeCopyRects(const Region& copied, const Point& delta);
void writeSolidRects(Region *changed, const PixelBuffer* pb);
void findSolidRect(const Rect& rect, Region *changed, const PixelBuffer* pb);
void writeRects(const Region& changed, const PixelBuffer* pb);
@@ -103,6 +115,8 @@
std::vector<Encoder*> encoders;
std::vector<int> activeEncoders;
+ Region lossyRegion;
+
struct EncoderStats {
unsigned rects;
unsigned long long bytes;
diff --git a/common/rfb/Encoder.h b/common/rfb/Encoder.h
index a8a447e..66a10d2 100644
--- a/common/rfb/Encoder.h
+++ b/common/rfb/Encoder.h
@@ -35,6 +35,8 @@
// Give us the raw frame buffer, and not something converted to
// the what the client is asking for.
EncoderUseNativePF = 1 << 0,
+ // Encoder does not encode pixels perfectly accurate
+ EncoderLossy = 1 << 1,
};
class Encoder {
diff --git a/common/rfb/TightJPEGEncoder.cxx b/common/rfb/TightJPEGEncoder.cxx
index 7bb6126..385207f 100644
--- a/common/rfb/TightJPEGEncoder.cxx
+++ b/common/rfb/TightJPEGEncoder.cxx
@@ -64,7 +64,7 @@
TightJPEGEncoder::TightJPEGEncoder(SConnection* conn) :
- Encoder(conn, encodingTight, EncoderUseNativePF, -1),
+ Encoder(conn, encodingTight, (EncoderFlags)(EncoderUseNativePF | EncoderLossy), -1),
qualityLevel(-1), fineQuality(-1), fineSubsampling(subsampleUndefined)
{
}
diff --git a/common/rfb/VNCSConnectionST.cxx b/common/rfb/VNCSConnectionST.cxx
index 0b79dc1..77a6058 100644
--- a/common/rfb/VNCSConnectionST.cxx
+++ b/common/rfb/VNCSConnectionST.cxx
@@ -223,6 +223,9 @@
}
}
}
+
+ // Drop any lossy tracking that is now outside the framebuffer
+ encodeManager.pruneLosslessRefresh(Region(server->pb->getRect()));
}
// Just update the whole screen at the moment because we're too lazy to
// work out what's actually changed.
@@ -1026,7 +1029,8 @@
// Return if there is nothing to send the client.
- if (updates.is_empty() && !writer()->needFakeUpdate())
+ if (updates.is_empty() && !writer()->needFakeUpdate() &&
+ !encodeManager.needsLosslessRefresh(req))
return;
// The `updates' object could change, make sure we have valid update info.
@@ -1055,12 +1059,16 @@
damagedCursorRegion.assign_union(ui.changed.intersect(renderedCursorRect));
}
- if (ui.is_empty() && !writer()->needFakeUpdate())
+ if (ui.is_empty() && !writer()->needFakeUpdate() &&
+ !encodeManager.needsLosslessRefresh(req))
return;
writeRTTPing();
- encodeManager.writeUpdate(ui, server->getPixelBuffer(), cursor);
+ if (!ui.is_empty())
+ encodeManager.writeUpdate(ui, server->getPixelBuffer(), cursor);
+ else
+ encodeManager.writeLosslessRefresh(req, server->getPixelBuffer(), cursor);
writeRTTPing();