diff --git a/rfbplayer/rfbplayer.cxx b/rfbplayer/rfbplayer.cxx
index 72614f1..fac9181 100644
--- a/rfbplayer/rfbplayer.cxx
+++ b/rfbplayer/rfbplayer.cxx
@@ -226,7 +226,7 @@
   seekMode(false), fileName(_fileName), lastPos(0), timeStatic(0), 
   speedEdit(0), posTrackBar(0), speedUpDown(0), acceptBell(_acceptBell), 
   rfbReader(0), sessionTimeMs(0), sliderDraging(false), sliderStepMs(0), 
-  loopPlayback(false) {
+  loopPlayback(false), imageDataStartTime(0) {
 
   CTRL_BAR_HEIGHT = 28;
 
@@ -824,7 +824,7 @@
     // It's a special exception to perform backward seeking.
     // We only rewind the stream and seek the offset
     if (strcmp(e.str(), "[REWIND]") == 0) {
-      long seekOffset = getSeekOffset();
+      long seekOffset = max(getSeekOffset(), imageDataStartTime);
       rewind();
       setPos(seekOffset);
       updatePos(seekOffset);
@@ -838,6 +838,9 @@
 void RfbPlayer::serverInit() {
   RfbProto::serverInit();
 
+  // Save the image data start time
+  imageDataStartTime = is->getTimeOffset();
+
   // Resize the backing buffer
   buffer->setSize(cp.width, cp.height);
 
@@ -1075,7 +1078,7 @@
 }
 
 void RfbPlayer::setPos(long pos) {
-  is->setTimeOffset(max(pos, 0));
+  is->setTimeOffset(max(pos, imageDataStartTime));
 }
 
 long RfbPlayer::getSeekOffset() {