diff --git a/common/rfb/CSecurityTLS.cxx b/common/rfb/CSecurityTLS.cxx
index b943c10..7ca01d5 100644
--- a/common/rfb/CSecurityTLS.cxx
+++ b/common/rfb/CSecurityTLS.cxx
@@ -69,7 +69,7 @@
 
 CSecurityTLS::CSecurityTLS(CConnection* cc, bool _anon)
   : CSecurity(cc), session(NULL), anon_cred(NULL), cert_cred(NULL),
-    anon(_anon), fis(NULL), fos(NULL)
+    anon(_anon), tlsis(NULL), tlsos(NULL)
 {
   cafile = X509CA.getData();
   crlfile = X509CRL.getData();
@@ -116,6 +116,15 @@
     cert_cred = 0;
   }
 
+  if (tlsis) {
+    delete tlsis;
+    tlsis = NULL;
+  }
+  if (tlsos) {
+    delete tlsos;
+    tlsos = NULL;
+  }
+
   if (session) {
     gnutls_deinit(session);
     session = 0;
@@ -127,11 +136,6 @@
 {
   shutdown(true);
 
-  if (fis)
-    delete fis;
-  if (fos)
-    delete fos;
-
   delete[] cafile;
   delete[] crlfile;
 
@@ -165,17 +169,16 @@
       throw AuthFailureException("gnutls_set_default_priority failed");
 
     setParam();
-  }
 
-  rdr::TLSInStream *tlsis = new rdr::TLSInStream(is, session);
-  rdr::TLSOutStream *tlsos = new rdr::TLSOutStream(os, session);
+    // Create these early as they set up the push/pull functions
+    // for GnuTLS
+    tlsis = new rdr::TLSInStream(is, session);
+    tlsos = new rdr::TLSOutStream(os, session);
+  }
 
   int err;
   err = gnutls_handshake(session);
   if (err != GNUTLS_E_SUCCESS) {
-    delete tlsis;
-    delete tlsos;
-
     if (!gnutls_error_is_fatal(err))
       return false;
 
@@ -186,7 +189,7 @@
 
   checkSession();
 
-  cc->setStreams(fis = tlsis, fos = tlsos);
+  cc->setStreams(tlsis, tlsos);
 
   return true;
 }
diff --git a/common/rfb/CSecurityTLS.h b/common/rfb/CSecurityTLS.h
index 6791a4a..0d5f899 100644
--- a/common/rfb/CSecurityTLS.h
+++ b/common/rfb/CSecurityTLS.h
@@ -69,8 +69,9 @@
     bool anon;
 
     char *cafile, *crlfile;
-    rdr::InStream* fis;
-    rdr::OutStream* fos;
+
+    rdr::InStream* tlsis;
+    rdr::OutStream* tlsos;
   };
 }
 
diff --git a/common/rfb/SSecurityTLS.cxx b/common/rfb/SSecurityTLS.cxx
index 72b83db..bf77b9b 100644
--- a/common/rfb/SSecurityTLS.cxx
+++ b/common/rfb/SSecurityTLS.cxx
@@ -51,7 +51,7 @@
 
 SSecurityTLS::SSecurityTLS(SConnection* sc, bool _anon)
   : SSecurity(sc), session(NULL), dh_params(NULL), anon_cred(NULL),
-    cert_cred(NULL), anon(_anon), fis(NULL), fos(NULL)
+    cert_cred(NULL), anon(_anon), tlsis(NULL), tlsos(NULL)
 {
   certfile = X509_CertFile.getData();
   keyfile = X509_KeyFile.getData();
@@ -84,6 +84,15 @@
     cert_cred = 0;
   }
 
+  if (tlsis) {
+    delete tlsis;
+    tlsis = NULL;
+  }
+  if (tlsos) {
+    delete tlsos;
+    tlsos = NULL;
+  }
+
   if (session) {
     gnutls_deinit(session);
     session = 0;
@@ -95,11 +104,6 @@
 {
   shutdown();
 
-  if (fis)
-    delete fis;
-  if (fos)
-    delete fos;
-
   delete[] keyfile;
   delete[] certfile;
 
@@ -108,12 +112,12 @@
 
 bool SSecurityTLS::processMsg()
 {
-  rdr::InStream* is = sc->getInStream();
-  rdr::OutStream* os = sc->getOutStream();
-
   vlog.debug("Process security message (session %p)", session);
 
   if (!session) {
+    rdr::InStream* is = sc->getInStream();
+    rdr::OutStream* os = sc->getOutStream();
+
     if (gnutls_init(&session, GNUTLS_SERVER) != GNUTLS_E_SUCCESS)
       throw AuthFailureException("gnutls_init failed");
 
@@ -130,17 +134,16 @@
 
     os->writeU8(1);
     os->flush();
-  }
 
-  rdr::TLSInStream *tlsis = new rdr::TLSInStream(is, session);
-  rdr::TLSOutStream *tlsos = new rdr::TLSOutStream(os, session);
+    // Create these early as they set up the push/pull functions
+    // for GnuTLS
+    tlsis = new rdr::TLSInStream(is, session);
+    tlsos = new rdr::TLSOutStream(os, session);
+  }
 
   int err;
   err = gnutls_handshake(session);
   if (err != GNUTLS_E_SUCCESS) {
-    delete tlsis;
-    delete tlsos;
-
     if (!gnutls_error_is_fatal(err)) {
       vlog.debug("Deferring completion of TLS handshake: %s", gnutls_strerror(err));
       return false;
@@ -152,7 +155,7 @@
 
   vlog.debug("Handshake completed");
 
-  sc->setStreams(fis = tlsis, fos = tlsos);
+  sc->setStreams(tlsis, tlsos);
 
   return true;
 }
diff --git a/common/rfb/SSecurityTLS.h b/common/rfb/SSecurityTLS.h
index e137b28..6d32e3b 100644
--- a/common/rfb/SSecurityTLS.h
+++ b/common/rfb/SSecurityTLS.h
@@ -63,8 +63,8 @@
     int type;
     bool anon;
 
-    rdr::InStream* fis;
-    rdr::OutStream* fos;
+    rdr::InStream* tlsis;
+    rdr::OutStream* tlsos;
   };
 
 }