diff --git a/package.json b/package.json
index 6ccdc7c..5e69680 100644
--- a/package.json
+++ b/package.json
@@ -28,6 +28,7 @@
     "debug": "^3.1.0",
     "interface-connection": "^0.3.2",
     "pull-cat": "^1.1.11",
+    "pull-defer": "^0.2.2",
     "pull-handshake": "^1.1.4",
     "pull-reader": "^1.2.9",
     "pull-stream": "^3.6.7",
diff --git a/src/crypto.js b/src/crypto.js
index 1f1edd3..9c61f26 100644
--- a/src/crypto.js
+++ b/src/crypto.js
@@ -5,7 +5,6 @@ const debug = require('debug')
 const Errors = require('./errors')
 const xsalsa20 = require('xsalsa20')
 const KEY_LENGTH = require('./key-generator').KEY_LENGTH
-const NONCE_LENGTH = require('./key-generator').NONCE_LENGTH
 
 const log = debug('libp2p:pnet')
 log.trace = debug('libp2p:pnet:trace')
@@ -41,10 +40,6 @@ module.exports.createUnboxStream = (remote, psk) => {
     ensureBuffer(),
     pull.map((chunk) => {
       if (!xor) {
-        if (!remote.nonce || remote.nonce.byteLength !== NONCE_LENGTH) {
-          log.trace('No nonce was read, throwing an error')
-          throw new Error(Errors.INVALID_PEER)
-        }
         xor = xsalsa20(remote.nonce, psk)
         log.trace('Decryption enabled')
       }
diff --git a/src/state.js b/src/state.js
index f288427..b084e26 100644
--- a/src/state.js
+++ b/src/state.js
@@ -6,12 +6,14 @@ const pair = require('pull-pair')
 const Reader = require('pull-reader')
 const cat = require('pull-cat')
 const pull = require('pull-stream')
+const deferred = require('pull-defer')
 
 const cryptoStreams = require('./crypto')
 const NONCE_LENGTH = require('./key-generator').NONCE_LENGTH
 
 const log = debug('libp2p:pnet')
 log.err = debug('libp2p:pnet:err')
+log.trace = debug('libp2p:pnet:trace')
 
 /**
  * Keeps track of the state of a given connection, such as the local psk
@@ -67,19 +69,21 @@ class State {
     // The outer stream needs to be returned before we setup the
     // rest of the streams, so we're delaying the execution
     setTimeout(() => {
-      // Read the nonce first, this queues up the read when data
-      // is available, so we will have the nonce before any
-      // decryption occurs
+      // Read the nonce first, once we have it resolve the
+      // deferred source, so we keep reading
+      const deferredSource = deferred.source()
       this.rawReader.read(NONCE_LENGTH, (err, data) => {
         if (err) {
           log.err('There was an error attempting to read the nonce', err)
         }
+        log.trace('remote nonce received')
         this.remote.nonce = data
+        deferredSource.resolve(this.rawReader.read())
       })
 
       this.innerRawStream = {
         sink: this.rawPairStream.sink,
-        source: this.rawReader.read()
+        source: deferredSource
       }
 
       // Create the pull exchange between the two inner streams