Memory Leak in Notifications

[vhpoet]

After doing 100+ transactions with a speed of 2-5tps the ledger started throwing 500 error on payments.

Here are some logs

{"name":"ledger","hostname":"ip-10-0-4-69.us-west-2.compute.internal","pid":28065,"module":"error-handler","level":50,"err":{"message":"not opened","name":"Error","stack":"Error: not opened
    at WebSocket.send (/var/app/five-bells-wallet/node_modules/ws/lib/WebSocket.js:219:16)
    at NotificationBroadcaster.<anonymous> (/var/app/five-bells-wallet/node_modules/five-bells-ledger/src/controllers/accounts.js:179:20)
    at emitOne (events.js:82:20)
    at NotificationBroadcaster.emit (events.js:169:7)
    at NotificationBroadcaster.sendNotifications (/var/app/five-bells-wallet/node_modules/five-bells-ledger/src/lib/notificationBroadcasterWebsocket.js:52:12)
    at next (native)
    at onFulfilled (/var/app/five-bells-wallet/node_modules/co/index.js:65:19)
    at /var/app/five-bells-wallet/node_modules/co/index.js:54:5
    at co (/var/app/five-bells-wallet/node_modules/co/index.js:50:10)
    at toPromise (/var/app/five-bells-wallet/node_modules/co/index.js:118:63)
    at next (/var/app/five-bells-wallet/node_modules/co/index.js:99:29)
    at onFulfilled (/var/app/five-bells-wallet/node_modules/co/index.js:69:7)"},"msg":"not opened","time":"2016-09-14T22:26:43.993Z","v":0}{"name":"ledger","hostname":"ip-10-0-4-69.us-west-2.compute.internal","pid":28065,"module":"error-handler","level":50,"msg":"Error: not opened
    at WebSocket.send (/var/app/five-bells-wallet/node_modules/ws/lib/WebSocket.js:219:16)
    at NotificationBroadcaster.<anonymous> (/var/app/five-bells-wallet/node_modules/five-bells-ledger/src/controllers/accounts.js:179:20)
    at emitOne (events.js:82:20)
    at NotificationBroadcaster.emit (events.js:169:7)
    at NotificationBroadcaster.sendNotifications (/var/app/five-bells-wallet/node_modules/five-bells-ledger/src/lib/notificationBroadcasterWebsocket.js:52:12)
    at next (native)
    at onFulfilled (/var/app/five-bells-wallet/node_modules/co/index.js:65:19)
    at /var/app/five-bells-wallet/node_modules/co/index.js:54:5
    at co (/var/app/five-bells-wallet/node_modules/co/index.js:50:10)
    at toPromise (/var/app/five-bells-wallet/node_modules/co/index.js:118:63)
    at next (/var/app/five-bells-wallet/node_modules/co/index.js:99:29)
    at onFulfilled (/var/app/five-bells-wallet/node_modules/co/index.js:69:7)","time":"2016-09-14T22:26:43.993Z","v":0}{"name":"ledger","hostname":"ip-10-0-4-69.us-west-2.compute.internal","pid":28065,"module":"app","req_id":"5ccda462-758e-4da8-ba79-104dd5be3509","level":50,"duration":29,"status":500,"headers":{"connection":"close","content-length":"716","content-type":"application/json","accept":"application/json","authorization":"Basic YWxpY2U6YWxpY2U=","host":"red.ilpdemo.org"},"msg":"  --> PUT /transfers/5b36f1f7-aaff-495d-8e6c-fed4c714c3be 500 29ms","time":"2016-09-14T22:26:43.994Z","v":0}

(node) warning: possible EventEmitter memory leak detected. 11 transfer-alice listeners added. Use emitter.setMaxListeners() to increase limit.
Trace
    at NotificationBroadcaster.addListener (events.js:239:17)
    at Object.subscribeTransfers (/var/app/five-bells-wallet/node_modules/five-bells-ledger/src/models/accounts.js:107:27)
    at Object.subscribeTransfers (/var/app/five-bells-wallet/node_modules/five-bells-ledger/src/controllers/accounts.js:178:23)
    at next (native)
    at onFulfilled (/var/app/five-bells-wallet/node_modules/co/index.js:65:19)
    at /var/app/five-bells-wallet/node_modules/co/index.js:54:5
    at Object.co (/var/app/five-bells-wallet/node_modules/co/index.js:50:10)
    at Object.toPromise (/var/app/five-bells-wallet/node_modules/co/index.js:118:63)
    at next (/var/app/five-bells-wallet/node_modules/co/index.js:99:29)
    at onFulfilled (/var/app/five-bells-wallet/node_modules/co/index.js:69:7)

[sharafian]

I've also been able to reproduce this error running a ledger on my machine. Sending transfers works fine, and I tried up to 5 tps without seeing any errors whatsoever.

Then I ran plugin-tests against my local ledger (which connects many instances of plugin-bells in short succession), and I got Error: not opened. The tests were unable to send transactions or connect reliably.

I went back to my first step (sending 5 tps) on the same local ledger without restarting, and I was unable to submit any transactions, even though I was running the same code that worked before. I got the same Error: not opened.

I'm using the latest master on ilp-plugin-bells and five-bells-ledger. Here's the stack trace on my ledger:

Error: not opened
        at WebSocket.send (/Users/sharafian/Documents/prog/five-bells-ledger/node_modules/ws/lib/WebSocket.js:219:16)
        at NotificationBroadcaster.model.subscribeTransfers (/Users/sharafian/Documents/prog/five-bells-ledger/src/controllers/accounts.js:179:20)
        at emitOne (events.js:101:20)
        at NotificationBroadcaster.emit (events.js:188:7)
        at NotificationBroadcaster.sendNotifications (/Users/sharafian/Documents/prog/five-bells-ledger/src/lib/notificationBroadcasterWebsocket.js:52:12)
        at next (native)
        at onFulfilled (/Users/sharafian/Documents/prog/five-bells-ledger/node_modules/co/index.js:65:19)
        at /Users/sharafian/Documents/prog/five-bells-ledger/node_modules/co/index.js:54:5
        at co (/Users/sharafian/Documents/prog/five-bells-ledger/node_modules/co/index.js:50:10)
        at toPromise (/Users/sharafian/Documents/prog/five-bells-ledger/node_modules/co/index.js:118:63)
        at next (/Users/sharafian/Documents/prog/five-bells-ledger/node_modules/co/index.js:99:29)
        at onFulfilled (/Users/sharafian/Documents/prog/five-bells-ledger/node_modules/co/index.js:69:7)