Socket hang up #285

52cs · 2022-08-29T06:32:28Z

Socket hang up when using with opensearch:2.2.0 but elasticsearch works OK

ConnectionError: socket hang up
    at ClientRequest.onError (/home/jim/project/private-topcoder-repo/node_modules/@opensearch-project/opensearch/lib/Connection.js:130:16)
    at ClientRequest.emit (node:events:513:28)
    at Socket.socketOnEnd (node:_http_client:505:9)
    at Socket.emit (node:events:525:35)
    at endReadableNT (node:internal/streams/readable:1358:12)
    at processTicksAndRejections (node:internal/process/task_queues:83:21) {
  meta: {
    body: null,
    statusCode: null,
    headers: null,
    meta: {
      context: null,
      request: [Object],
      name: 'opensearch-js',
      connection: [Object],
      attempts: 3,
      aborted: false
    }
  }
}

error: [AxiosError: Request failed with status code 400] {
  code: 'ERR_BAD_REQUEST',
  config: {
    transitional: {
      silentJSONParsing: true,
      forcedJSONParsing: true,
      clarifyTimeoutError: false
    },
    adapter: [Function: httpAdapter],
    transformRequest: [ [Function: transformRequest] ],
    transformResponse: [ [Function: transformResponse] ],
    timeout: 0,
    xsrfCookieName: 'XSRF-TOKEN',
    xsrfHeaderName: 'X-XSRF-TOKEN',
    maxContentLength: -1,
    maxBodyLength: -1,
    env: { FormData: [Function] },
    validateStatus: [Function: validateStatus],
    headers: {
      Accept: 'application/json, text/plain, */*',
      'User-Agent': 'axios/0.27.2',
      host: 'topcoder.looker.com'
    },
    method: 'get',
    url: 'https://topcoder.looker.com/looks/QWMnPdYcHvrRj8M6CFXQ97vG83fNC9DK.json?apply_formatting=true',
    data: undefined
  },
  request: <ref *1> ClientRequest {
    _events: [Object: null prototype] {
      abort: [Function (anonymous)],
      aborted: [Function (anonymous)],
      connect: [Function (anonymous)],
      error: [Function (anonymous)],
      socket: [Function (anonymous)],
      timeout: [Function (anonymous)],
      prefinish: [Function: requestOnPrefinish]
    },
    _eventsCount: 7,
    _maxListeners: undefined,
    outputData: [],
    outputSize: 0,
    writable: true,
    destroyed: false,
    _last: true,
    chunkedEncoding: false,
    shouldKeepAlive: false,
    maxRequestsOnConnectionReached: false,
    _defaultKeepAlive: true,
    useChunkedEncodingByDefault: false,
    sendDate: false,
    _removedConnection: false,
    _removedContLen: false,
    _removedTE: false,
    _contentLength: 0,
    _hasBody: true,
    _trailer: '',
    finished: true,
    _headerSent: true,
    _closed: false,
    socket: Socket {
      connecting: false,
      _hadError: false,
      _parent: null,
      _host: null,
      _readableState: [ReadableState],
      _events: [Object: null prototype],
      _eventsCount: 7,
      _maxListeners: undefined,
      _writableState: [WritableState],
      allowHalfOpen: false,
      _sockname: null,
      _pendingData: null,
      _pendingEncoding: '',
      server: null,
      _server: null,
      parser: null,
      _httpMessage: [Circular *1],
      [Symbol(async_id_symbol)]: 68,
      [Symbol(kHandle)]: [TCP],
      [Symbol(lastWriteQueueSize)]: 0,
      [Symbol(timeout)]: null,
      [Symbol(kBuffer)]: null,
      [Symbol(kBufferCb)]: null,
      [Symbol(kBufferGen)]: null,
      [Symbol(kCapture)]: false,
      [Symbol(kSetNoDelay)]: false,
      [Symbol(kSetKeepAlive)]: true,
      [Symbol(kSetKeepAliveInitialDelay)]: 60,
      [Symbol(kBytesRead)]: 0,
      [Symbol(kBytesWritten)]: 0,
      [Symbol(RequestTimeout)]: undefined
    },
    _header: 'GET https://topcoder.looker.com/looks/QWMnPdYcHvrRj8M6CFXQ97vG83fNC9DK.json?apply_formatting=true HTTP/1.1\r\n' +
      'Accept: application/json, text/plain, */*\r\n' +
      'User-Agent: axios/0.27.2\r\n' +
      'host: topcoder.looker.com\r\n' +
      'Connection: close\r\n' +
      '\r\n',
    _keepAliveTimeout: 0,
    _onPendingData: [Function: nop],
    agent: Agent {
      _events: [Object: null prototype],
      _eventsCount: 2,
      _maxListeners: undefined,
      defaultPort: 80,
      protocol: 'http:',
      options: [Object: null prototype],
      requests: [Object: null prototype] {},
      sockets: [Object: null prototype],
      freeSockets: [Object: null prototype] {},
      keepAliveMsecs: 1000,
      keepAlive: false,
      maxSockets: Infinity,
      maxFreeSockets: 256,
      scheduling: 'lifo',
      maxTotalSockets: Infinity,
      totalSocketCount: 1,
      [Symbol(kCapture)]: false
    },
    socketPath: undefined,
    method: 'GET',
    maxHeaderSize: undefined,
    insecureHTTPParser: undefined,
    path: 'https://topcoder.looker.com/looks/QWMnPdYcHvrRj8M6CFXQ97vG83fNC9DK.json?apply_formatting=true',
    _ended: true,
    res: IncomingMessage {
      _readableState: [ReadableState],
      _events: [Object: null prototype],
      _eventsCount: 4,
      _maxListeners: undefined,
      socket: [Socket],
      httpVersionMajor: 1,
      httpVersionMinor: 1,
      httpVersion: '1.1',
      complete: true,
      rawHeaders: [Array],
      rawTrailers: [],
      aborted: false,
      upgrade: false,
      url: '',
      method: null,
      statusCode: 400,
      statusMessage: 'Bad Request',
      client: [Socket],
      _consuming: false,
      _dumped: false,
      req: [Circular *1],
      responseUrl: 'https://topcoder.looker.com/looks/QWMnPdYcHvrRj8M6CFXQ97vG83fNC9DK.json?apply_formatting=true',
      redirects: [],
      [Symbol(kCapture)]: false,
      [Symbol(kHeaders)]: [Object],
      [Symbol(kHeadersCount)]: 12,
      [Symbol(kTrailers)]: null,
      [Symbol(kTrailersCount)]: 0,
      [Symbol(RequestTimeout)]: undefined
    },
    aborted: false,
    timeoutCb: null,
    upgradeOrConnect: false,
    parser: null,
    maxHeadersCount: null,
    reusedSocket: false,
    host: '127.0.0.1',
    protocol: 'http:',
    _redirectable: Writable {
      _writableState: [WritableState],
      _events: [Object: null prototype],
      _eventsCount: 3,
      _maxListeners: undefined,
      _options: [Object],
      _ended: true,
      _ending: true,
      _redirectCount: 0,
      _redirects: [],
      _requestBodyLength: 0,
      _requestBodyBuffers: [],
      _onNativeResponse: [Function (anonymous)],
      _currentRequest: [Circular *1],
      _currentUrl: 'https://topcoder.looker.com/looks/QWMnPdYcHvrRj8M6CFXQ97vG83fNC9DK.json?apply_formatting=true',
      [Symbol(kCapture)]: false
    },
    [Symbol(kCapture)]: false,
    [Symbol(kNeedDrain)]: false,
    [Symbol(corked)]: 0,
    [Symbol(kOutHeaders)]: [Object: null prototype] {
      accept: [Array],
      'user-agent': [Array],
      host: [Array]
    },
    [Symbol(kUniqueHeaders)]: null
  },
  response: {
    status: 400,
    statusText: 'Bad Request',
    headers: {
      'content-length': '248',
      connection: 'keep-alive',
      'content-type': 'text/html',
      date: 'Mon, 29 Aug 2022 05:59:06 GMT',
      'keep-alive': 'timeout=4',
      'proxy-connection': 'keep-alive'
    },
    config: {
      transitional: [Object],
      adapter: [Function: httpAdapter],
      transformRequest: [Array],
      transformResponse: [Array],
      timeout: 0,
      xsrfCookieName: 'XSRF-TOKEN',
      xsrfHeaderName: 'X-XSRF-TOKEN',
      maxContentLength: -1,
      maxBodyLength: -1,
      env: [Object],
      validateStatus: [Function: validateStatus],
      headers: [Object],
      method: 'get',
      url: 'https://topcoder.looker.com/looks/QWMnPdYcHvrRj8M6CFXQ97vG83fNC9DK.json?apply_formatting=true',
      data: undefined
    },
    request: <ref *1> ClientRequest {
      _events: [Object: null prototype],
      _eventsCount: 7,
      _maxListeners: undefined,
      outputData: [],
      outputSize: 0,
      writable: true,
      destroyed: false,
      _last: true,
      chunkedEncoding: false,
      shouldKeepAlive: false,
      maxRequestsOnConnectionReached: false,
      _defaultKeepAlive: true,
      useChunkedEncodingByDefault: false,
      sendDate: false,
      _removedConnection: false,
      _removedContLen: false,
      _removedTE: false,
      _contentLength: 0,
      _hasBody: true,
      _trailer: '',
      finished: true,
      _headerSent: true,
      _closed: false,
      socket: [Socket],
      _header: 'GET https://topcoder.looker.com/looks/QWMnPdYcHvrRj8M6CFXQ97vG83fNC9DK.json?apply_formatting=true HTTP/1.1\r\n' +
        'Accept: application/json, text/plain, */*\r\n' +
        'User-Agent: axios/0.27.2\r\n' +
        'host: topcoder.looker.com\r\n' +
        'Connection: close\r\n' +
        '\r\n',
      _keepAliveTimeout: 0,
      _onPendingData: [Function: nop],
      agent: [Agent],
      socketPath: undefined,
      method: 'GET',
      maxHeaderSize: undefined,
      insecureHTTPParser: undefined,
      path: 'https://topcoder.looker.com/looks/QWMnPdYcHvrRj8M6CFXQ97vG83fNC9DK.json?apply_formatting=true',
      _ended: true,
      res: [IncomingMessage],
      aborted: false,
      timeoutCb: null,
      upgradeOrConnect: false,
      parser: null,
      maxHeadersCount: null,
      reusedSocket: false,
      host: '127.0.0.1',
      protocol: 'http:',
      _redirectable: [Writable],
      [Symbol(kCapture)]: false,
      [Symbol(kNeedDrain)]: false,
      [Symbol(corked)]: 0,
      [Symbol(kOutHeaders)]: [Object: null prototype],
      [Symbol(kUniqueHeaders)]: null
    },
    data: '<html>\r\n' +
      '<head><title>400 The plain HTTP request was sent to HTTPS port</title></head>\r\n' +
      '<body>\r\n' +
      '<center><h1>400 Bad Request</h1></center>\r\n' +
      '<center>The plain HTTP request was sent to HTTPS port</center>\r\n' +
      '<hr><center>nginx</center>\r\n' +
      '</body>\r\n' +
      '</html>\r\n'
  },
  retry: 0
}
error: undefined

The text was updated successfully, but these errors were encountered:

dblock · 2022-08-29T15:30:10Z

I see a 400 The plain HTTP request was sent to HTTPS port which tells me that somewhere you have SSL configured (OpenSearch?) whereas it wasn't before (with ES?), yet the client isn't aware of it and thinks it's talking to a non-SSL endpoint.

I see a bunch of requests in the above to looker.com which doesn't seem related.

Neither of those is the error from the OpenSearch connection, care to try and add more information on what changed?

52cs · 2022-08-29T21:53:14Z

@dblock
Thanks for your reply!
Sorry for the confusion.

Please pay attention to the first error, the second error is related to axios behind proxy, I will develop a tool to get rid of it in the future.

Here is some code snip to help you better understand the errors.

const { Client } = require('@opensearch-project/opensearch')
// const { Client } = require('@elastic/elasticsearch')

let esClient

function getESClient () {
  if (esClient) {
    return esClient
  }
  esClient = new Client({
    node: config.OPENSEARCH_HOST
  })  
  return esClient
}

class Heatmap {
  constructor() {
    this.config = config.opensearchConfig
    logger.info(config)
    this.client = helper.getESClient()
  }

  async setup() {
    let exists = await this.client.indices.exists({ index: this.config.index })
    ...
  }
}

heatmap = new Heatmap()
  try {
    await heatmap.setup()
  } catch (err) {
    console.log(err)                                // not throw err here, so we see the second error
  }

The error is coming from this line, the first execution line of opensearch which complain that socket hang up:

let exists = await this.client.indices.exists({ index: this.config.index })

ConnectionError: socket hang up
    at ClientRequest.onError (/home/jim/project/private-topcoder-repo/node_modules/@opensearch-project/opensearch/lib/Connection.js:130:16)
    at ClientRequest.emit (node:events:513:28)
    at Socket.socketOnEnd (node:_http_client:505:9)
    at Socket.emit (node:events:525:35)
    at endReadableNT (node:internal/streams/readable:1358:12)
    at processTicksAndRejections (node:internal/process/task_queues:83:21) {
  meta: {
    body: null,
    statusCode: null,
    headers: null,
    meta: {
      context: null,
      request: [Object],
      name: 'opensearch-js',
      connection: [Object],
      attempts: 3,
      aborted: false
    }
  }
}

dblock · 2022-09-01T15:51:14Z

What version of the server are you talking to? Any errors on the server? Can we see the HTTP request/response log before the HUP?

52cs · 2022-09-01T23:52:33Z

server version: opensearchproject/opensearch:2.2.0
client side config: (no config for SSL)

const { Client } = require('@opensearch-project/opensearch')
// const { Client } = require('@elastic/elasticsearch')

let esClient

function getESClient () {
  if (esClient) {
    return esClient
  }
  esClient = new Client({
    node: config.OPENSEARCH_HOST
  })  
  return esClient
}

api call which triggers the error: (this is the first api call, opensearch should use a lazy connection)

async setup() {
    let exists = await this.client.indices.exists({ index: this.config.index })
    ...
  }

server side error: (seems SSL related error, but there is no error when using elasticsearch client with the same setting)

fix-opensearch-1  | [2022-09-01T23:40:19,381][ERROR][o.o.s.s.h.n.SecuritySSLNettyHttpServerTransport] [8858e9709006] Exception during establishing a SSL connection: io.netty.handler.ssl.NotSslRecordException: not an SSL/TLS record: 48454144202f746563683420485454502f312e310d0a757365722d6167656e743a206f70656e7365617263682d6a732f322e302e3020286c696e757820352e31352e302d34362d67656e657269632d7836343b204e6f64652e6a73207631362e31372e30290d0a486f73743a206c6f63616c686f73743a393230300d0a436f6e6e656374696f6e3a206b6565702d616c6976650d0a0d0a
fix-opensearch-1  | io.netty.handler.ssl.NotSslRecordException: not an SSL/TLS record: 48454144202f746563683420485454502f312e310d0a757365722d6167656e743a206f70656e7365617263682d6a732f322e302e3020286c696e757820352e31352e302d34362d67656e657269632d7836343b204e6f64652e6a73207631362e31372e30290d0a486f73743a206c6f63616c686f73743a393230300d0a436f6e6e656374696f6e3a206b6565702d616c6976650d0a0d0a
fix-opensearch-1  | 	at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1215) ~[netty-handler-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1285) ~[netty-handler-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:510) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:449) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:279) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at java.lang.Thread.run(Thread.java:833) [?:?]
fix-opensearch-1  | [2022-09-01T23:40:19,381][ERROR][o.o.s.s.h.n.SecuritySSLNettyHttpServerTransport] [8858e9709006] Exception during establishing a SSL connection: io.netty.handler.ssl.NotSslRecordException: not an SSL/TLS record: 48454144202f746563683420485454502f312e310d0a757365722d6167656e743a206f70656e7365617263682d6a732f322e302e3020286c696e757820352e31352e302d34362d67656e657269632d7836343b204e6f64652e6a73207631362e31372e30290d0a486f73743a206c6f63616c686f73743a393230300d0a436f6e6e656374696f6e3a206b6565702d616c6976650d0a0d0a
fix-opensearch-1  | io.netty.handler.ssl.NotSslRecordException: not an SSL/TLS record: 48454144202f746563683420485454502f312e310d0a757365722d6167656e743a206f70656e7365617263682d6a732f322e302e3020286c696e757820352e31352e302d34362d67656e657269632d7836343b204e6f64652e6a73207631362e31372e30290d0a486f73743a206c6f63616c686f73743a393230300d0a436f6e6e656374696f6e3a206b6565702d616c6976650d0a0d0a
fix-opensearch-1  | 	at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1215) ~[netty-handler-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1285) ~[netty-handler-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:510) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:449) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:279) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at java.lang.Thread.run(Thread.java:833) [?:?]
fix-opensearch-1  | [2022-09-01T23:40:19,383][ERROR][o.o.s.s.h.n.SecuritySSLNettyHttpServerTransport] [8858e9709006] Exception during establishing a SSL connection: io.netty.handler.ssl.NotSslRecordException: not an SSL/TLS record: 48454144202f746563683420485454502f312e310d0a757365722d6167656e743a206f70656e7365617263682d6a732f322e302e3020286c696e757820352e31352e302d34362d67656e657269632d7836343b204e6f64652e6a73207631362e31372e30290d0a486f73743a206c6f63616c686f73743a393230300d0a436f6e6e656374696f6e3a206b6565702d616c6976650d0a0d0a
fix-opensearch-1  | io.netty.handler.ssl.NotSslRecordException: not an SSL/TLS record: 48454144202f746563683420485454502f312e310d0a757365722d6167656e743a206f70656e7365617263682d6a732f322e302e3020286c696e757820352e31352e302d34362d67656e657269632d7836343b204e6f64652e6a73207631362e31372e30290d0a486f73743a206c6f63616c686f73743a393230300d0a436f6e6e656374696f6e3a206b6565702d616c6976650d0a0d0a
fix-opensearch-1  | 	at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1215) ~[netty-handler-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1285) ~[netty-handler-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:510) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:449) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:279) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at java.lang.Thread.run(Thread.java:833) [?:?]
fix-opensearch-1  | [2022-09-01T23:40:19,388][WARN ][o.o.h.AbstractHttpServerTransport] [8858e9709006] caught exception while handling client http traffic, closing connection Netty4HttpChannel{localAddress=/172.25.0.2:9200, remoteAddress=/172.25.0.1:46584}
fix-opensearch-1  | io.netty.handler.codec.DecoderException: io.netty.handler.ssl.NotSslRecordException: not an SSL/TLS record: 48454144202f746563683420485454502f312e310d0a757365722d6167656e743a206f70656e7365617263682d6a732f322e302e3020286c696e757820352e31352e302d34362d67656e657269632d7836343b204e6f64652e6a73207631362e31372e30290d0a486f73743a206c6f63616c686f73743a393230300d0a436f6e6e656374696f6e3a206b6565702d616c6976650d0a0d0a
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:480) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:279) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at java.lang.Thread.run(Thread.java:833) [?:?]
fix-opensearch-1  | Caused by: io.netty.handler.ssl.NotSslRecordException: not an SSL/TLS record: 48454144202f746563683420485454502f312e310d0a757365722d6167656e743a206f70656e7365617263682d6a732f322e302e3020286c696e757820352e31352e302d34362d67656e657269632d7836343b204e6f64652e6a73207631362e31372e30290d0a486f73743a206c6f63616c686f73743a393230300d0a436f6e6e656374696f6e3a206b6565702d616c6976650d0a0d0a
fix-opensearch-1  | 	at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1215) ~[netty-handler-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1285) ~[netty-handler-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:510) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:449) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	... 16 more
fix-opensearch-1  | [2022-09-01T23:40:19,388][WARN ][o.o.h.AbstractHttpServerTransport] [8858e9709006] caught exception while handling client http traffic, closing connection Netty4HttpChannel{localAddress=/172.25.0.2:9200, remoteAddress=/172.25.0.1:46588}
fix-opensearch-1  | io.netty.handler.codec.DecoderException: io.netty.handler.ssl.NotSslRecordException: not an SSL/TLS record: 48454144202f746563683420485454502f312e310d0a757365722d6167656e743a206f70656e7365617263682d6a732f322e302e3020286c696e757820352e31352e302d34362d67656e657269632d7836343b204e6f64652e6a73207631362e31372e30290d0a486f73743a206c6f63616c686f73743a393230300d0a436f6e6e656374696f6e3a206b6565702d616c6976650d0a0d0a
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:480) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:279) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at java.lang.Thread.run(Thread.java:833) [?:?]
fix-opensearch-1  | Caused by: io.netty.handler.ssl.NotSslRecordException: not an SSL/TLS record: 48454144202f746563683420485454502f312e310d0a757365722d6167656e743a206f70656e7365617263682d6a732f322e302e3020286c696e757820352e31352e302d34362d67656e657269632d7836343b204e6f64652e6a73207631362e31372e30290d0a486f73743a206c6f63616c686f73743a393230300d0a436f6e6e656374696f6e3a206b6565702d616c6976650d0a0d0a
fix-opensearch-1  | 	at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1215) ~[netty-handler-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1285) ~[netty-handler-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:510) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:449) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	... 16 more
fix-opensearch-1  | [2022-09-01T23:40:19,388][ERROR][o.o.s.s.h.n.SecuritySSLNettyHttpServerTransport] [8858e9709006] Exception during establishing a SSL connection: io.netty.handler.ssl.NotSslRecordException: not an SSL/TLS record: 48454144202f746563683420485454502f312e310d0a757365722d6167656e743a206f70656e7365617263682d6a732f322e302e3020286c696e757820352e31352e302d34362d67656e657269632d7836343b204e6f64652e6a73207631362e31372e30290d0a486f73743a206c6f63616c686f73743a393230300d0a436f6e6e656374696f6e3a206b6565702d616c6976650d0a0d0a
fix-opensearch-1  | io.netty.handler.ssl.NotSslRecordException: not an SSL/TLS record: 48454144202f746563683420485454502f312e310d0a757365722d6167656e743a206f70656e7365617263682d6a732f322e302e3020286c696e757820352e31352e302d34362d67656e657269632d7836343b204e6f64652e6a73207631362e31372e30290d0a486f73743a206c6f63616c686f73743a393230300d0a436f6e6e656374696f6e3a206b6565702d616c6976650d0a0d0a
fix-opensearch-1  | 	at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1215) ~[netty-handler-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1285) ~[netty-handler-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:510) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:449) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:279) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at java.lang.Thread.run(Thread.java:833) [?:?]
fix-opensearch-1  | [2022-09-01T23:40:19,390][WARN ][o.o.h.AbstractHttpServerTransport] [8858e9709006] caught exception while handling client http traffic, closing connection Netty4HttpChannel{localAddress=/172.25.0.2:9200, remoteAddress=/172.25.0.1:46590}
fix-opensearch-1  | io.netty.handler.codec.DecoderException: io.netty.handler.ssl.NotSslRecordException: not an SSL/TLS record: 48454144202f746563683420485454502f312e310d0a757365722d6167656e743a206f70656e7365617263682d6a732f322e302e3020286c696e757820352e31352e302d34362d67656e657269632d7836343b204e6f64652e6a73207631362e31372e30290d0a486f73743a206c6f63616c686f73743a393230300d0a436f6e6e656374696f6e3a206b6565702d616c6976650d0a0d0a
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:480) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:279) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at java.lang.Thread.run(Thread.java:833) [?:?]
fix-opensearch-1  | Caused by: io.netty.handler.ssl.NotSslRecordException: not an SSL/TLS record: 48454144202f746563683420485454502f312e310d0a757365722d6167656e743a206f70656e7365617263682d6a732f322e302e3020286c696e757820352e31352e302d34362d67656e657269632d7836343b204e6f64652e6a73207631362e31372e30290d0a486f73743a206c6f63616c686f73743a393230300d0a436f6e6e656374696f6e3a206b6565702d616c6976650d0a0d0a
fix-opensearch-1  | 	at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1215) ~[netty-handler-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1285) ~[netty-handler-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:510) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:449) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	... 16 more
fix-opensearch-1  | [2022-09-01T23:40:19,390][WARN ][o.o.h.AbstractHttpServerTransport] [8858e9709006] caught exception while handling client http traffic, closing connection Netty4HttpChannel{localAddress=/172.25.0.2:9200, remoteAddress=/172.25.0.1:46592}
fix-opensearch-1  | io.netty.handler.codec.DecoderException: io.netty.handler.ssl.NotSslRecordException: not an SSL/TLS record: 48454144202f746563683420485454502f312e310d0a757365722d6167656e743a206f70656e7365617263682d6a732f322e302e3020286c696e757820352e31352e302d34362d67656e657269632d7836343b204e6f64652e6a73207631362e31372e30290d0a486f73743a206c6f63616c686f73743a393230300d0a436f6e6e656374696f6e3a206b6565702d616c6976650d0a0d0a
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:480) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:279) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:722) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:623) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:586) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:496) [netty-transport-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [netty-common-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at java.lang.Thread.run(Thread.java:833) [?:?]
fix-opensearch-1  | Caused by: io.netty.handler.ssl.NotSslRecordException: not an SSL/TLS record: 48454144202f746563683420485454502f312e310d0a757365722d6167656e743a206f70656e7365617263682d6a732f322e302e3020286c696e757820352e31352e302d34362d67656e657269632d7836343b204e6f64652e6a73207631362e31372e30290d0a486f73743a206c6f63616c686f73743a393230300d0a436f6e6e656374696f6e3a206b6565702d616c6976650d0a0d0a
fix-opensearch-1  | 	at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1215) ~[netty-handler-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1285) ~[netty-handler-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:510) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:449) ~[netty-codec-4.1.79.Final.jar:4.1.79.Final]
fix-opensearch-1  | 	... 16 more
fix-opensearch-1  | [2022-09-01T23:40:19,393][INFO ][o.o.c.m.MetadataCreateIndexService] [8858e9709006] [security-auditlog-2022.09.01] creating index, cause [auto(bulk api)], templates [], shards [1]/[1]
fix-opensearch-1  | [2022-09-01T23:40:19,459][INFO ][o.o.a.u.d.DestinationMigrationCoordinator] [8858e9709006] Detected cluster change event for destination migration
fix-opensearch-1  | [2022-09-01T23:40:19,563][INFO ][o.o.a.u.d.DestinationMigrationCoordinator] [8858e9709006] Detected cluster change event for destination migration
fix-opensearch-1  | [2022-09-01T23:40:19,589][INFO ][o.o.c.m.MetadataMappingService] [8858e9709006] [security-auditlog-2022.09.01/CQwebf4xRbO7UoKOhLPk0Q] create_mapping
fix-opensearch-1  | [2022-09-01T23:40:19,627][INFO ][o.o.a.u.d.DestinationMigrationCoordinator] [8858e9709006] Detected cluster change event for destination migration

client side error:

{"name":"ConnectionError","meta":{"body":null,"statusCode":null,"headers":null,"meta":{"context":null,"request":{"params":{"method":"HEAD","path":"/tech4","body":null,"querystring":"","headers":{"user-agent":"opensearch-js/2.0.0 (linux 5.15.0-46-generic-x64; Node.js v16.17.0)"},"timeout":30000},"options":{},"id":1},"name":"opensearch-js","connection":{"url":"http://localhost:9200/","id":"http://localhost:9200/","headers":{},"deadCount":4,"resurrectTimeout":1662076099388,"_openRequests":0,"status":"dead","roles":{"data":true,"ingest":true}},"attempts":3,"aborted":false}}}
error: ConnectionError: socket hang up
    at ClientRequest.onError (/home/jim/project/private-topcoder-repo/node_modules/@opensearch-project/opensearch/lib/Connection.js:130:16)
    at ClientRequest.emit (node:events:513:28)
    at Socket.socketOnEnd (node:_http_client:505:9)
    at Socket.emit (node:events:525:35)
    at endReadableNT (node:internal/streams/readable:1358:12)
    at processTicksAndRejections (node:internal/process/task_queues:83:21) {
  meta: {
    body: null,
    statusCode: null,
    headers: null,
    meta: {
      context: null,
      request: [Object],
      name: 'opensearch-js',
      connection: [Object],
      attempts: 3,
      aborted: false
    }
  },
  retry: 0
}
error: ConnectionError: socket hang up
    at ClientRequest.onError (/home/jim/project/private-topcoder-repo/node_modules/@opensearch-project/opensearch/lib/Connection.js:130:16)
    at ClientRequest.emit (node:events:513:28)
    at Socket.socketOnEnd (node:_http_client:505:9)
    at Socket.emit (node:events:525:35)
    at endReadableNT (node:internal/streams/readable:1358:12)
    at processTicksAndRejections (node:internal/process/task_queues:83:21)

Re-paste the client side error detail for better view:
{"name":"ConnectionError","meta":{"body":null,"statusCode":null,"headers":null,"meta":{"context":null,"request":{"params":{"method":"HEAD","path":"/tech4","body":null,"querystring":"","headers":{"user-agent":"opensearch-js/2.0.0 (linux 5.15.0-46-generic-x64; Node.js v16.17.0)"},"timeout":30000},"options":{},"id":1},"name":"opensearch-js","connection":{"url":"http://localhost:9200/","id":"http://localhost:9200/","headers":{},"deadCount":4,"resurrectTimeout":1662076099388,"_openRequests":0,"status":"dead","roles":{"data":true,"ingest":true}},"attempts":3,"aborted":false}}}

My docker-compose.yml:

version: '3' 
services:
  opensearch:
    image: opensearchproject/opensearch:2.2.0
    environment:
      - discovery.type=single-node
      - bootstrap.memory_lock=true
    ports:
      - '9200:9200'
      - '9600:9600'

rochdev · 2022-09-02T18:48:12Z

I'm getting the same error. Even a simple client.ping() results in a socket hang up error when the exact same code works perfectly with Elasticsearch as the server. In both cases the image from Docker Hub is used in single node mode.

Reproduction:

docker-compose.yml

version: "2"
services:
  opensearch:
    image: opensearchproject/opensearch
    environment:
      - discovery.type=single-node
    ports:
      - "127.0.0.1:9200:9200"
      - "127.0.0.1:9600:9600"

test.js

const opensearch = require('@opensearch-project/opensearch')

const client = new opensearch.Client({
  node: 'http://localhost:9200'
})

client.ping().catch(err => {
  console.log(err)
})

I tried using an older version of OpenSearch and got the same result.

52cs · 2022-09-02T22:32:27Z

My docker-compose.yml:

version: '3' 
services:
  opensearch:
    image: opensearchproject/opensearch:2.2.0
    environment:
      - discovery.type=single-node
      - bootstrap.memory_lock=true
    ports:
      - '9200:9200'
      - '9600:9600'

dblock · 2022-09-05T15:58:14Z

I think I know where this confusion comes from. Opensearch has SSL enabled by default using self-signed certificates, unlike Elasticsearch that doesn’t include SSL by default.

If you look at the output of docker-compose up with the above file you'll see this:

"/usr/share/opensearch/plugins/opensearch-security/tools/securityadmin.sh" -cd "/usr/share/opensearch/config/opensearch-security" -icl -key "/usr/share/opensearch/config/kirk-key.pem" -cert "/usr/share/opensearch/config/kirk.pem" -cacert "/usr/share/opensearch/config/root-ca.pem" -nhnv

... and then

t-opensearch-1  | [2022-09-05T15:55:11,780][INFO ][o.o.s.s.DefaultSecurityKeyStore] [64c7216ee7de] TLS Transport Client Provider : JDK
t-opensearch-1  | [2022-09-05T15:55:11,781][INFO ][o.o.s.s.DefaultSecurityKeyStore] [64c7216ee7de] TLS Transport Server Provider : JDK
t-opensearch-1  | [2022-09-05T15:55:11,781][INFO ][o.o.s.s.DefaultSecurityKeyStore] [64c7216ee7de] TLS HTTP Provider             : JDK
t-opensearch-1  | [2022-09-05T15:55:11,782][INFO ][o.o.s.s.DefaultSecurityKeyStore] [64c7216ee7de] Enabled TLS protocols for transport layer : [TLSv1.3, TLSv1.2]
t-opensearch-1  | [2022-09-05T15:55:11,798][INFO ][o.o.s.s.DefaultSecurityKeyStore] [64c7216ee7de] Enabled TLS protocols for HTTP layer      : [TLSv1.3, TLSv1.2]

Then curl --insecure -uadmin:admin -v https://localhost:9200 will work, but not curl -uadmin:admin -v http://localhost:9200.

So, what is the value of config.OPENSEARCH_HOST above? It should be https://....

52cs · 2022-09-05T21:09:27Z

Problem has been solved. I should read docs and source code example/test cases carefully. Just curious, why opensearch enable SSL by default? Seems that there is no way to turn it off from JavaScript client side, aka, opensearch-js

…

On Mon, Sep 5, 2022, 11:58 PM Daniel (dB.) Doubrovkine < ***@***.***> wrote: I think I know where this confusion comes from. Opensearch has SSL enabled by default using self-signed certificates, unlike Elasticsearch that doesn’t include SSL by default. If you look at the output of docker-compose up with the above file you'll see this: "/usr/share/opensearch/plugins/opensearch-security/tools/securityadmin.sh" -cd "/usr/share/opensearch/config/opensearch-security" -icl -key "/usr/share/opensearch/config/kirk-key.pem" -cert "/usr/share/opensearch/config/kirk.pem" -cacert "/usr/share/opensearch/config/root-ca.pem" -nhnv ... and then t-opensearch-1 | [2022-09-05T15:55:11,780][INFO ][o.o.s.s.DefaultSecurityKeyStore] [64c7216ee7de] TLS Transport Client Provider : JDK t-opensearch-1 | [2022-09-05T15:55:11,781][INFO ][o.o.s.s.DefaultSecurityKeyStore] [64c7216ee7de] TLS Transport Server Provider : JDK t-opensearch-1 | [2022-09-05T15:55:11,781][INFO ][o.o.s.s.DefaultSecurityKeyStore] [64c7216ee7de] TLS HTTP Provider : JDK t-opensearch-1 | [2022-09-05T15:55:11,782][INFO ][o.o.s.s.DefaultSecurityKeyStore] [64c7216ee7de] Enabled TLS protocols for transport layer : [TLSv1.3, TLSv1.2] t-opensearch-1 | [2022-09-05T15:55:11,798][INFO ][o.o.s.s.DefaultSecurityKeyStore] [64c7216ee7de] Enabled TLS protocols for HTTP layer : [TLSv1.3, TLSv1.2] Then curl --insecure -uadmin:admin -v https://localhost:9200 will work, but not curl -uadmin:admin -v http://localhost:9200. So, what is the value of config.OPENSEARCH_HOST above? It should be https://.... — Reply to this email directly, view it on GitHub <#285 (comment)>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/AW6Q2LIZWIAJ3UEXNWW44J3V4YKCDANCNFSM574RNFRQ> . You are receiving this because you authored the thread.Message ID: ***@***.***>

dblock · 2022-09-06T14:48:52Z

Enabling SSL is part of a strategy to make OpenSearch secure by default to avoid things like https://www.techrepublic.com/article/thousands-of-unprotected-elasticsearch-databases-are-being-ransomed/. See opensearch-project/OpenSearch#2095 for details. Sending credentials over an unencrypted channel is just not ok. But if you want to disable SSL on the server, you can by removing the security configuration. Client then will work just fine without SSL. I don't recommend it.

52cs · 2022-09-06T19:02:37Z

Hmm... It is the official docker image of open search:2.2.0 includes the SSL configuration by default. It is not easy to disable SSL by not modifying the docker image. I will dig into it if necessary.

…

On Tue, Sep 6, 2022, 10:49 PM Daniel (dB.) Doubrovkine < ***@***.***> wrote: Enabling SSL is part of a strategy to make OpenSearch secure by default to avoid things like https://www.techrepublic.com/article/thousands-of-unprotected-elasticsearch-databases-are-being-ransomed/. See opensearch-project/OpenSearch#2095 <opensearch-project/OpenSearch#2095> for details. If you want to disable SSL on the server, you can by removing the security configuration. Client then will work just fine without SSL. I don't recommend it. — Reply to this email directly, view it on GitHub <#285 (comment)>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/AW6Q2LJ3XLQ6LSI2BXEUKX3V45KV5ANCNFSM574RNFRQ> . You are receiving this because you authored the thread.Message ID: ***@***.***>

dblock · 2022-09-06T19:13:08Z

Yes, you're right. The -min distribution doesn't have any configuration, so maybe you can build up from there instead? But I really don't think anyone should be running any services without SSL enabled. What's your scenario?

52cs · 2022-09-06T22:19:53Z

I'm a fresh. Just use it for local development. And not familiar with SSL. Any material of SSL recommend for a starter?

…

On Wed, Sep 7, 2022, 3:13 AM Daniel (dB.) Doubrovkine < ***@***.***> wrote: Yes, you're right. The -min distribution doesn't have any configuration, so you can build up from there. But I really don't think anyone should be running any services without SSL enabled. What's your scenario? — Reply to this email directly, view it on GitHub <#285 (comment)>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/AW6Q2LJB2PJ77PHAQTTFYPTV46JU7ANCNFSM574RNFRQ> . You are receiving this because you authored the thread.Message ID: ***@***.***>

dblock · 2022-09-07T15:51:26Z

No specific recommendations on SSL, but even for local development I think it's pretty easy to just have it enabled. Good habits.

ankur6971 · 2023-11-01T08:39:26Z

I faced similar issue and based on the help provided above, it was a very simple change at my end. My dev setup was a docker cluster of 3 opensearch nodes.

Not working code:

    const host = <A.B.C.D>;
    const protocol = "http";
    const ports = [<Cluster Node Port 1>, <Cluster Node Port 2>, <Cluster Node Port 3>];
    const auth = "admin:SuperSecret"; // For testing only. Don't store credentials in code.

    return new Client({
        nodes: [
            protocol + "://" + auth + "@" + host + ":" + ports[0],
            protocol + "://" + auth + "@" + host + ":" + ports[1],
            protocol + "://" + auth + "@" + host + ":" + ports[2],
        ],
        ssl: {
            rejectUnauthorized: false,
        },
    });

Working code:

    const host = <A.B.C.D>;
    const protocol = "https"; // <-- Changing http to https worked.
    const ports = [<Cluster Node Port 1>, <Cluster Node Port 2>, <Cluster Node Port 3>];
    const auth = "admin:SuperSecret"; // For testing only. Don't store credentials in code.

    return new Client({
        nodes: [
            protocol + "://" + auth + "@" + host + ":" + ports[0],
            protocol + "://" + auth + "@" + host + ":" + ports[1],
            protocol + "://" + auth + "@" + host + ":" + ports[2],
        ],
        ssl: {
            rejectUnauthorized: false,
        },
    });

dblock closed this as completed Sep 6, 2022

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Socket hang up #285

Socket hang up #285

52cs commented Aug 29, 2022

dblock commented Aug 29, 2022 •

edited

Loading

52cs commented Aug 29, 2022

dblock commented Sep 1, 2022 •

edited

Loading

52cs commented Sep 1, 2022 •

edited

Loading

rochdev commented Sep 2, 2022

52cs commented Sep 2, 2022

dblock commented Sep 5, 2022

52cs commented Sep 5, 2022 via email

dblock commented Sep 6, 2022 •

edited

Loading

52cs commented Sep 6, 2022 via email

dblock commented Sep 6, 2022 •

edited

Loading

52cs commented Sep 6, 2022 via email

dblock commented Sep 7, 2022

ankur6971 commented Nov 1, 2023

Socket hang up #285

Socket hang up #285

Comments

52cs commented Aug 29, 2022

dblock commented Aug 29, 2022 • edited Loading

52cs commented Aug 29, 2022

dblock commented Sep 1, 2022 • edited Loading

52cs commented Sep 1, 2022 • edited Loading

rochdev commented Sep 2, 2022

docker-compose.yml

test.js

52cs commented Sep 2, 2022

dblock commented Sep 5, 2022

52cs commented Sep 5, 2022 via email

dblock commented Sep 6, 2022 • edited Loading

52cs commented Sep 6, 2022 via email

dblock commented Sep 6, 2022 • edited Loading

52cs commented Sep 6, 2022 via email

dblock commented Sep 7, 2022

ankur6971 commented Nov 1, 2023

Not working code:

Working code:

dblock commented Aug 29, 2022 •

edited

Loading

dblock commented Sep 1, 2022 •

edited

Loading

52cs commented Sep 1, 2022 •

edited

Loading

dblock commented Sep 6, 2022 •

edited

Loading

dblock commented Sep 6, 2022 •

edited

Loading