Add support for UNIX domain sockets (#120)

* added support for UNIX domain sockets * untested progress * moved cython server string parse to c * regression test for host parse * updated cppcheck suppression * golang bindings * update c_client cppcheck * removed duplicated line * removed github workflow changes * moved unix client to test_unix * missed file * remove atoi usage * warning explanation * removed unused cppcheck suppression * startall for cython valgrind * better string iteration * sorry to commit after requesting review * reverted server string parsing behavior * code hygiene * version bump * syntax credit * version bump not overwritten by pre-commit hook * timezone matching email * padding type * fix port interpretation * revert to existing distribution of responsibilities * deduplicate golang test cases * remove unnecessary inline in header * rewrote to match default branch parsing more closely * more descriptive names * apt-get update before install
douban · Dec 11, 2023 · d32820f · d32820f
1 parent 9522d8a
commit d32820f
Show file tree

Hide file tree

Showing 19 changed files with 260 additions and 70 deletions.
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
@@ -14,6 +14,7 @@ jobs:
     - uses: actions/checkout@v2
     - name: Setup system dependencies
       run: |
+        sudo apt-get update
         sudo apt-get -y install cppcheck
     - name: Run cppcheck
       run: |

diff --git a/.github/workflows/golang.yml b/.github/workflows/golang.yml
@@ -19,6 +19,7 @@ jobs:
     - uses: actions/checkout@v2
     - name: Setup system dependencies
       run: |
+        sudo apt-get update
         sudo apt-get -y install memcached g++
     - name: Set up Golang
       uses: actions/setup-go@v2

diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
@@ -19,6 +19,7 @@ jobs:
     - uses: actions/checkout@v2
     - name: Setup system dependencies
       run: |
+        sudo apt-get update
         sudo apt-get -y install valgrind memcached g++
     - name: Set up Python
       uses: actions/setup-python@v2
@@ -29,14 +30,14 @@ jobs:
         python -m pip install --upgrade pip
         pip install setuptools future pytest greenify gevent numpy
     - name: Start memcached servers
-      run: ./misc/memcached_server start
+      run: ./misc/memcached_server startall
     - name: Run unittest
       run: |
         if [[ ${{ matrix.compiler }} = "gcc" ]]; then export CC=gcc CXX=g++; fi
         if [[ ${{ matrix.compiler }} = "clang" ]]; then export CC=clang CXX=clang++; fi
         ./misc/travis/unittest.sh
     - name: Stop memcached servers
-      run: ./misc/memcached_server stop
+      run: ./misc/memcached_server stopall
 
   benchmark:
     runs-on: ubuntu-latest
@@ -48,6 +49,7 @@ jobs:
     - uses: actions/checkout@v2
     - name: Setup system dependencies
       run: |
+        sudo apt-get update
         sudo apt-get -y install memcached libmemcached-dev g++
     - name: Set up Python
       uses: actions/setup-python@v2

diff --git a/include/Common.h b/include/Common.h
@@ -226,6 +226,8 @@ typedef enum {
 } op_code_t;
 
 const char* errCodeToString(err_code_t err);
+bool isLocalSocket(const char* host);
+server_string_split_t splitServerString(char* input);
 
 } // namespace mc
 } // namespace douban
diff --git a/include/Connection.h b/include/Connection.h
@@ -61,7 +61,8 @@ class Connection {
     size_t m_counter;
 
  protected:
-    int connectPoll(int fd, struct addrinfo* ai_ptr);
+    int connectPoll(int fd, const sockaddr* ai_ptr, const socklen_t ai_addrlen);
+    int unixSocketConnect();
 
     char m_name[MC_NI_MAXHOST + 1 + MC_NI_MAXSERV];
     char m_host[MC_NI_MAXHOST];
@@ -70,6 +71,7 @@ class Connection {
     int m_socketFd;
     bool m_alive;
     bool m_hasAlias;
+    bool m_unixSocket;
     time_t m_deadUntil;
     io::BufferWriter* m_buffer_writer; // for send
     io::BufferReader* m_buffer_reader; // for recv

diff --git a/include/Export.h b/include/Export.h
@@ -38,6 +38,12 @@ typedef enum {
   RET_OK = 0
 } err_code_t;
 
+typedef struct {
+  char* host;
+  char* port;
+  char* alias;
+} server_string_split_t;
+
 
 typedef int64_t exptime_t;
 typedef uint32_t flags_t;

diff --git a/include/c_client.h b/include/c_client.h
@@ -68,6 +68,7 @@ extern "C" {
   err_code_t client_quit(void* client);
 
   const char* err_code_to_string(err_code_t err);
+  server_string_split_t splitServerString(char* input);
 #ifdef __cplusplus
 }
 #endif
diff --git a/libmc/__init__.py b/libmc/__init__.py
@@ -27,11 +27,11 @@
     __file__ as _libmc_so_file
 )
 
-__VERSION__ = "1.4.2"
-__version__ = "v1.4.2"
+__VERSION__ = "1.4.3"
+__version__ = "v1.4.3"
 __author__ = "mckelvin"
 __email__ = "[email protected]"
-__date__ = "Thu May 20 18:44:42 2021 +0800"
+__date__ = "Fri Dec  1 07:43:12 2023 +0800"
 
 
 class Client(PyClient):

diff --git a/libmc/_client.pyx b/libmc/_client.pyx
@@ -45,6 +45,8 @@ cdef extern from "Common.h" namespace "douban::mc":
         VERSION_OP
         QUIT_OP
 
+    server_string_split_t splitServerString(char* input) nogil
+
 
 cdef extern from "Export.h":
     ctypedef enum config_options_t:
@@ -99,6 +101,11 @@ cdef extern from "Export.h":
         RET_INCOMPLETE_BUFFER_ERR
         RET_OK
 
+    ctypedef struct server_string_split_t:
+        char* host
+        char* port
+        char* alias
+
     ctypedef struct unsigned_result_t:
         char* key
         size_t key_len
@@ -378,33 +385,21 @@ cdef class PyClient:
 
         servers_ = []
         for srv in servers:
-            addr_alias = srv.split(' ')
-            addr = addr_alias[0]
-            if len(addr_alias) == 1:
-                alias = None
-            else:
-                alias = addr_alias[1]
-
-            host_port = addr.split(':')
-            host = host_port[0]
-            if len(host_port) == 1:
-                port = MC_DEFAULT_PORT
-            else:
-                port = int(host_port[1])
             if PY_MAJOR_VERSION > 2:
-                host = PyUnicode_AsUTF8String(host)
-                alias = PyUnicode_AsUTF8String(alias) if alias else None
-            servers_.append((host, port, alias))
+                srv = PyUnicode_AsUTF8String(srv)
+            srv = PyString_AsString(srv)
+            servers_.append(srv)
 
         Py_INCREF(servers_)
         for i in range(n):
-            host, port, alias = servers_[i]
-            c_hosts[i] = PyString_AsString(host)
-            c_ports[i] = PyInt_AsLong(port)
-            if alias is None:
-                c_aliases[i] = NULL
+            c_split = splitServerString(servers_[i])
+
+            c_hosts[i] = c_split.host
+            c_aliases[i] = c_split.alias
+            if c_split.port == NULL:
+                c_ports[i] = MC_DEFAULT_PORT
             else:
-                c_aliases[i] = PyString_AsString(alias)
+                c_ports[i] = PyInt_AsLong(int(<bytes>c_split.port))
 
         if init:
             rv = self._imp.init(c_hosts, c_ports, n, c_aliases)

diff --git a/misc/.cppcheck-supp b/misc/.cppcheck-supp
@@ -45,5 +45,5 @@ unusedFunction:src/c_client.cpp:149
 unusedFunction:src/c_client.cpp:154
 unusedFunction:src/c_client.cpp:159
 unusedFunction:src/Utility.cpp:43
-*:src/Connection.cpp:30
-*:src/Connection.cpp:35
+*:src/Connection.cpp:32
+*:src/Connection.cpp:37
diff --git a/misc/memcached_server b/misc/memcached_server
@@ -32,6 +32,18 @@ function start()
     fi
 }
 
+function unix()
+{
+    name="${1:-unix_test}"
+    if [ ! -f $basedir/var/log/${name}.log ]; then
+        mkdir -p $basedir/var/log
+        touch $basedir/var/log/${name}.log
+    fi
+    mkdir -p $basedir/var/run
+    $cmd -d -u $USER -s $basedir/var/run/${name}.socket -t $threads -m ${memory} -P $basedir/var/run/${name}.pid > $basedir/var/log/${name}.log 2>&1
+    echo "Starting the memcached server on '$basedir/var/run/${name}.socket'... "
+}
+
 function stop()
 {
     port="$1"
@@ -41,7 +53,6 @@ function stop()
         kill -TERM `ps -ef | grep "$cmd" | grep $port | grep -v grep | awk '{ print $2 }'`
         echo "Stopping the memcached server on port '$port'... "
     fi
-    rm -rf $basedir
 }
 
 case "$1" in
@@ -64,6 +75,7 @@ case "$1" in
                 stop $port &
             done
             wait
+            rm -rf $basedir
         fi
         ;;
     restart)
@@ -78,6 +90,27 @@ case "$1" in
             wait
         fi
         ;;
+    unix)
+        shift
+        unix $@
+        ;;
+    startall)
+        unix &
+        for port in $PORTS; do
+            start $port &
+        done
+        wait
+        ;;
+    stopall)
+        if [ `ls $basedir/var/run/ | grep -c .pid` -ge 1 ]; then
+            names="`basename $basedir/var/run/*.pid | cut -d. -f1`"
+            for name in $names; do
+                stop $name &
+            done
+        fi
+        wait
+        rm -rf $basedir
+        ;;
     *)
         printf 'Usage: %s {start|stop|restart} <port>\n' "$prog"
         exit 1

diff --git a/misc/travis/cpptest.sh b/misc/travis/cpptest.sh
@@ -2,7 +2,7 @@
 set -ex
 
 echo "CXX=${CXX}"
-./misc/memcached_server start &>/dev/null &
+./misc/memcached_server startall &>/dev/null &
 python misc/generate_hash_dataset.py tests/resources/keys.txt &>/dev/null &
 mkdir -p build
 cd build
@@ -11,4 +11,4 @@ make -j8 &>/dev/null
 wait
 ARGS=-V make test
 cd ..
-./misc/memcached_server stop &>/dev/null &
+./misc/memcached_server stopall &>/dev/null &
diff --git a/src/Common.cpp b/src/Common.cpp
@@ -50,5 +50,48 @@ const char* errCodeToString(err_code_t err) {
   }
 }
 
+bool isLocalSocket(const char* host) {
+  // errors on the side of false negatives, allowing syntax expansion;
+  // starting slash to denote socket paths is from pylibmc
+  return host[0] == '/';
+}
+
+// modifies input string and output pointers reference input
+server_string_split_t splitServerString(char* input) {
+  bool escaped = false;
+  server_string_split_t res = { input, NULL, NULL };
+  for (;; input++) {
+    switch (*input)
+    {
+      case '\0':
+        return res;
+      case ':':
+        if (res.alias == NULL) {
+          *input = '\0';
+          if (res.port == NULL) {
+            res.port = input + 1;
+          }
+        }
+        escaped = false;
+        continue;
+      case ' ':
+        if (!escaped) {
+          *input = '\0';
+          if (res.alias == NULL) {
+            res.alias = input + 1;
+            continue;
+          } else {
+            return res;
+          }
+        }
+      default:
+        escaped = false;
+        continue;
+      case '\\':
+        escaped ^= 1;
+    }
+  }
+}
+
 } // namespace mc
 } // namespace douban