[SYSTEMDS-3426] Python NN Builtin (Affine,Relu)

This commit adds the new interface for easy usage of our neural network
in python. The design take inspiration from other neural network
frameworks.
This specific commit contains the building blocks of Affine and Relu.

Closes apache#1848

Co-authored-by: Duc Thai Vu <[email protected]>
Co-authored-by: Rahul Joshi <[email protected]>

...

src/main/python/systemds/operator/algorithm/builtin/pageRank.py

@@ -30,9 +30,6 @@
 
 
 def pageRank(G: Matrix,
-             p: Matrix,
-             e: Matrix,
-             u: Matrix,
              **kwargs: Dict[str, VALID_INPUT_TYPES]):
     """
      DML builtin method for PageRank algorithm (power iterations)
@@ -41,14 +38,16 @@ def pageRank(G: Matrix,
     
     :param G: Input Matrix
     :param p: initial page rank vector (number of nodes), e.g., rand intialized
+        default rand initialized with seed
     :param e: additional customization, default vector of ones
-    :param u: personalization vector (number of nodes)
+    :param u: personalization vector (number of nodes), default vector of ones
     :param alpha: teleport probability
     :param max_iter: maximum number of iterations
+    :param seed: seed for default rand initialization of page rank vector
     :return: computed pagerank
     """
 
-    params_dict = {'G': G, 'p': p, 'e': e, 'u': u}
+    params_dict = {'G': G}
     params_dict.update(kwargs)
     return Matrix(G.sds_context,
         'pageRank',

src/main/python/systemds/operator/nn/__init__.py

@@ -0,0 +1,20 @@
+# -------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# -------------------------------------------------------------

src/main/python/systemds/operator/nn/affine.py

@@ -0,0 +1,114 @@
+# -------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# -------------------------------------------------------------
+import os
+
+from systemds.context import SystemDSContext
+from systemds.operator import Matrix, Source, MultiReturn
+from systemds.utils.helpers import get_path_to_script_layers
+
+
+class Affine:
+    _source: Source = None
+    weight: Matrix
+    bias: Matrix
+
+    def __new__(cls, *args, **kwargs):
+        return super().__new__(cls)
+
+    def __init__(self, sds_context: SystemDSContext, d, m, seed=-1):
+        """
+        sds_context: The systemdsContext to construct the layer inside of
+        d: The number of features that are input to the affine layer
+        m: The number of neurons that are contained in the layer, 
+            and the number of features output
+        """
+        Affine._create_source(sds_context)
+
+        # bypassing overload limitation in python
+        self.forward = self._instance_forward
+        self.backward = self._instance_backward
+
+        # init weight and bias
+        self.weight = Matrix(sds_context, '')
+        self.bias = Matrix(sds_context, '')
+        params_dict = {'D': d, 'M': m, 'seed': seed}
+        out = [self.weight, self.bias]
+        op = MultiReturn(sds_context, "affine::init", output_nodes=out, named_input_nodes=params_dict)
+        self.weight._unnamed_input_nodes = [op]
+        self.bias._unnamed_input_nodes = [op]
+        op._source_node = self._source
+
+    @staticmethod
+    def forward(X: Matrix, W: Matrix, b: Matrix):
+        """
+        X: An input matrix
+        W: The hidden weights for the affine layer
+        b: The bias added in the output.
+        return out: An output matrix.
+        """
+        Affine._create_source(X.sds_context)
+        return Affine._source.forward(X, W, b)
+
+    @staticmethod
+    def backward(dout:Matrix, X: Matrix, W: Matrix, b: Matrix):
+        """
+        dout: The gradient of the output, passed from the upstream
+        X: The input matrix of this layer
+        W: The hidden weights for the affine layer
+        b: The bias added in the output
+        return dX, dW, db: The gradients of: input X, weights and bias.
+        """
+        sds = X.sds_context
+        Affine._create_source(sds)
+        params_dict = {'dout': dout, 'X': X, 'W': W, 'b': b}
+        dX = Matrix(sds, '')
+        dW = Matrix(sds, '')
+        db = Matrix(sds, '')
+        out = [dX, dW, db]
+        op = MultiReturn(sds, "affine::backward", output_nodes=out, named_input_nodes=params_dict)
+        dX._unnamed_input_nodes = [op]
+        dW._unnamed_input_nodes = [op]
+        db._unnamed_input_nodes = [op]
+        op._source_node = Affine._source
+        return op
+
+    def _instance_forward(self, X: Matrix):
+        """
+        X: The input matrix
+        return out: The output matrix
+        """
+        self._X = X
+        return Affine.forward(X, self.weight, self.bias)
+
+    def _instance_backward(self, dout: Matrix, X: Matrix):
+        """
+        dout: The gradient of the output, passed from the upstream layer
+        X: The input to this layer.
+        return dX, dW,db: gradient of input, weights and bias, respectively
+        """
+        return Affine.backward(dout, X, self.weight, self.bias)
+
+    @staticmethod
+    def _create_source(sds: SystemDSContext):
+        if Affine._source is None or Affine._source.sds_context != sds:
+            path = get_path_to_script_layers()
+            path = os.path.join(path, "affine.dml")
+            Affine._source = sds.source(path, "affine")

src/main/python/systemds/operator/nn/relu.py

@@ -0,0 +1,68 @@
+# -------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# -------------------------------------------------------------
+import os.path
+
+from systemds.context import SystemDSContext
+from systemds.operator import Matrix, Source
+from systemds.utils.helpers import get_path_to_script_layers
+
+
+class ReLU:
+    _source: Source = None
+
+    def __init__(self, sds: SystemDSContext):
+        ReLU._create_source(sds)
+        self.forward = self._instance_forward
+        self.backward = self._instance_backward
+
+    @staticmethod
+    def forward(X: Matrix):
+        """
+        X: input matrix
+        return out: output matrix
+        """
+        ReLU._create_source(X.sds_context)
+        return ReLU._source.forward(X)
+
+    @staticmethod
+    def backward(dout: Matrix, X: Matrix):
+        """
+        dout: gradient of output, passed from the upstream
+        X: input matrix
+        return dX: gradient of input
+        """
+        ReLU._create_source(dout.sds_context)
+        return ReLU._source.backward(dout, X)
+
+    def _instance_forward(self, X: Matrix):
+        self._X = X
+        return ReLU.forward(X)
+
+    def _instance_backward(self, dout: Matrix, X: Matrix):
+        return ReLU.backward(dout, X)
+
+    @staticmethod
+    def _create_source(sds: SystemDSContext):
+        if ReLU._source is None or ReLU._source.sds_context != sds:
+            path = get_path_to_script_layers()
+            path = os.path.join(path, "relu.dml")
+            ReLU._source = sds.source(path, "relu")
+

src/main/python/systemds/operator/nodes/source.py

@@ -21,6 +21,8 @@
 
 __all__ = ["Source"]
 
+import platform
+
 from types import MethodType
 from typing import TYPE_CHECKING, Dict, Iterable, Sequence
 
@@ -142,6 +144,7 @@ def __init__(self, sds_context, path: str, name: str):
             func = f.get_func(sds_context, name)
             setattr(self, f._name, MethodType(func, self))
 
+
     def __parse_functions_from_script(self, path: str) -> Iterable[Func]:
         lines = self.__parse_lines_with_filter(path)
         functions = []
@@ -162,10 +165,18 @@ def __parse_lines_with_filter(self, path: str) -> Iterable[str]:
         lines = []
         with open(path) as file:
             insideBracket = 0
+            insideComment = False
             for l in file.readlines():
                 ls = l.strip()
                 if len(ls) == 0 or ls[0] == '#':
                     continue
+                elif insideComment:
+                    if ls.endswith("*/"):
+                        insideComment = False
+                        continue
+                elif ls.startswith("/*"):
+                    insideComment = True
+                    continue
                 elif insideBracket > 0:
                     for c in ls:
                         if c == '{':
@@ -193,7 +204,11 @@ def __parse_lines_with_filter(self, path: str) -> Iterable[str]:
         return filtered_lines
 
     def code_line(self, var_name: str, unnamed_input_vars: Sequence[str], named_input_vars: Dict[str, str]) -> str:
-        line = f'source({self.operation}) as { self.__name}'
+        if platform.system() == 'Windows':
+            source_path = self.operation.replace("\\","\\\\")
+        else:
+            source_path = self.operation
+        line = f'source({source_path}) as { self.__name}'
         return line
 
     def compute(self, verbose: bool = False, lineage: bool = False):

src/main/python/systemds/utils/helpers.py

@@ -64,10 +64,10 @@ def get_slice_string(i):
             raise NotImplementedError("Not Implemented slice with dynamic end")
         else:
             # + 1 since R and systemDS is 1 indexed.
-            return f'{i.start+1}:{i.stop}'
+            return f'{i.start + 1}:{i.stop}'
     else:
         # + 1 since R and systemDS is 1 indexed.
-        sliceIns = i+1
+        sliceIns = i + 1
     return sliceIns
 
 
@@ -77,5 +77,12 @@ def check_is_empty_slice(i):
 
 def check_no_less_than_zero(i: list):
     for x in i:
-        if(x < 0):
+        if (x < 0):
             raise ValueError("Negative index not supported in systemds")
+
+
+def get_path_to_script_layers() -> str:
+    root = os.environ.get("SYSTEMDS_ROOT")
+    if root is None:
+        root = get_module_dir()
+    return os.path.join(root, "scripts", "nn", "layers")

src/main/python/tests/nn/__init__.py

@@ -0,0 +1,20 @@
+# -------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# -------------------------------------------------------------